vanity 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/Appraisals +6 -6
- data/CHANGELOG +9 -3
- data/Gemfile.lock +1 -1
- data/README.md +299 -0
- data/doc/configuring.textile +8 -1
- data/doc/identity.textile +2 -0
- data/doc/metrics.textile +10 -0
- data/gemfiles/rails32.gemfile.lock +1 -1
- data/gemfiles/rails41.gemfile.lock +1 -1
- data/gemfiles/rails42.gemfile.lock +1 -1
- data/gemfiles/{rails4.gemfile → rails42_protected_attributes.gemfile} +2 -2
- data/gemfiles/rails42_protected_attributes.gemfile.lock +209 -0
- data/lib/generators/templates/vanity_migration.rb +1 -0
- data/lib/vanity/adapters/abstract_adapter.rb +11 -0
- data/lib/vanity/adapters/active_record_adapter.rb +15 -1
- data/lib/vanity/adapters/mock_adapter.rb +14 -0
- data/lib/vanity/adapters/mongodb_adapter.rb +14 -0
- data/lib/vanity/adapters/redis_adapter.rb +15 -0
- data/lib/vanity/configuration.rb +43 -11
- data/lib/vanity/experiment/ab_test.rb +145 -15
- data/lib/vanity/experiment/alternative.rb +4 -0
- data/lib/vanity/frameworks/rails.rb +69 -31
- data/lib/vanity/locales/vanity.en.yml +9 -0
- data/lib/vanity/locales/vanity.pt-BR.yml +4 -0
- data/lib/vanity/metric/active_record.rb +9 -1
- data/lib/vanity/templates/_ab_test.erb +9 -2
- data/lib/vanity/templates/_experiment.erb +21 -1
- data/lib/vanity/templates/vanity.css +11 -3
- data/lib/vanity/templates/vanity.js +35 -6
- data/lib/vanity/version.rb +1 -1
- data/test/commands/report_test.rb +1 -0
- data/test/dummy/config/application.rb +1 -0
- data/test/experiment/ab_test.rb +414 -0
- data/test/experiment/base_test.rb +16 -10
- data/test/frameworks/rails/action_controller_test.rb +14 -6
- data/test/frameworks/rails/action_mailer_test.rb +8 -6
- data/test/frameworks/rails/action_view_test.rb +1 -0
- data/test/helper_test.rb +2 -0
- data/test/metric/active_record_test.rb +56 -0
- data/test/playground_test.rb +3 -0
- data/test/test_helper.rb +28 -2
- data/test/web/rails/dashboard_test.rb +2 -0
- data/vanity.gemspec +2 -2
- metadata +8 -8
- data/README.rdoc +0 -231
- data/gemfiles/rails4.gemfile.lock +0 -179
data/test/experiment/ab_test.rb
CHANGED
@@ -37,15 +37,18 @@ class AbTestTest < ActionController::TestCase
|
|
37
37
|
assert_raises RuntimeError do
|
38
38
|
new_ab_test :none do
|
39
39
|
alternatives []
|
40
|
+
default nil
|
40
41
|
end
|
41
42
|
end
|
42
43
|
assert_raises RuntimeError do
|
43
44
|
new_ab_test :one do
|
44
45
|
alternatives "foo"
|
46
|
+
default "foo"
|
45
47
|
end
|
46
48
|
end
|
47
49
|
new_ab_test :two do
|
48
50
|
alternatives "foo", "bar"
|
51
|
+
default "foo"
|
49
52
|
metrics :coolness
|
50
53
|
end
|
51
54
|
end
|
@@ -53,6 +56,7 @@ class AbTestTest < ActionController::TestCase
|
|
53
56
|
def test_returning_alternative_by_value
|
54
57
|
new_ab_test :abcd do
|
55
58
|
alternatives :a, :b, :c, :d
|
59
|
+
default :a
|
56
60
|
metrics :coolness
|
57
61
|
end
|
58
62
|
assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
|
@@ -62,6 +66,7 @@ class AbTestTest < ActionController::TestCase
|
|
62
66
|
def test_alternative_name
|
63
67
|
new_ab_test :abcd do
|
64
68
|
alternatives :a, :b
|
69
|
+
default :a
|
65
70
|
metrics :coolness
|
66
71
|
end
|
67
72
|
assert_equal "option A", experiment(:abcd).alternative(:a).name
|
@@ -72,10 +77,12 @@ class AbTestTest < ActionController::TestCase
|
|
72
77
|
new_ab_test :ab do
|
73
78
|
metrics :coolness
|
74
79
|
alternatives :a, :b
|
80
|
+
default :a
|
75
81
|
end
|
76
82
|
new_ab_test :cd do
|
77
83
|
metrics :coolness
|
78
84
|
alternatives :a, :b
|
85
|
+
default :a
|
79
86
|
end
|
80
87
|
fingerprints = Vanity.playground.experiments.map { |id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
81
88
|
assert_equal 4, fingerprints.uniq.size
|
@@ -84,6 +91,7 @@ class AbTestTest < ActionController::TestCase
|
|
84
91
|
def test_alternative_fingerprint_is_consistent
|
85
92
|
new_ab_test :ab do
|
86
93
|
alternatives :a, :b
|
94
|
+
default :a
|
87
95
|
metrics :coolness
|
88
96
|
end
|
89
97
|
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
@@ -93,18 +101,305 @@ class AbTestTest < ActionController::TestCase
|
|
93
101
|
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
94
102
|
end
|
95
103
|
|
104
|
+
def test_ab_has_default
|
105
|
+
new_ab_test :ice_cream_flavor do
|
106
|
+
metrics :coolness
|
107
|
+
alternatives :a, :b, :c
|
108
|
+
default :b
|
109
|
+
end
|
110
|
+
exp = experiment(:ice_cream_flavor)
|
111
|
+
assert_equal exp.default, exp.alternative(:b)
|
112
|
+
end
|
113
|
+
|
114
|
+
def test_ab_sets_default_default
|
115
|
+
new_ab_test :ice_cream_flavor do
|
116
|
+
metrics :coolness
|
117
|
+
alternatives :a, :b, :c
|
118
|
+
# no default specified
|
119
|
+
end
|
120
|
+
exp = experiment(:ice_cream_flavor)
|
121
|
+
assert_equal exp.default, exp.alternative(:a)
|
122
|
+
end
|
123
|
+
|
124
|
+
def test_ab_overrides_unknown_default
|
125
|
+
new_ab_test :ice_cream_flavor do
|
126
|
+
metrics :coolness
|
127
|
+
alternatives :a, :b, :c
|
128
|
+
default :badname
|
129
|
+
end
|
130
|
+
exp = experiment(:ice_cream_flavor)
|
131
|
+
assert_equal exp.default, exp.alternative(:a)
|
132
|
+
end
|
133
|
+
|
134
|
+
def test_ab_can_only_set_default_once
|
135
|
+
assert_raise ArgumentError do
|
136
|
+
new_ab_test :ice_cream_flavor do
|
137
|
+
metrics :coolness
|
138
|
+
alternative :a, :b, :c
|
139
|
+
default :a
|
140
|
+
default :b
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_ab_can_only_have_one_default
|
146
|
+
assert_raise ArgumentError do
|
147
|
+
new_ab_test :ice_cream_flavor do
|
148
|
+
metrics :coolness
|
149
|
+
alternative :a, :b, :c
|
150
|
+
default :a, :b
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_ab_cannot_get_default_before_specified
|
156
|
+
assert_raise ArgumentError do
|
157
|
+
new_ab_test :ice_cream_flavor do
|
158
|
+
metrics :coolness
|
159
|
+
alternative :a, :b, :c
|
160
|
+
default
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
def test_ab_accepts_nil_default
|
166
|
+
new_ab_test :nil_default do
|
167
|
+
metrics :coolness
|
168
|
+
alternatives nil, 'foo'
|
169
|
+
default nil
|
170
|
+
end
|
171
|
+
exp = experiment(:nil_default)
|
172
|
+
assert_equal exp.default, exp.alternative(nil)
|
173
|
+
end
|
174
|
+
|
175
|
+
def test_ab_chooses_nil_default_default
|
176
|
+
new_ab_test :nil_default_default do
|
177
|
+
metrics :coolness
|
178
|
+
alternatives nil, 'foo'
|
179
|
+
# no default specified
|
180
|
+
end
|
181
|
+
exp = experiment(:nil_default_default)
|
182
|
+
assert_equal exp.default, exp.alternative(nil)
|
183
|
+
end
|
184
|
+
|
185
|
+
|
186
|
+
# -- Experiment Enabled/disabled --
|
187
|
+
|
188
|
+
# @example new test should be enabled regardless of collecting?
|
189
|
+
# regardless_of "Vanity.playground.collecting" do
|
190
|
+
# assert (new_ab_test :test).enabled?
|
191
|
+
# end
|
192
|
+
def regardless_of(attr_name, &block)
|
193
|
+
prev_val = eval "#{attr_name}?"
|
194
|
+
|
195
|
+
eval "#{attr_name}=true"
|
196
|
+
block.call(eval "#{attr_name}?")
|
197
|
+
nuke_playground
|
198
|
+
|
199
|
+
eval "#{attr_name}=false"
|
200
|
+
block.call(eval "#{attr_name}?")
|
201
|
+
nuke_playground
|
202
|
+
|
203
|
+
eval "#{attr_name}=prev_val"
|
204
|
+
end
|
205
|
+
|
206
|
+
def test_new_test_is_disabled_when_experiments_start_enabled_is_false
|
207
|
+
Vanity.configuration.experiments_start_enabled = false
|
208
|
+
exp = new_ab_test :test, enable: false do
|
209
|
+
metrics :coolness
|
210
|
+
default false
|
211
|
+
end
|
212
|
+
assert !exp.enabled?
|
213
|
+
end
|
214
|
+
|
215
|
+
def test_new_test_is_enabled_when_experiments_start_enabled_is_true
|
216
|
+
Vanity.configuration.experiments_start_enabled = true
|
217
|
+
exp = new_ab_test :test, enable: false do
|
218
|
+
metrics :coolness
|
219
|
+
default false
|
220
|
+
end
|
221
|
+
assert exp.enabled?
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_complete_sets_enabled_false
|
225
|
+
Vanity.playground.collecting = true
|
226
|
+
exp = new_ab_test :test do
|
227
|
+
metrics :coolness
|
228
|
+
default false
|
229
|
+
end
|
230
|
+
exp.complete! #active? => false
|
231
|
+
|
232
|
+
assert !exp.enabled?, "experiment should not be enabled but it is!"
|
233
|
+
end
|
234
|
+
|
235
|
+
def test_complete_keeps_enabled_true_while_not_collecting
|
236
|
+
exp = new_ab_test :test do
|
237
|
+
metrics :coolness
|
238
|
+
default false
|
239
|
+
end
|
240
|
+
Vanity.playground.collecting = false
|
241
|
+
exp.enabled = false
|
242
|
+
assert exp.enabled?
|
243
|
+
end
|
244
|
+
|
245
|
+
def test_set_enabled_while_active
|
246
|
+
Vanity.playground.collecting = true
|
247
|
+
exp = new_ab_test :test do
|
248
|
+
metrics :coolness
|
249
|
+
default false
|
250
|
+
end
|
251
|
+
|
252
|
+
exp.enabled = true
|
253
|
+
assert exp.enabled?
|
254
|
+
|
255
|
+
exp.enabled = false
|
256
|
+
assert !exp.enabled?
|
257
|
+
end
|
258
|
+
|
259
|
+
def test_cannot_set_enabled_for_inactive
|
260
|
+
Vanity.playground.collecting = true
|
261
|
+
exp = new_ab_test :test do
|
262
|
+
metrics :coolness
|
263
|
+
default false
|
264
|
+
end
|
265
|
+
exp.complete! #active? => false
|
266
|
+
assert !exp.enabled?
|
267
|
+
exp.enabled = true
|
268
|
+
assert !exp.enabled?
|
269
|
+
end
|
270
|
+
|
271
|
+
def test_always_enabled_while_not_collecting
|
272
|
+
Vanity.playground.collecting = false
|
273
|
+
exp = new_ab_test :test do
|
274
|
+
metrics :coolness
|
275
|
+
default false
|
276
|
+
end
|
277
|
+
assert exp.enabled?
|
278
|
+
exp.enabled = false
|
279
|
+
assert exp.enabled?
|
280
|
+
end
|
281
|
+
|
282
|
+
def test_enabled_persists_across_definitions
|
283
|
+
Vanity.configuration.experiments_start_enabled = false
|
284
|
+
Vanity.playground.collecting = true
|
285
|
+
new_ab_test :test, :enable => false do
|
286
|
+
metrics :coolness
|
287
|
+
default false
|
288
|
+
end
|
289
|
+
assert !experiment(:test).enabled? #starts off false
|
290
|
+
|
291
|
+
new_playground
|
292
|
+
metric "Coolness"
|
293
|
+
|
294
|
+
new_ab_test :test, :enable => false do
|
295
|
+
metrics :coolness
|
296
|
+
default false
|
297
|
+
end
|
298
|
+
assert !experiment(:test).enabled? #still false
|
299
|
+
experiment(:test).enabled = true
|
300
|
+
assert experiment(:test).enabled? #now true
|
301
|
+
|
302
|
+
new_playground
|
303
|
+
metric "Coolness"
|
304
|
+
|
305
|
+
new_ab_test :test, :enable => false do
|
306
|
+
metrics :coolness
|
307
|
+
default false
|
308
|
+
end
|
309
|
+
assert experiment(:test).enabled? #still true
|
310
|
+
experiment(:test).enabled = false
|
311
|
+
assert !experiment(:test).enabled? #now false again
|
312
|
+
end
|
96
313
|
|
314
|
+
def test_enabled_persists_across_definitions_when_starting_enabled
|
315
|
+
Vanity.configuration.experiments_start_enabled = true
|
316
|
+
Vanity.playground.collecting = true
|
317
|
+
new_ab_test :test, :enable => false do
|
318
|
+
metrics :coolness
|
319
|
+
default false
|
320
|
+
end
|
321
|
+
assert experiment(:test).enabled? #starts off true
|
322
|
+
|
323
|
+
new_playground
|
324
|
+
metric "Coolness"
|
325
|
+
|
326
|
+
new_ab_test :test, :enable => false do
|
327
|
+
metrics :coolness
|
328
|
+
default false
|
329
|
+
end
|
330
|
+
assert experiment(:test).enabled? #still true
|
331
|
+
experiment(:test).enabled = false
|
332
|
+
assert !experiment(:test).enabled? #now false
|
333
|
+
|
334
|
+
new_playground
|
335
|
+
metric "Coolness"
|
336
|
+
|
337
|
+
new_ab_test :test, :enable => false do
|
338
|
+
metrics :coolness
|
339
|
+
default false
|
340
|
+
end
|
341
|
+
assert !experiment(:test).enabled? #still false
|
342
|
+
experiment(:test).enabled = true
|
343
|
+
assert experiment(:test).enabled? #now true again
|
344
|
+
end
|
345
|
+
|
346
|
+
def test_choose_random_when_enabled
|
347
|
+
regardless_of "Vanity.playground.collecting" do
|
348
|
+
metric "Coolness"
|
349
|
+
|
350
|
+
exp = new_ab_test :test do
|
351
|
+
metrics :coolness
|
352
|
+
true_false
|
353
|
+
default false
|
354
|
+
identify { rand }
|
355
|
+
end
|
356
|
+
results = Set.new
|
357
|
+
100.times do
|
358
|
+
results << exp.choose.value
|
359
|
+
end
|
360
|
+
assert_equal results, [true, false].to_set
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
def test_choose_default_when_disabled
|
365
|
+
exp = new_ab_test :test do
|
366
|
+
metrics :coolness
|
367
|
+
alternatives 0, 1, 2, 3, 4, 5
|
368
|
+
default 3
|
369
|
+
end
|
370
|
+
|
371
|
+
exp.enabled = false
|
372
|
+
100.times.each do
|
373
|
+
assert_equal 3, exp.choose.value
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
377
|
+
def test_choose_outcome_when_finished
|
378
|
+
exp = new_ab_test :test do
|
379
|
+
metrics :coolness
|
380
|
+
alternatives 0,1,2,3,4,5
|
381
|
+
default 3
|
382
|
+
outcome_is { alternative(5) }
|
383
|
+
end
|
384
|
+
exp.complete!
|
385
|
+
100.times.each do
|
386
|
+
assert_equal 5, exp.choose.value
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
97
390
|
# -- Experiment metric --
|
98
391
|
|
99
392
|
def test_explicit_metric
|
100
393
|
new_ab_test :abcd do
|
101
394
|
metrics :coolness
|
395
|
+
default false
|
102
396
|
end
|
103
397
|
assert_equal [Vanity.playground.metric(:coolness)], experiment(:abcd).metrics
|
104
398
|
end
|
105
399
|
|
106
400
|
def test_implicit_metric
|
107
401
|
new_ab_test :abcd do
|
402
|
+
default false
|
108
403
|
end
|
109
404
|
assert_equal [Vanity.playground.metric(:abcd)], experiment(:abcd).metrics
|
110
405
|
end
|
@@ -113,6 +408,7 @@ class AbTestTest < ActionController::TestCase
|
|
113
408
|
metric "Coolness"
|
114
409
|
new_ab_test :abcd do
|
115
410
|
metrics :coolness
|
411
|
+
default false
|
116
412
|
end
|
117
413
|
Vanity.playground.track! :coolness
|
118
414
|
assert_equal 1, experiment(:abcd).alternatives.sum(&:conversions)
|
@@ -123,6 +419,7 @@ class AbTestTest < ActionController::TestCase
|
|
123
419
|
def test_track_with_identity_overrides_default
|
124
420
|
identities = ["quux"]
|
125
421
|
new_ab_test :foobar do
|
422
|
+
default "foo"
|
126
423
|
alternatives "foo", "bar"
|
127
424
|
identify { identities.pop || "6e98ec" }
|
128
425
|
metrics :coolness
|
@@ -141,6 +438,7 @@ class AbTestTest < ActionController::TestCase
|
|
141
438
|
Vanity.configuration.use_js = true
|
142
439
|
ids = (0...10).to_a
|
143
440
|
new_ab_test :foobar do
|
441
|
+
default "foo"
|
144
442
|
alternatives "foo", "bar"
|
145
443
|
identify { ids.pop }
|
146
444
|
metrics :coolness
|
@@ -155,6 +453,7 @@ class AbTestTest < ActionController::TestCase
|
|
155
453
|
def test_calls_on_assignment_on_new_assignment
|
156
454
|
on_assignment_called_times = 0
|
157
455
|
new_ab_test :foobar do
|
456
|
+
default "foo"
|
158
457
|
alternatives "foo", "bar"
|
159
458
|
identify { "6e98ec" }
|
160
459
|
metrics :coolness
|
@@ -167,6 +466,7 @@ class AbTestTest < ActionController::TestCase
|
|
167
466
|
def test_calls_on_assignment_when_given_valid_request
|
168
467
|
on_assignment_called_times = 0
|
169
468
|
new_ab_test :foobar do
|
469
|
+
default "foo"
|
170
470
|
alternatives "foo", "bar"
|
171
471
|
identify { "6e98ec" }
|
172
472
|
metrics :coolness
|
@@ -179,6 +479,7 @@ class AbTestTest < ActionController::TestCase
|
|
179
479
|
def test_does_not_call_on_assignment_when_given_invalid_request
|
180
480
|
on_assignment_called_times = 0
|
181
481
|
new_ab_test :foobar do
|
482
|
+
default "foo"
|
182
483
|
alternatives "foo", "bar"
|
183
484
|
identify { "6e98ec" }
|
184
485
|
metrics :coolness
|
@@ -194,6 +495,7 @@ class AbTestTest < ActionController::TestCase
|
|
194
495
|
on_assignment_called_times = 0
|
195
496
|
new_ab_test :foobar do
|
196
497
|
alternatives "foo", "bar"
|
498
|
+
default "foo"
|
197
499
|
identify { "6e98ec" }
|
198
500
|
metrics :coolness
|
199
501
|
on_assignment { on_assignment_called_times = on_assignment_called_times+1 }
|
@@ -205,6 +507,7 @@ class AbTestTest < ActionController::TestCase
|
|
205
507
|
def test_returns_the_same_alternative_consistently_when_on_assignment_is_set
|
206
508
|
new_ab_test :foobar do
|
207
509
|
alternatives "foo", "bar"
|
510
|
+
default "foo"
|
208
511
|
identify { "6e98ec" }
|
209
512
|
on_assignment {}
|
210
513
|
metrics :coolness
|
@@ -221,6 +524,7 @@ class AbTestTest < ActionController::TestCase
|
|
221
524
|
def test_ab_assigned
|
222
525
|
new_ab_test :foobar do
|
223
526
|
alternatives "foo", "bar"
|
527
|
+
default "foo"
|
224
528
|
identify { "6e98ec" }
|
225
529
|
metrics :coolness
|
226
530
|
end
|
@@ -233,6 +537,7 @@ class AbTestTest < ActionController::TestCase
|
|
233
537
|
identity = { :a => :b }
|
234
538
|
new_ab_test :foobar do
|
235
539
|
alternatives "foo", "bar"
|
540
|
+
default "foo"
|
236
541
|
identify { identity }
|
237
542
|
metrics :coolness
|
238
543
|
end
|
@@ -246,6 +551,7 @@ class AbTestTest < ActionController::TestCase
|
|
246
551
|
def test_returns_the_same_alternative_consistently_when_using_probabilities
|
247
552
|
new_ab_test :foobar do
|
248
553
|
alternatives "foo", "bar"
|
554
|
+
default "foo"
|
249
555
|
identify { "6e98ec" }
|
250
556
|
rebalance_frequency 10
|
251
557
|
metrics :coolness
|
@@ -258,9 +564,21 @@ class AbTestTest < ActionController::TestCase
|
|
258
564
|
end
|
259
565
|
end
|
260
566
|
|
567
|
+
def test_uses_configured_probabilities_for_new_assignments
|
568
|
+
new_ab_test :foobar do
|
569
|
+
alternatives "foo" => 30, "bar" => 60
|
570
|
+
identify { rand }
|
571
|
+
metrics :coolness
|
572
|
+
end
|
573
|
+
alts = Array.new(10_000) { experiment(:foobar).choose.value }.reduce({}) { |h,k| h[k] ||= 0; h[k] += 1; h }
|
574
|
+
assert_equal %w{bar foo}, alts.keys.sort
|
575
|
+
assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
|
576
|
+
end
|
577
|
+
|
261
578
|
def test_uses_probabilities_for_new_assignments
|
262
579
|
new_ab_test :foobar do
|
263
580
|
alternatives "foo", "bar"
|
581
|
+
default "foo"
|
264
582
|
identify { rand }
|
265
583
|
rebalance_frequency 10000
|
266
584
|
metrics :coolness
|
@@ -279,6 +597,7 @@ class AbTestTest < ActionController::TestCase
|
|
279
597
|
def test_rebalances_probabilities_after_rebalance_frequency_calls
|
280
598
|
new_ab_test :foobar do
|
281
599
|
alternatives "foo", "bar"
|
600
|
+
default "foo"
|
282
601
|
identify { rand }
|
283
602
|
rebalance_frequency 12
|
284
603
|
metrics :coolness
|
@@ -302,6 +621,7 @@ class AbTestTest < ActionController::TestCase
|
|
302
621
|
def test_rebalance_uses_bayes_score_probabilities_to_update_probabilities
|
303
622
|
new_ab_test :foobar do
|
304
623
|
alternatives "foo", "bar", "baa"
|
624
|
+
default "foo"
|
305
625
|
identify { rand }
|
306
626
|
rebalance_frequency 12
|
307
627
|
metrics :coolness
|
@@ -334,6 +654,7 @@ class AbTestTest < ActionController::TestCase
|
|
334
654
|
def test_returns_the_same_alternative_consistently
|
335
655
|
new_ab_test :foobar do
|
336
656
|
alternatives "foo", "bar"
|
657
|
+
default "foo"
|
337
658
|
identify { "6e98ec" }
|
338
659
|
metrics :coolness
|
339
660
|
end
|
@@ -347,6 +668,7 @@ class AbTestTest < ActionController::TestCase
|
|
347
668
|
def test_respects_out_of_band_assignment
|
348
669
|
new_ab_test :foobar do
|
349
670
|
alternatives "a", "b", "c"
|
671
|
+
default "a"
|
350
672
|
identify { "6e98ec" }
|
351
673
|
metrics :coolness
|
352
674
|
end
|
@@ -365,6 +687,7 @@ class AbTestTest < ActionController::TestCase
|
|
365
687
|
def test_returns_different_alternatives_for_each_participant
|
366
688
|
new_ab_test :foobar do
|
367
689
|
alternatives "foo", "bar"
|
690
|
+
default "foo"
|
368
691
|
identify { rand }
|
369
692
|
metrics :coolness
|
370
693
|
end
|
@@ -377,6 +700,7 @@ class AbTestTest < ActionController::TestCase
|
|
377
700
|
ids = (Array.new(200) { |i| i } * 5).shuffle
|
378
701
|
new_ab_test :foobar do
|
379
702
|
alternatives "foo", "bar"
|
703
|
+
default "foo"
|
380
704
|
identify { ids.pop }
|
381
705
|
metrics :coolness
|
382
706
|
end
|
@@ -390,6 +714,7 @@ class AbTestTest < ActionController::TestCase
|
|
390
714
|
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
391
715
|
new_ab_test :foobar do
|
392
716
|
alternatives "foo", "bar"
|
717
|
+
default "foo"
|
393
718
|
identify { ids.pop }
|
394
719
|
metrics :coolness
|
395
720
|
end
|
@@ -405,6 +730,7 @@ class AbTestTest < ActionController::TestCase
|
|
405
730
|
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
406
731
|
new_ab_test :foobar do
|
407
732
|
alternatives "foo", "bar"
|
733
|
+
default "foo"
|
408
734
|
identify { ids.pop }
|
409
735
|
metrics :coolness
|
410
736
|
end
|
@@ -420,6 +746,7 @@ class AbTestTest < ActionController::TestCase
|
|
420
746
|
def test_choose_records_participants_given_a_valid_request
|
421
747
|
new_ab_test :foobar do
|
422
748
|
alternatives "foo", "bar"
|
749
|
+
default "foo"
|
423
750
|
identify { "me" }
|
424
751
|
metrics :coolness
|
425
752
|
end
|
@@ -430,6 +757,7 @@ class AbTestTest < ActionController::TestCase
|
|
430
757
|
def test_choose_ignores_participants_given_an_invalid_request
|
431
758
|
new_ab_test :foobar do
|
432
759
|
alternatives "foo", "bar"
|
760
|
+
default "foo"
|
433
761
|
identify { "me" }
|
434
762
|
metrics :coolness
|
435
763
|
end
|
@@ -443,6 +771,7 @@ class AbTestTest < ActionController::TestCase
|
|
443
771
|
new_ab_test :simple do
|
444
772
|
identify { "me" }
|
445
773
|
metrics :coolness
|
774
|
+
default false
|
446
775
|
complete_if { alternatives.map(&:converted).sum >= 1 }
|
447
776
|
outcome_is { alternative(true) }
|
448
777
|
end
|
@@ -472,6 +801,7 @@ class AbTestTest < ActionController::TestCase
|
|
472
801
|
def test_ab_test_chooses_in_render
|
473
802
|
new_ab_test :simple do
|
474
803
|
metrics :coolness
|
804
|
+
default false
|
475
805
|
end
|
476
806
|
responses = Array.new(100) do
|
477
807
|
@controller = nil ; setup_controller_request_and_response
|
@@ -484,6 +814,7 @@ class AbTestTest < ActionController::TestCase
|
|
484
814
|
def test_ab_test_chooses_view_helper
|
485
815
|
new_ab_test :simple do
|
486
816
|
metrics :coolness
|
817
|
+
default false
|
487
818
|
end
|
488
819
|
responses = Array.new(100) do
|
489
820
|
@controller = nil ; setup_controller_request_and_response
|
@@ -496,6 +827,7 @@ class AbTestTest < ActionController::TestCase
|
|
496
827
|
def test_ab_test_with_capture
|
497
828
|
new_ab_test :simple do
|
498
829
|
metrics :coolness
|
830
|
+
default false
|
499
831
|
end
|
500
832
|
responses = Array.new(100) do
|
501
833
|
@controller = nil ; setup_controller_request_and_response
|
@@ -508,6 +840,7 @@ class AbTestTest < ActionController::TestCase
|
|
508
840
|
def test_ab_test_track
|
509
841
|
new_ab_test :simple do
|
510
842
|
metrics :coolness
|
843
|
+
default false
|
511
844
|
end
|
512
845
|
responses = Array.new(100) do
|
513
846
|
@controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
|
@@ -522,6 +855,7 @@ class AbTestTest < ActionController::TestCase
|
|
522
855
|
def test_with_given_choice
|
523
856
|
new_ab_test :simple do
|
524
857
|
alternatives :a, :b, :c
|
858
|
+
default :a
|
525
859
|
metrics :coolness
|
526
860
|
end
|
527
861
|
100.times do |i|
|
@@ -535,6 +869,7 @@ class AbTestTest < ActionController::TestCase
|
|
535
869
|
def test_which_chooses_non_existent_alternative
|
536
870
|
new_ab_test :simple do
|
537
871
|
metrics :coolness
|
872
|
+
default false
|
538
873
|
end
|
539
874
|
assert_raises ArgumentError do
|
540
875
|
experiment(:simple).chooses(404)
|
@@ -545,6 +880,7 @@ class AbTestTest < ActionController::TestCase
|
|
545
880
|
new_ab_test :simple do
|
546
881
|
identify { rand }
|
547
882
|
alternatives :a, :b, :c
|
883
|
+
default :a
|
548
884
|
metrics :coolness
|
549
885
|
end
|
550
886
|
responses = Array.new(100) { |i|
|
@@ -563,6 +899,7 @@ class AbTestTest < ActionController::TestCase
|
|
563
899
|
def test_calculate_score
|
564
900
|
new_ab_test :abcd do
|
565
901
|
alternatives :a, :b, :c, :d
|
902
|
+
default :a
|
566
903
|
metrics :coolness
|
567
904
|
end
|
568
905
|
score_result = experiment(:abcd).calculate_score
|
@@ -570,6 +907,7 @@ class AbTestTest < ActionController::TestCase
|
|
570
907
|
|
571
908
|
new_ab_test :bayes_abcd do
|
572
909
|
alternatives :a, :b, :c, :d
|
910
|
+
default :a
|
573
911
|
metrics :coolness
|
574
912
|
score_method :bayes_bandit_score
|
575
913
|
end
|
@@ -580,6 +918,7 @@ class AbTestTest < ActionController::TestCase
|
|
580
918
|
def test_scoring
|
581
919
|
new_ab_test :abcd do
|
582
920
|
alternatives :a, :b, :c, :d
|
921
|
+
default :a
|
583
922
|
metrics :coolness
|
584
923
|
end
|
585
924
|
# participating, conversions, rate, z-score
|
@@ -606,6 +945,7 @@ class AbTestTest < ActionController::TestCase
|
|
606
945
|
def test_bayes_scoring
|
607
946
|
new_ab_test :abcd do
|
608
947
|
alternatives :a, :b, :c, :d
|
948
|
+
default :a
|
609
949
|
metrics :coolness
|
610
950
|
end
|
611
951
|
# participating, conversions, rate, z-score
|
@@ -623,6 +963,7 @@ class AbTestTest < ActionController::TestCase
|
|
623
963
|
def test_scoring_with_no_performers
|
624
964
|
new_ab_test :abcd do
|
625
965
|
alternatives :a, :b, :c, :d
|
966
|
+
default :a
|
626
967
|
metrics :coolness
|
627
968
|
end
|
628
969
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
@@ -636,6 +977,7 @@ class AbTestTest < ActionController::TestCase
|
|
636
977
|
def test_scoring_with_one_performer
|
637
978
|
new_ab_test :abcd do
|
638
979
|
alternatives :a, :b, :c, :d
|
980
|
+
default :a
|
639
981
|
metrics :coolness
|
640
982
|
end
|
641
983
|
fake :abcd, :b=>[10,8]
|
@@ -651,6 +993,7 @@ class AbTestTest < ActionController::TestCase
|
|
651
993
|
def test_scoring_with_some_performers
|
652
994
|
new_ab_test :abcd do
|
653
995
|
alternatives :a, :b, :c, :d
|
996
|
+
default :a
|
654
997
|
metrics :coolness
|
655
998
|
end
|
656
999
|
fake :abcd, :b=>[10,8], :d=>[12,5]
|
@@ -670,6 +1013,7 @@ class AbTestTest < ActionController::TestCase
|
|
670
1013
|
def test_scoring_with_different_probability
|
671
1014
|
new_ab_test :abcd do
|
672
1015
|
alternatives :a, :b, :c, :d
|
1016
|
+
default :a
|
673
1017
|
metrics :coolness
|
674
1018
|
end
|
675
1019
|
fake :abcd, :b=>[10,8], :d=>[12,5]
|
@@ -685,6 +1029,7 @@ class AbTestTest < ActionController::TestCase
|
|
685
1029
|
def test_conclusion
|
686
1030
|
new_ab_test :abcd do
|
687
1031
|
alternatives :a, :b, :c, :d
|
1032
|
+
default :a
|
688
1033
|
metrics :coolness
|
689
1034
|
end
|
690
1035
|
# participating, conversions, rate, z-score
|
@@ -708,6 +1053,7 @@ Option D selected as the best alternative.
|
|
708
1053
|
def test_conclusion_with_some_performers
|
709
1054
|
new_ab_test :abcd do
|
710
1055
|
alternatives :a, :b, :c, :d
|
1056
|
+
default :a
|
711
1057
|
metrics :coolness
|
712
1058
|
end
|
713
1059
|
fake :abcd, :b=>[180, 45], :d=>[188, 61]
|
@@ -726,6 +1072,7 @@ Option D selected as the best alternative.
|
|
726
1072
|
def test_conclusion_without_clear_winner
|
727
1073
|
new_ab_test :abcd do
|
728
1074
|
alternatives :a, :b, :c, :d
|
1075
|
+
default :a
|
729
1076
|
metrics :coolness
|
730
1077
|
end
|
731
1078
|
fake :abcd, :b=>[180, 58], :d=>[188, 61]
|
@@ -743,6 +1090,7 @@ Option C did not convert.
|
|
743
1090
|
def test_conclusion_without_close_performers
|
744
1091
|
new_ab_test :abcd do
|
745
1092
|
alternatives :a, :b, :c, :d
|
1093
|
+
default :a
|
746
1094
|
metrics :coolness
|
747
1095
|
end
|
748
1096
|
fake :abcd, :b=>[186, 60], :d=>[188, 61]
|
@@ -760,6 +1108,7 @@ Option C did not convert.
|
|
760
1108
|
def test_conclusion_without_equal_performers
|
761
1109
|
new_ab_test :abcd do
|
762
1110
|
alternatives :a, :b, :c, :d
|
1111
|
+
default :a
|
763
1112
|
metrics :coolness
|
764
1113
|
end
|
765
1114
|
fake :abcd, :b=>[188, 61], :d=>[188, 61]
|
@@ -776,6 +1125,7 @@ Option C did not convert.
|
|
776
1125
|
def test_conclusion_with_one_performers
|
777
1126
|
new_ab_test :abcd do
|
778
1127
|
alternatives :a, :b, :c, :d
|
1128
|
+
default :a
|
779
1129
|
metrics :coolness
|
780
1130
|
end
|
781
1131
|
fake :abcd, :b=>[180, 45]
|
@@ -789,6 +1139,7 @@ This experiment did not run long enough to find a clear winner.
|
|
789
1139
|
def test_conclusion_with_no_performers
|
790
1140
|
new_ab_test :abcd do
|
791
1141
|
alternatives :a, :b, :c, :d
|
1142
|
+
default :a
|
792
1143
|
metrics :coolness
|
793
1144
|
end
|
794
1145
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
@@ -805,6 +1156,7 @@ This experiment did not run long enough to find a clear winner.
|
|
805
1156
|
identify { rand }
|
806
1157
|
complete_if { true }
|
807
1158
|
metrics :coolness
|
1159
|
+
default false
|
808
1160
|
end
|
809
1161
|
experiment(:simple).choose
|
810
1162
|
assert !experiment(:simple).active?
|
@@ -815,6 +1167,7 @@ This experiment did not run long enough to find a clear winner.
|
|
815
1167
|
identify { rand }
|
816
1168
|
complete_if { fail "Testing complete_if raises exception" }
|
817
1169
|
metrics :coolness
|
1170
|
+
default false
|
818
1171
|
end
|
819
1172
|
experiment(:simple).choose
|
820
1173
|
assert experiment(:simple).active?
|
@@ -826,6 +1179,7 @@ This experiment did not run long enough to find a clear winner.
|
|
826
1179
|
identify { ids.pop }
|
827
1180
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
828
1181
|
metrics :coolness
|
1182
|
+
default false
|
829
1183
|
end
|
830
1184
|
99.times do |i|
|
831
1185
|
experiment(:simple).choose
|
@@ -843,6 +1197,7 @@ This experiment did not run long enough to find a clear winner.
|
|
843
1197
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
844
1198
|
outcome_is { alternatives[1] }
|
845
1199
|
metrics :coolness
|
1200
|
+
default false
|
846
1201
|
end
|
847
1202
|
# Run experiment to completion (100 participants)
|
848
1203
|
results = Set.new
|
@@ -870,6 +1225,7 @@ This experiment did not run long enough to find a clear winner.
|
|
870
1225
|
new_ab_test :quick do
|
871
1226
|
outcome_is { alternatives[1] }
|
872
1227
|
metrics :coolness
|
1228
|
+
default false
|
873
1229
|
end
|
874
1230
|
experiment(:quick).complete!
|
875
1231
|
assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
|
@@ -878,6 +1234,7 @@ This experiment did not run long enough to find a clear winner.
|
|
878
1234
|
def test_completion_with_outcome
|
879
1235
|
new_ab_test :quick do
|
880
1236
|
metrics :coolness
|
1237
|
+
default false
|
881
1238
|
end
|
882
1239
|
experiment(:quick).complete!(1)
|
883
1240
|
assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
|
@@ -887,6 +1244,7 @@ This experiment did not run long enough to find a clear winner.
|
|
887
1244
|
new_ab_test :quick do
|
888
1245
|
outcome_is { raise RuntimeError }
|
889
1246
|
metrics :coolness
|
1247
|
+
default false
|
890
1248
|
end
|
891
1249
|
e = experiment(:quick)
|
892
1250
|
e.expects(:warn)
|
@@ -899,6 +1257,7 @@ This experiment did not run long enough to find a clear winner.
|
|
899
1257
|
new_ab_test :quick do
|
900
1258
|
outcome_is { nil }
|
901
1259
|
metrics :coolness
|
1260
|
+
default false
|
902
1261
|
end
|
903
1262
|
experiment(:quick).complete!
|
904
1263
|
assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
|
@@ -908,6 +1267,7 @@ This experiment did not run long enough to find a clear winner.
|
|
908
1267
|
new_ab_test :quick do
|
909
1268
|
outcome_is { "error" }
|
910
1269
|
metrics :coolness
|
1270
|
+
default false
|
911
1271
|
end
|
912
1272
|
experiment(:quick).complete!
|
913
1273
|
assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
|
@@ -917,6 +1277,7 @@ This experiment did not run long enough to find a clear winner.
|
|
917
1277
|
new_ab_test :quick do
|
918
1278
|
outcome_is { fail "Testing outcome_is raising exception" }
|
919
1279
|
metrics :coolness
|
1280
|
+
default false
|
920
1281
|
end
|
921
1282
|
experiment(:quick).complete!
|
922
1283
|
assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
|
@@ -925,6 +1286,7 @@ This experiment did not run long enough to find a clear winner.
|
|
925
1286
|
def test_outcome_choosing_best_alternative
|
926
1287
|
new_ab_test :quick do
|
927
1288
|
metrics :coolness
|
1289
|
+
default false
|
928
1290
|
end
|
929
1291
|
fake :quick, false=>[2,0], true=>10
|
930
1292
|
experiment(:quick).complete!
|
@@ -934,6 +1296,7 @@ This experiment did not run long enough to find a clear winner.
|
|
934
1296
|
def test_outcome_only_performing_alternative
|
935
1297
|
new_ab_test :quick do
|
936
1298
|
metrics :coolness
|
1299
|
+
default false
|
937
1300
|
end
|
938
1301
|
fake :quick, true=>2
|
939
1302
|
experiment(:quick).complete!
|
@@ -943,6 +1306,7 @@ This experiment did not run long enough to find a clear winner.
|
|
943
1306
|
def test_outcome_choosing_equal_alternatives
|
944
1307
|
new_ab_test :quick do
|
945
1308
|
metrics :coolness
|
1309
|
+
default false
|
946
1310
|
end
|
947
1311
|
fake :quick, false=>8, true=>8
|
948
1312
|
experiment(:quick).complete!
|
@@ -957,6 +1321,7 @@ This experiment did not run long enough to find a clear winner.
|
|
957
1321
|
metric "Coolness"
|
958
1322
|
new_ab_test :abcd do
|
959
1323
|
metrics :coolness
|
1324
|
+
default false
|
960
1325
|
end
|
961
1326
|
Vanity.playground.track! :coolness
|
962
1327
|
assert_equal 0, experiment(:abcd).alternatives.sum(&:conversions)
|
@@ -967,6 +1332,7 @@ This experiment did not run long enough to find a clear winner.
|
|
967
1332
|
new_ab_test :quick do
|
968
1333
|
outcome_is { alternatives[1] }
|
969
1334
|
metrics :coolness
|
1335
|
+
default false
|
970
1336
|
end
|
971
1337
|
experiment(:quick).complete!
|
972
1338
|
assert_nil experiment(:quick).outcome
|
@@ -975,6 +1341,7 @@ This experiment did not run long enough to find a clear winner.
|
|
975
1341
|
def test_chooses_records_participants
|
976
1342
|
new_ab_test :simple do
|
977
1343
|
alternatives :a, :b, :c
|
1344
|
+
default :a
|
978
1345
|
metrics :coolness
|
979
1346
|
end
|
980
1347
|
experiment(:simple).chooses(:b)
|
@@ -984,6 +1351,7 @@ This experiment did not run long enough to find a clear winner.
|
|
984
1351
|
def test_chooses_moves_participant_to_new_alternative
|
985
1352
|
new_ab_test :simple do
|
986
1353
|
alternatives :a, :b, :c
|
1354
|
+
default :a
|
987
1355
|
metrics :coolness
|
988
1356
|
identify { "1" }
|
989
1357
|
end
|
@@ -998,6 +1366,7 @@ This experiment did not run long enough to find a clear winner.
|
|
998
1366
|
def test_chooses_records_participants_only_once
|
999
1367
|
new_ab_test :simple do
|
1000
1368
|
alternatives :a, :b, :c
|
1369
|
+
default :a
|
1001
1370
|
metrics :coolness
|
1002
1371
|
end
|
1003
1372
|
2.times { experiment(:simple).chooses(:b) }
|
@@ -1007,6 +1376,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1007
1376
|
def test_chooses_records_participants_for_new_alternatives
|
1008
1377
|
new_ab_test :simple do
|
1009
1378
|
alternatives :a, :b, :c
|
1379
|
+
default :a
|
1010
1380
|
metrics :coolness
|
1011
1381
|
end
|
1012
1382
|
experiment(:simple).chooses(:b)
|
@@ -1017,6 +1387,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1017
1387
|
def test_chooses_records_participants_given_a_valid_request
|
1018
1388
|
new_ab_test :simple do
|
1019
1389
|
alternatives :a, :b, :c
|
1390
|
+
default :a
|
1020
1391
|
metrics :coolness
|
1021
1392
|
end
|
1022
1393
|
experiment(:simple).chooses(:a, dummy_request)
|
@@ -1026,6 +1397,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1026
1397
|
def test_chooses_ignores_participants_given_an_invalid_request
|
1027
1398
|
new_ab_test :simple do
|
1028
1399
|
alternatives :a, :b, :c
|
1400
|
+
default :a
|
1029
1401
|
metrics :coolness
|
1030
1402
|
end
|
1031
1403
|
request = dummy_request
|
@@ -1038,6 +1410,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1038
1410
|
not_collecting!
|
1039
1411
|
new_ab_test :simple do
|
1040
1412
|
alternatives :a, :b, :c
|
1413
|
+
default :a
|
1041
1414
|
metrics :coolness
|
1042
1415
|
end
|
1043
1416
|
assert !experiment(:simple).showing?(experiment(:simple).alternatives[1])
|
@@ -1050,16 +1423,56 @@ This experiment did not run long enough to find a clear winner.
|
|
1050
1423
|
not_collecting!
|
1051
1424
|
new_ab_test :simple do
|
1052
1425
|
alternatives :a, :b, :c
|
1426
|
+
default :a
|
1053
1427
|
metrics :coolness
|
1054
1428
|
end
|
1055
1429
|
choice = experiment(:simple).choose.value
|
1056
1430
|
assert [:a, :b, :c].include?(choice)
|
1057
1431
|
assert_equal choice, experiment(:simple).choose.value
|
1058
1432
|
end
|
1433
|
+
|
1434
|
+
# -- Reset --
|
1435
|
+
|
1436
|
+
def test_reset_clears_participants
|
1437
|
+
new_ab_test :simple do
|
1438
|
+
alternatives :a, :b, :c
|
1439
|
+
default :a
|
1440
|
+
metrics :coolness
|
1441
|
+
end
|
1442
|
+
experiment(:simple).chooses(:b)
|
1443
|
+
assert_equal experiment(:simple).alternatives[1].participants, 1
|
1444
|
+
experiment(:simple).reset
|
1445
|
+
assert_equal experiment(:simple).alternatives[1].participants, 0
|
1446
|
+
end
|
1447
|
+
|
1448
|
+
def test_clears_outcome_and_completed_at
|
1449
|
+
new_ab_test :simple do
|
1450
|
+
alternatives :a, :b, :c
|
1451
|
+
default :a
|
1452
|
+
metrics :coolness
|
1453
|
+
end
|
1454
|
+
experiment(:simple).reset
|
1455
|
+
assert_nil experiment(:simple).outcome
|
1456
|
+
assert_nil experiment(:simple).completed_at
|
1457
|
+
end
|
1458
|
+
|
1459
|
+
# -- Pick Winner --
|
1460
|
+
|
1461
|
+
def test_complete_with_argument_sets_outcome_and_completes
|
1462
|
+
new_ab_test :simple do
|
1463
|
+
alternatives :a, :b, :c
|
1464
|
+
default :a
|
1465
|
+
metrics :coolness
|
1466
|
+
end
|
1467
|
+
experiment(:simple).complete!(experiment(:simple).alternatives[1].id)
|
1468
|
+
assert_equal experiment(:simple).alternatives[1], experiment(:simple).outcome
|
1469
|
+
assert_not_nil experiment(:simple).completed_at
|
1470
|
+
end
|
1059
1471
|
|
1060
1472
|
def test_reset_clears_participants
|
1061
1473
|
new_ab_test :simple do
|
1062
1474
|
alternatives :a, :b, :c
|
1475
|
+
default :a
|
1063
1476
|
metrics :coolness
|
1064
1477
|
end
|
1065
1478
|
experiment(:simple).chooses(:b)
|
@@ -1071,6 +1484,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1071
1484
|
def test_clears_outcome_and_completed_at
|
1072
1485
|
new_ab_test :simple do
|
1073
1486
|
alternatives :a, :b, :c
|
1487
|
+
default :a
|
1074
1488
|
metrics :coolness
|
1075
1489
|
end
|
1076
1490
|
experiment(:simple).reset
|