vanity 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -2
  3. data/Appraisals +6 -6
  4. data/CHANGELOG +9 -3
  5. data/Gemfile.lock +1 -1
  6. data/README.md +299 -0
  7. data/doc/configuring.textile +8 -1
  8. data/doc/identity.textile +2 -0
  9. data/doc/metrics.textile +10 -0
  10. data/gemfiles/rails32.gemfile.lock +1 -1
  11. data/gemfiles/rails41.gemfile.lock +1 -1
  12. data/gemfiles/rails42.gemfile.lock +1 -1
  13. data/gemfiles/{rails4.gemfile → rails42_protected_attributes.gemfile} +2 -2
  14. data/gemfiles/rails42_protected_attributes.gemfile.lock +209 -0
  15. data/lib/generators/templates/vanity_migration.rb +1 -0
  16. data/lib/vanity/adapters/abstract_adapter.rb +11 -0
  17. data/lib/vanity/adapters/active_record_adapter.rb +15 -1
  18. data/lib/vanity/adapters/mock_adapter.rb +14 -0
  19. data/lib/vanity/adapters/mongodb_adapter.rb +14 -0
  20. data/lib/vanity/adapters/redis_adapter.rb +15 -0
  21. data/lib/vanity/configuration.rb +43 -11
  22. data/lib/vanity/experiment/ab_test.rb +145 -15
  23. data/lib/vanity/experiment/alternative.rb +4 -0
  24. data/lib/vanity/frameworks/rails.rb +69 -31
  25. data/lib/vanity/locales/vanity.en.yml +9 -0
  26. data/lib/vanity/locales/vanity.pt-BR.yml +4 -0
  27. data/lib/vanity/metric/active_record.rb +9 -1
  28. data/lib/vanity/templates/_ab_test.erb +9 -2
  29. data/lib/vanity/templates/_experiment.erb +21 -1
  30. data/lib/vanity/templates/vanity.css +11 -3
  31. data/lib/vanity/templates/vanity.js +35 -6
  32. data/lib/vanity/version.rb +1 -1
  33. data/test/commands/report_test.rb +1 -0
  34. data/test/dummy/config/application.rb +1 -0
  35. data/test/experiment/ab_test.rb +414 -0
  36. data/test/experiment/base_test.rb +16 -10
  37. data/test/frameworks/rails/action_controller_test.rb +14 -6
  38. data/test/frameworks/rails/action_mailer_test.rb +8 -6
  39. data/test/frameworks/rails/action_view_test.rb +1 -0
  40. data/test/helper_test.rb +2 -0
  41. data/test/metric/active_record_test.rb +56 -0
  42. data/test/playground_test.rb +3 -0
  43. data/test/test_helper.rb +28 -2
  44. data/test/web/rails/dashboard_test.rb +2 -0
  45. data/vanity.gemspec +2 -2
  46. metadata +8 -8
  47. data/README.rdoc +0 -231
  48. data/gemfiles/rails4.gemfile.lock +0 -179
@@ -37,15 +37,18 @@ class AbTestTest < ActionController::TestCase
37
37
  assert_raises RuntimeError do
38
38
  new_ab_test :none do
39
39
  alternatives []
40
+ default nil
40
41
  end
41
42
  end
42
43
  assert_raises RuntimeError do
43
44
  new_ab_test :one do
44
45
  alternatives "foo"
46
+ default "foo"
45
47
  end
46
48
  end
47
49
  new_ab_test :two do
48
50
  alternatives "foo", "bar"
51
+ default "foo"
49
52
  metrics :coolness
50
53
  end
51
54
  end
@@ -53,6 +56,7 @@ class AbTestTest < ActionController::TestCase
53
56
  def test_returning_alternative_by_value
54
57
  new_ab_test :abcd do
55
58
  alternatives :a, :b, :c, :d
59
+ default :a
56
60
  metrics :coolness
57
61
  end
58
62
  assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
@@ -62,6 +66,7 @@ class AbTestTest < ActionController::TestCase
62
66
  def test_alternative_name
63
67
  new_ab_test :abcd do
64
68
  alternatives :a, :b
69
+ default :a
65
70
  metrics :coolness
66
71
  end
67
72
  assert_equal "option A", experiment(:abcd).alternative(:a).name
@@ -72,10 +77,12 @@ class AbTestTest < ActionController::TestCase
72
77
  new_ab_test :ab do
73
78
  metrics :coolness
74
79
  alternatives :a, :b
80
+ default :a
75
81
  end
76
82
  new_ab_test :cd do
77
83
  metrics :coolness
78
84
  alternatives :a, :b
85
+ default :a
79
86
  end
80
87
  fingerprints = Vanity.playground.experiments.map { |id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
81
88
  assert_equal 4, fingerprints.uniq.size
@@ -84,6 +91,7 @@ class AbTestTest < ActionController::TestCase
84
91
  def test_alternative_fingerprint_is_consistent
85
92
  new_ab_test :ab do
86
93
  alternatives :a, :b
94
+ default :a
87
95
  metrics :coolness
88
96
  end
89
97
  fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
@@ -93,18 +101,305 @@ class AbTestTest < ActionController::TestCase
93
101
  assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
94
102
  end
95
103
 
104
+ def test_ab_has_default
105
+ new_ab_test :ice_cream_flavor do
106
+ metrics :coolness
107
+ alternatives :a, :b, :c
108
+ default :b
109
+ end
110
+ exp = experiment(:ice_cream_flavor)
111
+ assert_equal exp.default, exp.alternative(:b)
112
+ end
113
+
114
+ def test_ab_sets_default_default
115
+ new_ab_test :ice_cream_flavor do
116
+ metrics :coolness
117
+ alternatives :a, :b, :c
118
+ # no default specified
119
+ end
120
+ exp = experiment(:ice_cream_flavor)
121
+ assert_equal exp.default, exp.alternative(:a)
122
+ end
123
+
124
+ def test_ab_overrides_unknown_default
125
+ new_ab_test :ice_cream_flavor do
126
+ metrics :coolness
127
+ alternatives :a, :b, :c
128
+ default :badname
129
+ end
130
+ exp = experiment(:ice_cream_flavor)
131
+ assert_equal exp.default, exp.alternative(:a)
132
+ end
133
+
134
+ def test_ab_can_only_set_default_once
135
+ assert_raise ArgumentError do
136
+ new_ab_test :ice_cream_flavor do
137
+ metrics :coolness
138
+ alternative :a, :b, :c
139
+ default :a
140
+ default :b
141
+ end
142
+ end
143
+ end
144
+
145
+ def test_ab_can_only_have_one_default
146
+ assert_raise ArgumentError do
147
+ new_ab_test :ice_cream_flavor do
148
+ metrics :coolness
149
+ alternative :a, :b, :c
150
+ default :a, :b
151
+ end
152
+ end
153
+ end
154
+
155
+ def test_ab_cannot_get_default_before_specified
156
+ assert_raise ArgumentError do
157
+ new_ab_test :ice_cream_flavor do
158
+ metrics :coolness
159
+ alternative :a, :b, :c
160
+ default
161
+ end
162
+ end
163
+ end
164
+
165
+ def test_ab_accepts_nil_default
166
+ new_ab_test :nil_default do
167
+ metrics :coolness
168
+ alternatives nil, 'foo'
169
+ default nil
170
+ end
171
+ exp = experiment(:nil_default)
172
+ assert_equal exp.default, exp.alternative(nil)
173
+ end
174
+
175
+ def test_ab_chooses_nil_default_default
176
+ new_ab_test :nil_default_default do
177
+ metrics :coolness
178
+ alternatives nil, 'foo'
179
+ # no default specified
180
+ end
181
+ exp = experiment(:nil_default_default)
182
+ assert_equal exp.default, exp.alternative(nil)
183
+ end
184
+
185
+
186
+ # -- Experiment Enabled/disabled --
187
+
188
+ # @example new test should be enabled regardless of collecting?
189
+ # regardless_of "Vanity.playground.collecting" do
190
+ # assert (new_ab_test :test).enabled?
191
+ # end
192
+ def regardless_of(attr_name, &block)
193
+ prev_val = eval "#{attr_name}?"
194
+
195
+ eval "#{attr_name}=true"
196
+ block.call(eval "#{attr_name}?")
197
+ nuke_playground
198
+
199
+ eval "#{attr_name}=false"
200
+ block.call(eval "#{attr_name}?")
201
+ nuke_playground
202
+
203
+ eval "#{attr_name}=prev_val"
204
+ end
205
+
206
+ def test_new_test_is_disabled_when_experiments_start_enabled_is_false
207
+ Vanity.configuration.experiments_start_enabled = false
208
+ exp = new_ab_test :test, enable: false do
209
+ metrics :coolness
210
+ default false
211
+ end
212
+ assert !exp.enabled?
213
+ end
214
+
215
+ def test_new_test_is_enabled_when_experiments_start_enabled_is_true
216
+ Vanity.configuration.experiments_start_enabled = true
217
+ exp = new_ab_test :test, enable: false do
218
+ metrics :coolness
219
+ default false
220
+ end
221
+ assert exp.enabled?
222
+ end
223
+
224
+ def test_complete_sets_enabled_false
225
+ Vanity.playground.collecting = true
226
+ exp = new_ab_test :test do
227
+ metrics :coolness
228
+ default false
229
+ end
230
+ exp.complete! #active? => false
231
+
232
+ assert !exp.enabled?, "experiment should not be enabled but it is!"
233
+ end
234
+
235
+ def test_complete_keeps_enabled_true_while_not_collecting
236
+ exp = new_ab_test :test do
237
+ metrics :coolness
238
+ default false
239
+ end
240
+ Vanity.playground.collecting = false
241
+ exp.enabled = false
242
+ assert exp.enabled?
243
+ end
244
+
245
+ def test_set_enabled_while_active
246
+ Vanity.playground.collecting = true
247
+ exp = new_ab_test :test do
248
+ metrics :coolness
249
+ default false
250
+ end
251
+
252
+ exp.enabled = true
253
+ assert exp.enabled?
254
+
255
+ exp.enabled = false
256
+ assert !exp.enabled?
257
+ end
258
+
259
+ def test_cannot_set_enabled_for_inactive
260
+ Vanity.playground.collecting = true
261
+ exp = new_ab_test :test do
262
+ metrics :coolness
263
+ default false
264
+ end
265
+ exp.complete! #active? => false
266
+ assert !exp.enabled?
267
+ exp.enabled = true
268
+ assert !exp.enabled?
269
+ end
270
+
271
+ def test_always_enabled_while_not_collecting
272
+ Vanity.playground.collecting = false
273
+ exp = new_ab_test :test do
274
+ metrics :coolness
275
+ default false
276
+ end
277
+ assert exp.enabled?
278
+ exp.enabled = false
279
+ assert exp.enabled?
280
+ end
281
+
282
+ def test_enabled_persists_across_definitions
283
+ Vanity.configuration.experiments_start_enabled = false
284
+ Vanity.playground.collecting = true
285
+ new_ab_test :test, :enable => false do
286
+ metrics :coolness
287
+ default false
288
+ end
289
+ assert !experiment(:test).enabled? #starts off false
290
+
291
+ new_playground
292
+ metric "Coolness"
293
+
294
+ new_ab_test :test, :enable => false do
295
+ metrics :coolness
296
+ default false
297
+ end
298
+ assert !experiment(:test).enabled? #still false
299
+ experiment(:test).enabled = true
300
+ assert experiment(:test).enabled? #now true
301
+
302
+ new_playground
303
+ metric "Coolness"
304
+
305
+ new_ab_test :test, :enable => false do
306
+ metrics :coolness
307
+ default false
308
+ end
309
+ assert experiment(:test).enabled? #still true
310
+ experiment(:test).enabled = false
311
+ assert !experiment(:test).enabled? #now false again
312
+ end
96
313
 
314
+ def test_enabled_persists_across_definitions_when_starting_enabled
315
+ Vanity.configuration.experiments_start_enabled = true
316
+ Vanity.playground.collecting = true
317
+ new_ab_test :test, :enable => false do
318
+ metrics :coolness
319
+ default false
320
+ end
321
+ assert experiment(:test).enabled? #starts off true
322
+
323
+ new_playground
324
+ metric "Coolness"
325
+
326
+ new_ab_test :test, :enable => false do
327
+ metrics :coolness
328
+ default false
329
+ end
330
+ assert experiment(:test).enabled? #still true
331
+ experiment(:test).enabled = false
332
+ assert !experiment(:test).enabled? #now false
333
+
334
+ new_playground
335
+ metric "Coolness"
336
+
337
+ new_ab_test :test, :enable => false do
338
+ metrics :coolness
339
+ default false
340
+ end
341
+ assert !experiment(:test).enabled? #still false
342
+ experiment(:test).enabled = true
343
+ assert experiment(:test).enabled? #now true again
344
+ end
345
+
346
+ def test_choose_random_when_enabled
347
+ regardless_of "Vanity.playground.collecting" do
348
+ metric "Coolness"
349
+
350
+ exp = new_ab_test :test do
351
+ metrics :coolness
352
+ true_false
353
+ default false
354
+ identify { rand }
355
+ end
356
+ results = Set.new
357
+ 100.times do
358
+ results << exp.choose.value
359
+ end
360
+ assert_equal results, [true, false].to_set
361
+ end
362
+ end
363
+
364
+ def test_choose_default_when_disabled
365
+ exp = new_ab_test :test do
366
+ metrics :coolness
367
+ alternatives 0, 1, 2, 3, 4, 5
368
+ default 3
369
+ end
370
+
371
+ exp.enabled = false
372
+ 100.times.each do
373
+ assert_equal 3, exp.choose.value
374
+ end
375
+ end
376
+
377
+ def test_choose_outcome_when_finished
378
+ exp = new_ab_test :test do
379
+ metrics :coolness
380
+ alternatives 0,1,2,3,4,5
381
+ default 3
382
+ outcome_is { alternative(5) }
383
+ end
384
+ exp.complete!
385
+ 100.times.each do
386
+ assert_equal 5, exp.choose.value
387
+ end
388
+ end
389
+
97
390
  # -- Experiment metric --
98
391
 
99
392
  def test_explicit_metric
100
393
  new_ab_test :abcd do
101
394
  metrics :coolness
395
+ default false
102
396
  end
103
397
  assert_equal [Vanity.playground.metric(:coolness)], experiment(:abcd).metrics
104
398
  end
105
399
 
106
400
  def test_implicit_metric
107
401
  new_ab_test :abcd do
402
+ default false
108
403
  end
109
404
  assert_equal [Vanity.playground.metric(:abcd)], experiment(:abcd).metrics
110
405
  end
@@ -113,6 +408,7 @@ class AbTestTest < ActionController::TestCase
113
408
  metric "Coolness"
114
409
  new_ab_test :abcd do
115
410
  metrics :coolness
411
+ default false
116
412
  end
117
413
  Vanity.playground.track! :coolness
118
414
  assert_equal 1, experiment(:abcd).alternatives.sum(&:conversions)
@@ -123,6 +419,7 @@ class AbTestTest < ActionController::TestCase
123
419
  def test_track_with_identity_overrides_default
124
420
  identities = ["quux"]
125
421
  new_ab_test :foobar do
422
+ default "foo"
126
423
  alternatives "foo", "bar"
127
424
  identify { identities.pop || "6e98ec" }
128
425
  metrics :coolness
@@ -141,6 +438,7 @@ class AbTestTest < ActionController::TestCase
141
438
  Vanity.configuration.use_js = true
142
439
  ids = (0...10).to_a
143
440
  new_ab_test :foobar do
441
+ default "foo"
144
442
  alternatives "foo", "bar"
145
443
  identify { ids.pop }
146
444
  metrics :coolness
@@ -155,6 +453,7 @@ class AbTestTest < ActionController::TestCase
155
453
  def test_calls_on_assignment_on_new_assignment
156
454
  on_assignment_called_times = 0
157
455
  new_ab_test :foobar do
456
+ default "foo"
158
457
  alternatives "foo", "bar"
159
458
  identify { "6e98ec" }
160
459
  metrics :coolness
@@ -167,6 +466,7 @@ class AbTestTest < ActionController::TestCase
167
466
  def test_calls_on_assignment_when_given_valid_request
168
467
  on_assignment_called_times = 0
169
468
  new_ab_test :foobar do
469
+ default "foo"
170
470
  alternatives "foo", "bar"
171
471
  identify { "6e98ec" }
172
472
  metrics :coolness
@@ -179,6 +479,7 @@ class AbTestTest < ActionController::TestCase
179
479
  def test_does_not_call_on_assignment_when_given_invalid_request
180
480
  on_assignment_called_times = 0
181
481
  new_ab_test :foobar do
482
+ default "foo"
182
483
  alternatives "foo", "bar"
183
484
  identify { "6e98ec" }
184
485
  metrics :coolness
@@ -194,6 +495,7 @@ class AbTestTest < ActionController::TestCase
194
495
  on_assignment_called_times = 0
195
496
  new_ab_test :foobar do
196
497
  alternatives "foo", "bar"
498
+ default "foo"
197
499
  identify { "6e98ec" }
198
500
  metrics :coolness
199
501
  on_assignment { on_assignment_called_times = on_assignment_called_times+1 }
@@ -205,6 +507,7 @@ class AbTestTest < ActionController::TestCase
205
507
  def test_returns_the_same_alternative_consistently_when_on_assignment_is_set
206
508
  new_ab_test :foobar do
207
509
  alternatives "foo", "bar"
510
+ default "foo"
208
511
  identify { "6e98ec" }
209
512
  on_assignment {}
210
513
  metrics :coolness
@@ -221,6 +524,7 @@ class AbTestTest < ActionController::TestCase
221
524
  def test_ab_assigned
222
525
  new_ab_test :foobar do
223
526
  alternatives "foo", "bar"
527
+ default "foo"
224
528
  identify { "6e98ec" }
225
529
  metrics :coolness
226
530
  end
@@ -233,6 +537,7 @@ class AbTestTest < ActionController::TestCase
233
537
  identity = { :a => :b }
234
538
  new_ab_test :foobar do
235
539
  alternatives "foo", "bar"
540
+ default "foo"
236
541
  identify { identity }
237
542
  metrics :coolness
238
543
  end
@@ -246,6 +551,7 @@ class AbTestTest < ActionController::TestCase
246
551
  def test_returns_the_same_alternative_consistently_when_using_probabilities
247
552
  new_ab_test :foobar do
248
553
  alternatives "foo", "bar"
554
+ default "foo"
249
555
  identify { "6e98ec" }
250
556
  rebalance_frequency 10
251
557
  metrics :coolness
@@ -258,9 +564,21 @@ class AbTestTest < ActionController::TestCase
258
564
  end
259
565
  end
260
566
 
567
+ def test_uses_configured_probabilities_for_new_assignments
568
+ new_ab_test :foobar do
569
+ alternatives "foo" => 30, "bar" => 60
570
+ identify { rand }
571
+ metrics :coolness
572
+ end
573
+ alts = Array.new(10_000) { experiment(:foobar).choose.value }.reduce({}) { |h,k| h[k] ||= 0; h[k] += 1; h }
574
+ assert_equal %w{bar foo}, alts.keys.sort
575
+ assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
576
+ end
577
+
261
578
  def test_uses_probabilities_for_new_assignments
262
579
  new_ab_test :foobar do
263
580
  alternatives "foo", "bar"
581
+ default "foo"
264
582
  identify { rand }
265
583
  rebalance_frequency 10000
266
584
  metrics :coolness
@@ -279,6 +597,7 @@ class AbTestTest < ActionController::TestCase
279
597
  def test_rebalances_probabilities_after_rebalance_frequency_calls
280
598
  new_ab_test :foobar do
281
599
  alternatives "foo", "bar"
600
+ default "foo"
282
601
  identify { rand }
283
602
  rebalance_frequency 12
284
603
  metrics :coolness
@@ -302,6 +621,7 @@ class AbTestTest < ActionController::TestCase
302
621
  def test_rebalance_uses_bayes_score_probabilities_to_update_probabilities
303
622
  new_ab_test :foobar do
304
623
  alternatives "foo", "bar", "baa"
624
+ default "foo"
305
625
  identify { rand }
306
626
  rebalance_frequency 12
307
627
  metrics :coolness
@@ -334,6 +654,7 @@ class AbTestTest < ActionController::TestCase
334
654
  def test_returns_the_same_alternative_consistently
335
655
  new_ab_test :foobar do
336
656
  alternatives "foo", "bar"
657
+ default "foo"
337
658
  identify { "6e98ec" }
338
659
  metrics :coolness
339
660
  end
@@ -347,6 +668,7 @@ class AbTestTest < ActionController::TestCase
347
668
  def test_respects_out_of_band_assignment
348
669
  new_ab_test :foobar do
349
670
  alternatives "a", "b", "c"
671
+ default "a"
350
672
  identify { "6e98ec" }
351
673
  metrics :coolness
352
674
  end
@@ -365,6 +687,7 @@ class AbTestTest < ActionController::TestCase
365
687
  def test_returns_different_alternatives_for_each_participant
366
688
  new_ab_test :foobar do
367
689
  alternatives "foo", "bar"
690
+ default "foo"
368
691
  identify { rand }
369
692
  metrics :coolness
370
693
  end
@@ -377,6 +700,7 @@ class AbTestTest < ActionController::TestCase
377
700
  ids = (Array.new(200) { |i| i } * 5).shuffle
378
701
  new_ab_test :foobar do
379
702
  alternatives "foo", "bar"
703
+ default "foo"
380
704
  identify { ids.pop }
381
705
  metrics :coolness
382
706
  end
@@ -390,6 +714,7 @@ class AbTestTest < ActionController::TestCase
390
714
  ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
391
715
  new_ab_test :foobar do
392
716
  alternatives "foo", "bar"
717
+ default "foo"
393
718
  identify { ids.pop }
394
719
  metrics :coolness
395
720
  end
@@ -405,6 +730,7 @@ class AbTestTest < ActionController::TestCase
405
730
  ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
406
731
  new_ab_test :foobar do
407
732
  alternatives "foo", "bar"
733
+ default "foo"
408
734
  identify { ids.pop }
409
735
  metrics :coolness
410
736
  end
@@ -420,6 +746,7 @@ class AbTestTest < ActionController::TestCase
420
746
  def test_choose_records_participants_given_a_valid_request
421
747
  new_ab_test :foobar do
422
748
  alternatives "foo", "bar"
749
+ default "foo"
423
750
  identify { "me" }
424
751
  metrics :coolness
425
752
  end
@@ -430,6 +757,7 @@ class AbTestTest < ActionController::TestCase
430
757
  def test_choose_ignores_participants_given_an_invalid_request
431
758
  new_ab_test :foobar do
432
759
  alternatives "foo", "bar"
760
+ default "foo"
433
761
  identify { "me" }
434
762
  metrics :coolness
435
763
  end
@@ -443,6 +771,7 @@ class AbTestTest < ActionController::TestCase
443
771
  new_ab_test :simple do
444
772
  identify { "me" }
445
773
  metrics :coolness
774
+ default false
446
775
  complete_if { alternatives.map(&:converted).sum >= 1 }
447
776
  outcome_is { alternative(true) }
448
777
  end
@@ -472,6 +801,7 @@ class AbTestTest < ActionController::TestCase
472
801
  def test_ab_test_chooses_in_render
473
802
  new_ab_test :simple do
474
803
  metrics :coolness
804
+ default false
475
805
  end
476
806
  responses = Array.new(100) do
477
807
  @controller = nil ; setup_controller_request_and_response
@@ -484,6 +814,7 @@ class AbTestTest < ActionController::TestCase
484
814
  def test_ab_test_chooses_view_helper
485
815
  new_ab_test :simple do
486
816
  metrics :coolness
817
+ default false
487
818
  end
488
819
  responses = Array.new(100) do
489
820
  @controller = nil ; setup_controller_request_and_response
@@ -496,6 +827,7 @@ class AbTestTest < ActionController::TestCase
496
827
  def test_ab_test_with_capture
497
828
  new_ab_test :simple do
498
829
  metrics :coolness
830
+ default false
499
831
  end
500
832
  responses = Array.new(100) do
501
833
  @controller = nil ; setup_controller_request_and_response
@@ -508,6 +840,7 @@ class AbTestTest < ActionController::TestCase
508
840
  def test_ab_test_track
509
841
  new_ab_test :simple do
510
842
  metrics :coolness
843
+ default false
511
844
  end
512
845
  responses = Array.new(100) do
513
846
  @controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
@@ -522,6 +855,7 @@ class AbTestTest < ActionController::TestCase
522
855
  def test_with_given_choice
523
856
  new_ab_test :simple do
524
857
  alternatives :a, :b, :c
858
+ default :a
525
859
  metrics :coolness
526
860
  end
527
861
  100.times do |i|
@@ -535,6 +869,7 @@ class AbTestTest < ActionController::TestCase
535
869
  def test_which_chooses_non_existent_alternative
536
870
  new_ab_test :simple do
537
871
  metrics :coolness
872
+ default false
538
873
  end
539
874
  assert_raises ArgumentError do
540
875
  experiment(:simple).chooses(404)
@@ -545,6 +880,7 @@ class AbTestTest < ActionController::TestCase
545
880
  new_ab_test :simple do
546
881
  identify { rand }
547
882
  alternatives :a, :b, :c
883
+ default :a
548
884
  metrics :coolness
549
885
  end
550
886
  responses = Array.new(100) { |i|
@@ -563,6 +899,7 @@ class AbTestTest < ActionController::TestCase
563
899
  def test_calculate_score
564
900
  new_ab_test :abcd do
565
901
  alternatives :a, :b, :c, :d
902
+ default :a
566
903
  metrics :coolness
567
904
  end
568
905
  score_result = experiment(:abcd).calculate_score
@@ -570,6 +907,7 @@ class AbTestTest < ActionController::TestCase
570
907
 
571
908
  new_ab_test :bayes_abcd do
572
909
  alternatives :a, :b, :c, :d
910
+ default :a
573
911
  metrics :coolness
574
912
  score_method :bayes_bandit_score
575
913
  end
@@ -580,6 +918,7 @@ class AbTestTest < ActionController::TestCase
580
918
  def test_scoring
581
919
  new_ab_test :abcd do
582
920
  alternatives :a, :b, :c, :d
921
+ default :a
583
922
  metrics :coolness
584
923
  end
585
924
  # participating, conversions, rate, z-score
@@ -606,6 +945,7 @@ class AbTestTest < ActionController::TestCase
606
945
  def test_bayes_scoring
607
946
  new_ab_test :abcd do
608
947
  alternatives :a, :b, :c, :d
948
+ default :a
609
949
  metrics :coolness
610
950
  end
611
951
  # participating, conversions, rate, z-score
@@ -623,6 +963,7 @@ class AbTestTest < ActionController::TestCase
623
963
  def test_scoring_with_no_performers
624
964
  new_ab_test :abcd do
625
965
  alternatives :a, :b, :c, :d
966
+ default :a
626
967
  metrics :coolness
627
968
  end
628
969
  assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
@@ -636,6 +977,7 @@ class AbTestTest < ActionController::TestCase
636
977
  def test_scoring_with_one_performer
637
978
  new_ab_test :abcd do
638
979
  alternatives :a, :b, :c, :d
980
+ default :a
639
981
  metrics :coolness
640
982
  end
641
983
  fake :abcd, :b=>[10,8]
@@ -651,6 +993,7 @@ class AbTestTest < ActionController::TestCase
651
993
  def test_scoring_with_some_performers
652
994
  new_ab_test :abcd do
653
995
  alternatives :a, :b, :c, :d
996
+ default :a
654
997
  metrics :coolness
655
998
  end
656
999
  fake :abcd, :b=>[10,8], :d=>[12,5]
@@ -670,6 +1013,7 @@ class AbTestTest < ActionController::TestCase
670
1013
  def test_scoring_with_different_probability
671
1014
  new_ab_test :abcd do
672
1015
  alternatives :a, :b, :c, :d
1016
+ default :a
673
1017
  metrics :coolness
674
1018
  end
675
1019
  fake :abcd, :b=>[10,8], :d=>[12,5]
@@ -685,6 +1029,7 @@ class AbTestTest < ActionController::TestCase
685
1029
  def test_conclusion
686
1030
  new_ab_test :abcd do
687
1031
  alternatives :a, :b, :c, :d
1032
+ default :a
688
1033
  metrics :coolness
689
1034
  end
690
1035
  # participating, conversions, rate, z-score
@@ -708,6 +1053,7 @@ Option D selected as the best alternative.
708
1053
  def test_conclusion_with_some_performers
709
1054
  new_ab_test :abcd do
710
1055
  alternatives :a, :b, :c, :d
1056
+ default :a
711
1057
  metrics :coolness
712
1058
  end
713
1059
  fake :abcd, :b=>[180, 45], :d=>[188, 61]
@@ -726,6 +1072,7 @@ Option D selected as the best alternative.
726
1072
  def test_conclusion_without_clear_winner
727
1073
  new_ab_test :abcd do
728
1074
  alternatives :a, :b, :c, :d
1075
+ default :a
729
1076
  metrics :coolness
730
1077
  end
731
1078
  fake :abcd, :b=>[180, 58], :d=>[188, 61]
@@ -743,6 +1090,7 @@ Option C did not convert.
743
1090
  def test_conclusion_without_close_performers
744
1091
  new_ab_test :abcd do
745
1092
  alternatives :a, :b, :c, :d
1093
+ default :a
746
1094
  metrics :coolness
747
1095
  end
748
1096
  fake :abcd, :b=>[186, 60], :d=>[188, 61]
@@ -760,6 +1108,7 @@ Option C did not convert.
760
1108
  def test_conclusion_without_equal_performers
761
1109
  new_ab_test :abcd do
762
1110
  alternatives :a, :b, :c, :d
1111
+ default :a
763
1112
  metrics :coolness
764
1113
  end
765
1114
  fake :abcd, :b=>[188, 61], :d=>[188, 61]
@@ -776,6 +1125,7 @@ Option C did not convert.
776
1125
  def test_conclusion_with_one_performers
777
1126
  new_ab_test :abcd do
778
1127
  alternatives :a, :b, :c, :d
1128
+ default :a
779
1129
  metrics :coolness
780
1130
  end
781
1131
  fake :abcd, :b=>[180, 45]
@@ -789,6 +1139,7 @@ This experiment did not run long enough to find a clear winner.
789
1139
  def test_conclusion_with_no_performers
790
1140
  new_ab_test :abcd do
791
1141
  alternatives :a, :b, :c, :d
1142
+ default :a
792
1143
  metrics :coolness
793
1144
  end
794
1145
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
@@ -805,6 +1156,7 @@ This experiment did not run long enough to find a clear winner.
805
1156
  identify { rand }
806
1157
  complete_if { true }
807
1158
  metrics :coolness
1159
+ default false
808
1160
  end
809
1161
  experiment(:simple).choose
810
1162
  assert !experiment(:simple).active?
@@ -815,6 +1167,7 @@ This experiment did not run long enough to find a clear winner.
815
1167
  identify { rand }
816
1168
  complete_if { fail "Testing complete_if raises exception" }
817
1169
  metrics :coolness
1170
+ default false
818
1171
  end
819
1172
  experiment(:simple).choose
820
1173
  assert experiment(:simple).active?
@@ -826,6 +1179,7 @@ This experiment did not run long enough to find a clear winner.
826
1179
  identify { ids.pop }
827
1180
  complete_if { alternatives.map(&:participants).sum >= 100 }
828
1181
  metrics :coolness
1182
+ default false
829
1183
  end
830
1184
  99.times do |i|
831
1185
  experiment(:simple).choose
@@ -843,6 +1197,7 @@ This experiment did not run long enough to find a clear winner.
843
1197
  complete_if { alternatives.map(&:participants).sum >= 100 }
844
1198
  outcome_is { alternatives[1] }
845
1199
  metrics :coolness
1200
+ default false
846
1201
  end
847
1202
  # Run experiment to completion (100 participants)
848
1203
  results = Set.new
@@ -870,6 +1225,7 @@ This experiment did not run long enough to find a clear winner.
870
1225
  new_ab_test :quick do
871
1226
  outcome_is { alternatives[1] }
872
1227
  metrics :coolness
1228
+ default false
873
1229
  end
874
1230
  experiment(:quick).complete!
875
1231
  assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
@@ -878,6 +1234,7 @@ This experiment did not run long enough to find a clear winner.
878
1234
  def test_completion_with_outcome
879
1235
  new_ab_test :quick do
880
1236
  metrics :coolness
1237
+ default false
881
1238
  end
882
1239
  experiment(:quick).complete!(1)
883
1240
  assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
@@ -887,6 +1244,7 @@ This experiment did not run long enough to find a clear winner.
887
1244
  new_ab_test :quick do
888
1245
  outcome_is { raise RuntimeError }
889
1246
  metrics :coolness
1247
+ default false
890
1248
  end
891
1249
  e = experiment(:quick)
892
1250
  e.expects(:warn)
@@ -899,6 +1257,7 @@ This experiment did not run long enough to find a clear winner.
899
1257
  new_ab_test :quick do
900
1258
  outcome_is { nil }
901
1259
  metrics :coolness
1260
+ default false
902
1261
  end
903
1262
  experiment(:quick).complete!
904
1263
  assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
@@ -908,6 +1267,7 @@ This experiment did not run long enough to find a clear winner.
908
1267
  new_ab_test :quick do
909
1268
  outcome_is { "error" }
910
1269
  metrics :coolness
1270
+ default false
911
1271
  end
912
1272
  experiment(:quick).complete!
913
1273
  assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
@@ -917,6 +1277,7 @@ This experiment did not run long enough to find a clear winner.
917
1277
  new_ab_test :quick do
918
1278
  outcome_is { fail "Testing outcome_is raising exception" }
919
1279
  metrics :coolness
1280
+ default false
920
1281
  end
921
1282
  experiment(:quick).complete!
922
1283
  assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
@@ -925,6 +1286,7 @@ This experiment did not run long enough to find a clear winner.
925
1286
  def test_outcome_choosing_best_alternative
926
1287
  new_ab_test :quick do
927
1288
  metrics :coolness
1289
+ default false
928
1290
  end
929
1291
  fake :quick, false=>[2,0], true=>10
930
1292
  experiment(:quick).complete!
@@ -934,6 +1296,7 @@ This experiment did not run long enough to find a clear winner.
934
1296
  def test_outcome_only_performing_alternative
935
1297
  new_ab_test :quick do
936
1298
  metrics :coolness
1299
+ default false
937
1300
  end
938
1301
  fake :quick, true=>2
939
1302
  experiment(:quick).complete!
@@ -943,6 +1306,7 @@ This experiment did not run long enough to find a clear winner.
943
1306
  def test_outcome_choosing_equal_alternatives
944
1307
  new_ab_test :quick do
945
1308
  metrics :coolness
1309
+ default false
946
1310
  end
947
1311
  fake :quick, false=>8, true=>8
948
1312
  experiment(:quick).complete!
@@ -957,6 +1321,7 @@ This experiment did not run long enough to find a clear winner.
957
1321
  metric "Coolness"
958
1322
  new_ab_test :abcd do
959
1323
  metrics :coolness
1324
+ default false
960
1325
  end
961
1326
  Vanity.playground.track! :coolness
962
1327
  assert_equal 0, experiment(:abcd).alternatives.sum(&:conversions)
@@ -967,6 +1332,7 @@ This experiment did not run long enough to find a clear winner.
967
1332
  new_ab_test :quick do
968
1333
  outcome_is { alternatives[1] }
969
1334
  metrics :coolness
1335
+ default false
970
1336
  end
971
1337
  experiment(:quick).complete!
972
1338
  assert_nil experiment(:quick).outcome
@@ -975,6 +1341,7 @@ This experiment did not run long enough to find a clear winner.
975
1341
  def test_chooses_records_participants
976
1342
  new_ab_test :simple do
977
1343
  alternatives :a, :b, :c
1344
+ default :a
978
1345
  metrics :coolness
979
1346
  end
980
1347
  experiment(:simple).chooses(:b)
@@ -984,6 +1351,7 @@ This experiment did not run long enough to find a clear winner.
984
1351
  def test_chooses_moves_participant_to_new_alternative
985
1352
  new_ab_test :simple do
986
1353
  alternatives :a, :b, :c
1354
+ default :a
987
1355
  metrics :coolness
988
1356
  identify { "1" }
989
1357
  end
@@ -998,6 +1366,7 @@ This experiment did not run long enough to find a clear winner.
998
1366
  def test_chooses_records_participants_only_once
999
1367
  new_ab_test :simple do
1000
1368
  alternatives :a, :b, :c
1369
+ default :a
1001
1370
  metrics :coolness
1002
1371
  end
1003
1372
  2.times { experiment(:simple).chooses(:b) }
@@ -1007,6 +1376,7 @@ This experiment did not run long enough to find a clear winner.
1007
1376
  def test_chooses_records_participants_for_new_alternatives
1008
1377
  new_ab_test :simple do
1009
1378
  alternatives :a, :b, :c
1379
+ default :a
1010
1380
  metrics :coolness
1011
1381
  end
1012
1382
  experiment(:simple).chooses(:b)
@@ -1017,6 +1387,7 @@ This experiment did not run long enough to find a clear winner.
1017
1387
  def test_chooses_records_participants_given_a_valid_request
1018
1388
  new_ab_test :simple do
1019
1389
  alternatives :a, :b, :c
1390
+ default :a
1020
1391
  metrics :coolness
1021
1392
  end
1022
1393
  experiment(:simple).chooses(:a, dummy_request)
@@ -1026,6 +1397,7 @@ This experiment did not run long enough to find a clear winner.
1026
1397
  def test_chooses_ignores_participants_given_an_invalid_request
1027
1398
  new_ab_test :simple do
1028
1399
  alternatives :a, :b, :c
1400
+ default :a
1029
1401
  metrics :coolness
1030
1402
  end
1031
1403
  request = dummy_request
@@ -1038,6 +1410,7 @@ This experiment did not run long enough to find a clear winner.
1038
1410
  not_collecting!
1039
1411
  new_ab_test :simple do
1040
1412
  alternatives :a, :b, :c
1413
+ default :a
1041
1414
  metrics :coolness
1042
1415
  end
1043
1416
  assert !experiment(:simple).showing?(experiment(:simple).alternatives[1])
@@ -1050,16 +1423,56 @@ This experiment did not run long enough to find a clear winner.
1050
1423
  not_collecting!
1051
1424
  new_ab_test :simple do
1052
1425
  alternatives :a, :b, :c
1426
+ default :a
1053
1427
  metrics :coolness
1054
1428
  end
1055
1429
  choice = experiment(:simple).choose.value
1056
1430
  assert [:a, :b, :c].include?(choice)
1057
1431
  assert_equal choice, experiment(:simple).choose.value
1058
1432
  end
1433
+
1434
+ # -- Reset --
1435
+
1436
+ def test_reset_clears_participants
1437
+ new_ab_test :simple do
1438
+ alternatives :a, :b, :c
1439
+ default :a
1440
+ metrics :coolness
1441
+ end
1442
+ experiment(:simple).chooses(:b)
1443
+ assert_equal experiment(:simple).alternatives[1].participants, 1
1444
+ experiment(:simple).reset
1445
+ assert_equal experiment(:simple).alternatives[1].participants, 0
1446
+ end
1447
+
1448
+ def test_clears_outcome_and_completed_at
1449
+ new_ab_test :simple do
1450
+ alternatives :a, :b, :c
1451
+ default :a
1452
+ metrics :coolness
1453
+ end
1454
+ experiment(:simple).reset
1455
+ assert_nil experiment(:simple).outcome
1456
+ assert_nil experiment(:simple).completed_at
1457
+ end
1458
+
1459
+ # -- Pick Winner --
1460
+
1461
+ def test_complete_with_argument_sets_outcome_and_completes
1462
+ new_ab_test :simple do
1463
+ alternatives :a, :b, :c
1464
+ default :a
1465
+ metrics :coolness
1466
+ end
1467
+ experiment(:simple).complete!(experiment(:simple).alternatives[1].id)
1468
+ assert_equal experiment(:simple).alternatives[1], experiment(:simple).outcome
1469
+ assert_not_nil experiment(:simple).completed_at
1470
+ end
1059
1471
 
1060
1472
  def test_reset_clears_participants
1061
1473
  new_ab_test :simple do
1062
1474
  alternatives :a, :b, :c
1475
+ default :a
1063
1476
  metrics :coolness
1064
1477
  end
1065
1478
  experiment(:simple).chooses(:b)
@@ -1071,6 +1484,7 @@ This experiment did not run long enough to find a clear winner.
1071
1484
  def test_clears_outcome_and_completed_at
1072
1485
  new_ab_test :simple do
1073
1486
  alternatives :a, :b, :c
1487
+ default :a
1074
1488
  metrics :coolness
1075
1489
  end
1076
1490
  experiment(:simple).reset