vanity 3.0.2 → 4.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linting.yml +28 -0
- data/.github/workflows/test.yml +3 -6
- data/.rubocop.yml +114 -0
- data/.rubocop_todo.yml +67 -0
- data/Appraisals +9 -31
- data/CHANGELOG +13 -0
- data/Gemfile +7 -3
- data/Gemfile.lock +32 -4
- data/README.md +4 -9
- data/Rakefile +25 -24
- data/bin/vanity +1 -1
- data/doc/configuring.textile +1 -0
- data/gemfiles/rails52.gemfile +6 -3
- data/gemfiles/rails52.gemfile.lock +34 -9
- data/gemfiles/rails60.gemfile +6 -3
- data/gemfiles/rails60.gemfile.lock +34 -9
- data/gemfiles/rails61.gemfile +6 -3
- data/gemfiles/rails61.gemfile.lock +34 -9
- data/lib/generators/vanity/migration_generator.rb +5 -7
- data/lib/vanity/adapters/abstract_adapter.rb +43 -45
- data/lib/vanity/adapters/active_record_adapter.rb +30 -30
- data/lib/vanity/adapters/mock_adapter.rb +14 -18
- data/lib/vanity/adapters/mongodb_adapter.rb +73 -69
- data/lib/vanity/adapters/redis_adapter.rb +19 -27
- data/lib/vanity/adapters.rb +1 -1
- data/lib/vanity/autoconnect.rb +6 -7
- data/lib/vanity/commands/list.rb +7 -7
- data/lib/vanity/commands/report.rb +18 -22
- data/lib/vanity/configuration.rb +23 -19
- data/lib/vanity/connection.rb +12 -14
- data/lib/vanity/experiment/ab_test.rb +95 -79
- data/lib/vanity/experiment/alternative.rb +3 -5
- data/lib/vanity/experiment/base.rb +24 -19
- data/lib/vanity/experiment/bayesian_bandit_score.rb +7 -13
- data/lib/vanity/experiment/definition.rb +6 -6
- data/lib/vanity/frameworks/rails.rb +39 -39
- data/lib/vanity/frameworks.rb +2 -2
- data/lib/vanity/helpers.rb +1 -1
- data/lib/vanity/metric/active_record.rb +21 -19
- data/lib/vanity/metric/base.rb +22 -23
- data/lib/vanity/metric/google_analytics.rb +6 -9
- data/lib/vanity/metric/remote.rb +3 -5
- data/lib/vanity/playground.rb +3 -6
- data/lib/vanity/vanity.rb +8 -12
- data/lib/vanity/version.rb +1 -1
- data/test/adapters/active_record_adapter_test.rb +1 -5
- data/test/adapters/mock_adapter_test.rb +0 -2
- data/test/adapters/mongodb_adapter_test.rb +1 -5
- data/test/adapters/redis_adapter_test.rb +2 -3
- data/test/adapters/shared_tests.rb +9 -12
- data/test/autoconnect_test.rb +3 -3
- data/test/cli_test.rb +0 -1
- data/test/configuration_test.rb +18 -34
- data/test/connection_test.rb +3 -3
- data/test/dummy/Rakefile +1 -1
- data/test/dummy/app/controllers/use_vanity_controller.rb +12 -8
- data/test/dummy/app/mailers/vanity_mailer.rb +3 -3
- data/test/dummy/config/application.rb +1 -1
- data/test/dummy/config/boot.rb +3 -3
- data/test/dummy/config/environment.rb +1 -1
- data/test/dummy/config/environments/development.rb +0 -1
- data/test/dummy/config/environments/test.rb +1 -1
- data/test/dummy/config/initializers/session_store.rb +1 -1
- data/test/dummy/config/initializers/vanity.rb +3 -0
- data/test/dummy/config/vanity.yml +7 -0
- data/test/dummy/config.ru +1 -1
- data/test/dummy/script/rails +2 -2
- data/test/experiment/ab_test.rb +188 -154
- data/test/experiment/base_test.rb +48 -32
- data/test/frameworks/rails/action_controller_test.rb +25 -25
- data/test/frameworks/rails/action_mailer_test.rb +2 -2
- data/test/frameworks/rails/action_view_test.rb +5 -6
- data/test/frameworks/rails/rails_test.rb +147 -181
- data/test/helper_test.rb +2 -2
- data/test/metric/active_record_test.rb +174 -212
- data/test/metric/base_test.rb +21 -46
- data/test/metric/google_analytics_test.rb +17 -25
- data/test/metric/remote_test.rb +7 -10
- data/test/playground_test.rb +7 -15
- data/test/templates_test.rb +16 -20
- data/test/test_helper.rb +28 -29
- data/test/vanity_test.rb +4 -10
- data/test/web/rails/dashboard_test.rb +21 -21
- data/vanity.gemspec +8 -7
- metadata +32 -30
- data/gemfiles/rails42.gemfile +0 -33
- data/gemfiles/rails42.gemfile.lock +0 -265
- data/gemfiles/rails42_protected_attributes.gemfile +0 -34
- data/gemfiles/rails42_protected_attributes.gemfile.lock +0 -264
- data/gemfiles/rails51.gemfile +0 -33
- data/gemfiles/rails51.gemfile.lock +0 -285
data/test/experiment/ab_test.rb
CHANGED
@@ -6,24 +6,23 @@ class AbTestController < ActionController::Base
|
|
6
6
|
|
7
7
|
def test_render
|
8
8
|
text = Vanity.ab_test(:simple)
|
9
|
-
render :
|
9
|
+
render plain: text, text: text
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_view
|
13
|
-
render :
|
13
|
+
render inline: "<%= ab_test(:simple) %>"
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_capture
|
17
|
-
render :
|
17
|
+
render inline: "<%= ab_test(:simple) do |value| %><%= value %><% end %>"
|
18
18
|
end
|
19
19
|
|
20
20
|
def track
|
21
21
|
Vanity.track!(:coolness)
|
22
|
-
render :
|
22
|
+
render plain: "", text: ""
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
26
|
class AbTestTest < ActionController::TestCase
|
28
27
|
tests AbTestController
|
29
28
|
|
@@ -85,7 +84,7 @@ class AbTestTest < ActionController::TestCase
|
|
85
84
|
alternatives :a, :b
|
86
85
|
default :a
|
87
86
|
end
|
88
|
-
fingerprints = Vanity.playground.experiments.map { |
|
87
|
+
fingerprints = Vanity.playground.experiments.map { |_id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
89
88
|
assert_equal 4, fingerprints.uniq.size
|
90
89
|
end
|
91
90
|
|
@@ -97,7 +96,7 @@ class AbTestTest < ActionController::TestCase
|
|
97
96
|
end
|
98
97
|
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
99
98
|
fingerprints.each do |fingerprint|
|
100
|
-
assert_match
|
99
|
+
assert_match(/^[0-9a-f]{10}$/i, fingerprint)
|
101
100
|
end
|
102
101
|
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
103
102
|
end
|
@@ -183,25 +182,24 @@ class AbTestTest < ActionController::TestCase
|
|
183
182
|
assert_equal exp.default, exp.alternative(nil)
|
184
183
|
end
|
185
184
|
|
186
|
-
|
187
185
|
# -- Experiment Enabled/disabled --
|
188
186
|
|
189
187
|
# @example new test should be enabled regardless of collecting?
|
190
188
|
# regardless_of "Vanity.playground.collecting" do
|
191
189
|
# assert (new_ab_test :test).enabled?
|
192
190
|
# end
|
193
|
-
def regardless_of(attr_name
|
194
|
-
prev_val = eval "#{attr_name}?"
|
191
|
+
def regardless_of(attr_name)
|
192
|
+
prev_val = eval "#{attr_name}?" # rubocop:todo Lint/UselessAssignment, Style/EvalWithLocation, Security/Eval
|
195
193
|
|
196
|
-
eval "#{attr_name}=true"
|
197
|
-
|
194
|
+
eval "#{attr_name}=true" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
195
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
198
196
|
nuke_playground
|
199
197
|
|
200
|
-
eval "#{attr_name}=false"
|
201
|
-
|
198
|
+
eval "#{attr_name}=false" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
199
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
202
200
|
nuke_playground
|
203
201
|
|
204
|
-
eval "#{attr_name}=prev_val"
|
202
|
+
eval "#{attr_name}=prev_val" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
205
203
|
end
|
206
204
|
|
207
205
|
def test_new_test_is_disabled_when_experiments_start_enabled_is_false
|
@@ -228,7 +226,7 @@ class AbTestTest < ActionController::TestCase
|
|
228
226
|
metrics :coolness
|
229
227
|
default false
|
230
228
|
end
|
231
|
-
exp.complete! #active? => false
|
229
|
+
exp.complete! # active? => false
|
232
230
|
|
233
231
|
assert !exp.enabled?, "experiment should not be enabled but it is!"
|
234
232
|
end
|
@@ -263,7 +261,7 @@ class AbTestTest < ActionController::TestCase
|
|
263
261
|
metrics :coolness
|
264
262
|
default false
|
265
263
|
end
|
266
|
-
exp.complete! #active? => false
|
264
|
+
exp.complete! # active? => false
|
267
265
|
assert !exp.enabled?
|
268
266
|
exp.enabled = true
|
269
267
|
assert !exp.enabled?
|
@@ -283,65 +281,65 @@ class AbTestTest < ActionController::TestCase
|
|
283
281
|
def test_enabled_persists_across_definitions
|
284
282
|
Vanity.configuration.experiments_start_enabled = false
|
285
283
|
Vanity.playground.collecting = true
|
286
|
-
new_ab_test :test, :
|
284
|
+
new_ab_test :test, enable: false do
|
287
285
|
metrics :coolness
|
288
286
|
default false
|
289
287
|
end
|
290
|
-
assert !experiment(:test).enabled? #starts off false
|
288
|
+
assert !experiment(:test).enabled? # starts off false
|
291
289
|
|
292
290
|
new_playground
|
293
291
|
metric "Coolness"
|
294
292
|
|
295
|
-
new_ab_test :test, :
|
293
|
+
new_ab_test :test, enable: false do
|
296
294
|
metrics :coolness
|
297
295
|
default false
|
298
296
|
end
|
299
|
-
assert !experiment(:test).enabled? #still false
|
297
|
+
assert !experiment(:test).enabled? # still false
|
300
298
|
experiment(:test).enabled = true
|
301
|
-
assert experiment(:test).enabled? #now true
|
299
|
+
assert experiment(:test).enabled? # now true
|
302
300
|
|
303
301
|
new_playground
|
304
302
|
metric "Coolness"
|
305
303
|
|
306
|
-
new_ab_test :test, :
|
304
|
+
new_ab_test :test, enable: false do
|
307
305
|
metrics :coolness
|
308
306
|
default false
|
309
307
|
end
|
310
|
-
assert experiment(:test).enabled? #still true
|
308
|
+
assert experiment(:test).enabled? # still true
|
311
309
|
experiment(:test).enabled = false
|
312
|
-
assert !experiment(:test).enabled? #now false again
|
310
|
+
assert !experiment(:test).enabled? # now false again
|
313
311
|
end
|
314
312
|
|
315
313
|
def test_enabled_persists_across_definitions_when_starting_enabled
|
316
314
|
Vanity.configuration.experiments_start_enabled = true
|
317
315
|
Vanity.playground.collecting = true
|
318
|
-
new_ab_test :test, :
|
316
|
+
new_ab_test :test, enable: false do
|
319
317
|
metrics :coolness
|
320
318
|
default false
|
321
319
|
end
|
322
|
-
assert experiment(:test).enabled? #starts off true
|
320
|
+
assert experiment(:test).enabled? # starts off true
|
323
321
|
|
324
322
|
new_playground
|
325
323
|
metric "Coolness"
|
326
324
|
|
327
|
-
new_ab_test :test, :
|
325
|
+
new_ab_test :test, enable: false do
|
328
326
|
metrics :coolness
|
329
327
|
default false
|
330
328
|
end
|
331
|
-
assert experiment(:test).enabled? #still true
|
329
|
+
assert experiment(:test).enabled? # still true
|
332
330
|
experiment(:test).enabled = false
|
333
|
-
assert !experiment(:test).enabled? #now false
|
331
|
+
assert !experiment(:test).enabled? # now false
|
334
332
|
|
335
333
|
new_playground
|
336
334
|
metric "Coolness"
|
337
335
|
|
338
|
-
new_ab_test :test, :
|
336
|
+
new_ab_test :test, enable: false do
|
339
337
|
metrics :coolness
|
340
338
|
default false
|
341
339
|
end
|
342
|
-
assert !experiment(:test).enabled? #still false
|
340
|
+
assert !experiment(:test).enabled? # still false
|
343
341
|
experiment(:test).enabled = true
|
344
|
-
assert experiment(:test).enabled? #now true again
|
342
|
+
assert experiment(:test).enabled? # now true again
|
345
343
|
end
|
346
344
|
|
347
345
|
def test_choose_random_when_enabled
|
@@ -378,7 +376,7 @@ class AbTestTest < ActionController::TestCase
|
|
378
376
|
def test_choose_outcome_when_finished
|
379
377
|
exp = new_ab_test :test do
|
380
378
|
metrics :coolness
|
381
|
-
alternatives 0,1,2,3,4,5
|
379
|
+
alternatives 0, 1, 2, 3, 4, 5
|
382
380
|
default 3
|
383
381
|
outcome_is { alternative(5) }
|
384
382
|
end
|
@@ -429,7 +427,7 @@ class AbTestTest < ActionController::TestCase
|
|
429
427
|
assert_equal 0, experiment(:foobar).alternatives.sum(&:converted)
|
430
428
|
experiment(:foobar).track!(:coolness, Time.now, 1)
|
431
429
|
assert_equal 1, experiment(:foobar).alternatives.sum(&:converted)
|
432
|
-
experiment(:foobar).track!(:coolness, Time.now, 1, :
|
430
|
+
experiment(:foobar).track!(:coolness, Time.now, 1, identity: "quux")
|
433
431
|
assert_equal 2, experiment(:foobar).alternatives.sum(&:converted)
|
434
432
|
end
|
435
433
|
|
@@ -458,7 +456,7 @@ class AbTestTest < ActionController::TestCase
|
|
458
456
|
alternatives "foo", "bar"
|
459
457
|
identify { "6e98ec" }
|
460
458
|
metrics :coolness
|
461
|
-
on_assignment { on_assignment_called_times
|
459
|
+
on_assignment { on_assignment_called_times += 1 }
|
462
460
|
end
|
463
461
|
2.times { experiment(:foobar).choose }
|
464
462
|
assert_equal 1, on_assignment_called_times
|
@@ -471,7 +469,7 @@ class AbTestTest < ActionController::TestCase
|
|
471
469
|
alternatives "foo", "bar"
|
472
470
|
identify { "6e98ec" }
|
473
471
|
metrics :coolness
|
474
|
-
on_assignment { on_assignment_called_times
|
472
|
+
on_assignment { on_assignment_called_times += 1 }
|
475
473
|
end
|
476
474
|
experiment(:foobar).choose(dummy_request)
|
477
475
|
assert_equal 1, on_assignment_called_times
|
@@ -484,7 +482,7 @@ class AbTestTest < ActionController::TestCase
|
|
484
482
|
alternatives "foo", "bar"
|
485
483
|
identify { "6e98ec" }
|
486
484
|
metrics :coolness
|
487
|
-
on_assignment { on_assignment_called_times
|
485
|
+
on_assignment { on_assignment_called_times += 1 }
|
488
486
|
end
|
489
487
|
request = dummy_request
|
490
488
|
request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
|
@@ -499,7 +497,7 @@ class AbTestTest < ActionController::TestCase
|
|
499
497
|
default "foo"
|
500
498
|
identify { "6e98ec" }
|
501
499
|
metrics :coolness
|
502
|
-
on_assignment { on_assignment_called_times
|
500
|
+
on_assignment { on_assignment_called_times += 1 }
|
503
501
|
end
|
504
502
|
2.times { experiment(:foobar).chooses("foo") }
|
505
503
|
assert_equal 1, on_assignment_called_times
|
@@ -510,16 +508,56 @@ class AbTestTest < ActionController::TestCase
|
|
510
508
|
alternatives "foo", "bar"
|
511
509
|
default "foo"
|
512
510
|
identify { "6e98ec" }
|
513
|
-
on_assignment {}
|
511
|
+
on_assignment {} # rubocop:todo Lint/EmptyBlock
|
514
512
|
metrics :coolness
|
515
513
|
end
|
516
514
|
assert value = experiment(:foobar).choose.value
|
517
|
-
assert_match
|
515
|
+
assert_match(/foo|bar/, value)
|
518
516
|
1000.times do
|
519
517
|
assert_equal value, experiment(:foobar).choose.value
|
520
518
|
end
|
521
519
|
end
|
522
520
|
|
521
|
+
def test_calls_default_on_assignment
|
522
|
+
on_assignment_called_times = 0
|
523
|
+
|
524
|
+
Vanity.configuration.on_assignment = proc do |_controller, _identity, _assignment|
|
525
|
+
on_assignment_called_times += 1
|
526
|
+
end
|
527
|
+
|
528
|
+
new_ab_test :foobar do
|
529
|
+
alternatives "foo", "bar"
|
530
|
+
default "foo"
|
531
|
+
identify { "6e98ec" }
|
532
|
+
metrics :coolness
|
533
|
+
end
|
534
|
+
|
535
|
+
2.times { experiment(:foobar).chooses("foo") }
|
536
|
+
assert_equal 1, on_assignment_called_times
|
537
|
+
end
|
538
|
+
|
539
|
+
def test_calls_on_assignment_defined_in_experiment
|
540
|
+
expected_value = 0
|
541
|
+
|
542
|
+
Vanity.configuration.on_assignment = proc do |_controller, _identity, _assignment|
|
543
|
+
expected_value = 1
|
544
|
+
end
|
545
|
+
|
546
|
+
new_ab_test :foobar do
|
547
|
+
alternatives "foo", "bar"
|
548
|
+
default "foo"
|
549
|
+
identify { "6e98ec" }
|
550
|
+
metrics :coolness
|
551
|
+
|
552
|
+
on_assignment do |_controller, _identity, _assignment|
|
553
|
+
expected_value = 20
|
554
|
+
end
|
555
|
+
end
|
556
|
+
|
557
|
+
2.times { experiment(:foobar).chooses("foo") }
|
558
|
+
assert_equal 20, expected_value
|
559
|
+
end
|
560
|
+
|
523
561
|
# -- ab_assigned --
|
524
562
|
|
525
563
|
def test_ab_assigned
|
@@ -535,7 +573,7 @@ class AbTestTest < ActionController::TestCase
|
|
535
573
|
end
|
536
574
|
|
537
575
|
def test_ab_assigned_object
|
538
|
-
identity = { :
|
576
|
+
identity = { a: :b }
|
539
577
|
new_ab_test :foobar do
|
540
578
|
alternatives "foo", "bar"
|
541
579
|
default "foo"
|
@@ -559,7 +597,7 @@ class AbTestTest < ActionController::TestCase
|
|
559
597
|
end
|
560
598
|
value = experiment(:foobar).choose.value
|
561
599
|
assert value
|
562
|
-
assert_match
|
600
|
+
assert_match(/foo|bar/, value)
|
563
601
|
100.times do
|
564
602
|
assert_equal value, experiment(:foobar).choose.value
|
565
603
|
end
|
@@ -571,7 +609,10 @@ class AbTestTest < ActionController::TestCase
|
|
571
609
|
identify { rand }
|
572
610
|
metrics :coolness
|
573
611
|
end
|
574
|
-
alts = Array.new(10_000) { experiment(:foobar).choose.value }.
|
612
|
+
alts = Array.new(10_000) { experiment(:foobar).choose.value }.each_with_object({}) do |k, h|
|
613
|
+
h[k] ||= 0
|
614
|
+
h[k] += 1
|
615
|
+
end
|
575
616
|
assert_equal %w{bar foo}, alts.keys.sort
|
576
617
|
assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
|
577
618
|
end
|
@@ -585,8 +626,8 @@ class AbTestTest < ActionController::TestCase
|
|
585
626
|
metrics :coolness
|
586
627
|
end
|
587
628
|
altered_alts = experiment(:foobar).alternatives
|
588
|
-
altered_alts[0].probability=30
|
589
|
-
altered_alts[1].probability=70
|
629
|
+
altered_alts[0].probability = 30
|
630
|
+
altered_alts[1].probability = 70
|
590
631
|
experiment(:foobar).set_alternative_probabilities altered_alts
|
591
632
|
alts = Array.new(600) { experiment(:foobar).choose.value }
|
592
633
|
assert_equal %w{bar foo}, alts.uniq.sort
|
@@ -603,10 +644,11 @@ class AbTestTest < ActionController::TestCase
|
|
603
644
|
rebalance_frequency 12
|
604
645
|
metrics :coolness
|
605
646
|
end
|
606
|
-
class <<experiment(:foobar)
|
647
|
+
class << experiment(:foobar)
|
607
648
|
def times_called
|
608
649
|
@times_called ||= 0
|
609
650
|
end
|
651
|
+
|
610
652
|
def rebalance!
|
611
653
|
@times_called = times_called + 1
|
612
654
|
end
|
@@ -629,20 +671,16 @@ class AbTestTest < ActionController::TestCase
|
|
629
671
|
end
|
630
672
|
corresponding_probabilities = [[experiment(:foobar).alternatives[0], 0.3], [experiment(:foobar).alternatives[1], 0.6], [experiment(:foobar).alternatives[2], 1.0]]
|
631
673
|
|
632
|
-
class <<experiment(:foobar)
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
def bayes_bandit_score(probability=90)
|
674
|
+
class << experiment(:foobar)
|
675
|
+
attr_reader :was_called, :use_probabilities
|
676
|
+
|
677
|
+
def bayes_bandit_score(_probability = 90)
|
637
678
|
@was_called = true
|
638
679
|
altered_alts = Vanity.playground.experiment(:foobar).alternatives
|
639
|
-
altered_alts[0].probability=30
|
640
|
-
altered_alts[1].probability=30
|
641
|
-
altered_alts[2].probability=40
|
642
|
-
Struct.new(:alts
|
643
|
-
end
|
644
|
-
def use_probabilities
|
645
|
-
@use_probabilities
|
680
|
+
altered_alts[0].probability = 30
|
681
|
+
altered_alts[1].probability = 30
|
682
|
+
altered_alts[2].probability = 40
|
683
|
+
Struct.new(:alts, :method).new(altered_alts, :bayes_bandit_score) # rubocop:todo Lint/StructNewOverride
|
646
684
|
end
|
647
685
|
end
|
648
686
|
experiment(:foobar).rebalance!
|
@@ -660,7 +698,7 @@ class AbTestTest < ActionController::TestCase
|
|
660
698
|
metrics :coolness
|
661
699
|
end
|
662
700
|
assert value = experiment(:foobar).choose.value
|
663
|
-
assert_match
|
701
|
+
assert_match(/foo|bar/, value)
|
664
702
|
1000.times do
|
665
703
|
assert_equal value, experiment(:foobar).choose.value
|
666
704
|
end
|
@@ -712,7 +750,7 @@ class AbTestTest < ActionController::TestCase
|
|
712
750
|
end
|
713
751
|
|
714
752
|
def test_records_each_converted_participant_only_once
|
715
|
-
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
753
|
+
ids = ((1..100).map { |i| [i, i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
716
754
|
new_ab_test :foobar do
|
717
755
|
alternatives "foo", "bar"
|
718
756
|
default "foo"
|
@@ -728,7 +766,7 @@ class AbTestTest < ActionController::TestCase
|
|
728
766
|
end
|
729
767
|
|
730
768
|
def test_records_conversion_only_for_participants
|
731
|
-
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
769
|
+
ids = ((1..100).map { |i| [-i, i, i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
732
770
|
new_ab_test :foobar do
|
733
771
|
alternatives "foo", "bar"
|
734
772
|
default "foo"
|
@@ -790,7 +828,6 @@ class AbTestTest < ActionController::TestCase
|
|
790
828
|
assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
|
791
829
|
end
|
792
830
|
|
793
|
-
|
794
831
|
# -- A/B helper methods --
|
795
832
|
|
796
833
|
def test_fail_if_no_experiment
|
@@ -806,7 +843,8 @@ class AbTestTest < ActionController::TestCase
|
|
806
843
|
default false
|
807
844
|
end
|
808
845
|
responses = Array.new(100) do
|
809
|
-
@controller = nil
|
846
|
+
@controller = nil
|
847
|
+
setup_controller_request_and_response
|
810
848
|
get :test_render
|
811
849
|
@response.body
|
812
850
|
end
|
@@ -820,7 +858,8 @@ class AbTestTest < ActionController::TestCase
|
|
820
858
|
default false
|
821
859
|
end
|
822
860
|
responses = Array.new(100) do
|
823
|
-
@controller = nil
|
861
|
+
@controller = nil
|
862
|
+
setup_controller_request_and_response
|
824
863
|
get :test_view
|
825
864
|
@response.body
|
826
865
|
end
|
@@ -834,7 +873,8 @@ class AbTestTest < ActionController::TestCase
|
|
834
873
|
default false
|
835
874
|
end
|
836
875
|
responses = Array.new(100) do
|
837
|
-
@controller = nil
|
876
|
+
@controller = nil
|
877
|
+
setup_controller_request_and_response
|
838
878
|
get :test_capture
|
839
879
|
@response.body
|
840
880
|
end
|
@@ -846,14 +886,13 @@ class AbTestTest < ActionController::TestCase
|
|
846
886
|
metrics :coolness
|
847
887
|
default false
|
848
888
|
end
|
849
|
-
responses = Array.new(100) do
|
850
|
-
@controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
|
889
|
+
responses = Array.new(100) do # rubocop:todo Lint/UselessAssignment
|
890
|
+
@controller.send(:cookies).each { |cookie| @controller.send(:cookies).delete(cookie.first) }
|
851
891
|
get :track
|
852
892
|
@response.body
|
853
893
|
end
|
854
894
|
end
|
855
895
|
|
856
|
-
|
857
896
|
# -- Testing with tests --
|
858
897
|
|
859
898
|
def test_with_given_choice
|
@@ -862,8 +901,9 @@ class AbTestTest < ActionController::TestCase
|
|
862
901
|
default :a
|
863
902
|
metrics :coolness
|
864
903
|
end
|
865
|
-
100.times do |
|
866
|
-
@controller = nil
|
904
|
+
100.times do |_i|
|
905
|
+
@controller = nil
|
906
|
+
setup_controller_request_and_response
|
867
907
|
experiment(:simple).chooses(:b)
|
868
908
|
get :test_render
|
869
909
|
assert "b", @response.body
|
@@ -887,8 +927,9 @@ class AbTestTest < ActionController::TestCase
|
|
887
927
|
default :a
|
888
928
|
metrics :coolness
|
889
929
|
end
|
890
|
-
responses = Array.new(100) do |
|
891
|
-
@controller = nil
|
930
|
+
responses = Array.new(100) do |_i|
|
931
|
+
@controller = nil
|
932
|
+
setup_controller_request_and_response
|
892
933
|
experiment(:simple).chooses(:b)
|
893
934
|
experiment(:simple).chooses(nil)
|
894
935
|
get :test_render
|
@@ -897,7 +938,6 @@ class AbTestTest < ActionController::TestCase
|
|
897
938
|
assert responses.uniq.size == 3
|
898
939
|
end
|
899
940
|
|
900
|
-
|
901
941
|
# -- Scoring --
|
902
942
|
|
903
943
|
def test_calculate_score
|
@@ -930,9 +970,9 @@ class AbTestTest < ActionController::TestCase
|
|
930
970
|
# Treatment A: 180 45 25.00% 1.33
|
931
971
|
# treatment B: 189 28 14.81% -1.13
|
932
972
|
# treatment C: 188 61 32.45% 2.94
|
933
|
-
fake :abcd, :
|
973
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
934
974
|
|
935
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
975
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score } # rubocop:todo Style/FormatString
|
936
976
|
assert_equal %w{-1.33 0.00 -2.46 1.58}, z_scores
|
937
977
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
938
978
|
assert_equal [90, 0, 99, 90], probabilities
|
@@ -957,11 +997,11 @@ class AbTestTest < ActionController::TestCase
|
|
957
997
|
# Treatment A: 180 45 25.00% 1.33
|
958
998
|
# treatment B: 189 28 14.81% -1.13
|
959
999
|
# treatment C: 188 61 32.45% 2.94
|
960
|
-
fake :abcd, :
|
1000
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
961
1001
|
|
962
1002
|
score_result = experiment(:abcd).bayes_bandit_score
|
963
|
-
probabilities = score_result.alts.map{|a| a.probability.round}
|
964
|
-
assert_equal [0,0,6,94], probabilities
|
1003
|
+
probabilities = score_result.alts.map { |a| a.probability.round }
|
1004
|
+
assert_equal [0, 0, 6, 94], probabilities
|
965
1005
|
end
|
966
1006
|
|
967
1007
|
def test_scoring_with_no_performers
|
@@ -984,13 +1024,13 @@ class AbTestTest < ActionController::TestCase
|
|
984
1024
|
default :a
|
985
1025
|
metrics :coolness
|
986
1026
|
end
|
987
|
-
fake :abcd, :
|
1027
|
+
fake :abcd, b: [10, 8]
|
988
1028
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
989
1029
|
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
990
1030
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
991
1031
|
assert_equal 1, experiment(:abcd).score.best.id
|
992
1032
|
assert_nil experiment(:abcd).score.choice
|
993
|
-
assert_includes [0,2,3], experiment(:abcd).score.base.id
|
1033
|
+
assert_includes [0, 2, 3], experiment(:abcd).score.base.id
|
994
1034
|
assert_equal 1, experiment(:abcd).score.least.id
|
995
1035
|
end
|
996
1036
|
|
@@ -1000,9 +1040,9 @@ class AbTestTest < ActionController::TestCase
|
|
1000
1040
|
default :a
|
1001
1041
|
metrics :coolness
|
1002
1042
|
end
|
1003
|
-
fake :abcd, :
|
1043
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1004
1044
|
|
1005
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase)
|
1045
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase) # rubocop:todo Style/FormatString
|
1006
1046
|
assert_equal %w{nan 2.01 nan 0.00}, z_scores
|
1007
1047
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
1008
1048
|
assert_equal [0, 95, 0, 0], probabilities
|
@@ -1020,14 +1060,13 @@ class AbTestTest < ActionController::TestCase
|
|
1020
1060
|
default :a
|
1021
1061
|
metrics :coolness
|
1022
1062
|
end
|
1023
|
-
fake :abcd, :
|
1063
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1024
1064
|
|
1025
1065
|
assert_equal 1, experiment(:abcd).score(90).choice.id
|
1026
1066
|
assert_equal 1, experiment(:abcd).score(95).choice.id
|
1027
1067
|
assert_nil experiment(:abcd).score(99).choice
|
1028
1068
|
end
|
1029
1069
|
|
1030
|
-
|
1031
1070
|
# -- Conclusion --
|
1032
1071
|
|
1033
1072
|
def test_conclusion
|
@@ -1041,16 +1080,16 @@ class AbTestTest < ActionController::TestCase
|
|
1041
1080
|
# Treatment A: 180 45 25.00% 1.33
|
1042
1081
|
# treatment B: 189 28 14.81% -1.13
|
1043
1082
|
# treatment C: 188 61 32.45% 2.94
|
1044
|
-
fake :abcd, :
|
1045
|
-
|
1046
|
-
assert_equal
|
1047
|
-
There are 739 participants in this experiment.
|
1048
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1049
|
-
With 90% probability this result is statistically significant.
|
1050
|
-
Option B converted at 25.0%.
|
1051
|
-
Option A converted at 19.2%.
|
1052
|
-
Option C converted at 14.8%.
|
1053
|
-
Option D selected as the best alternative.
|
1083
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1084
|
+
|
1085
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1086
|
+
There are 739 participants in this experiment.
|
1087
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1088
|
+
With 90% probability this result is statistically significant.
|
1089
|
+
Option B converted at 25.0%.
|
1090
|
+
Option A converted at 19.2%.
|
1091
|
+
Option C converted at 14.8%.
|
1092
|
+
Option D selected as the best alternative.
|
1054
1093
|
TEXT
|
1055
1094
|
end
|
1056
1095
|
|
@@ -1060,16 +1099,16 @@ Option D selected as the best alternative.
|
|
1060
1099
|
default :a
|
1061
1100
|
metrics :coolness
|
1062
1101
|
end
|
1063
|
-
fake :abcd, :
|
1102
|
+
fake :abcd, b: [180, 45], d: [188, 61]
|
1064
1103
|
|
1065
|
-
assert_equal
|
1066
|
-
There are 368 participants in this experiment.
|
1067
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1068
|
-
With 90% probability this result is statistically significant.
|
1069
|
-
Option B converted at 25.0%.
|
1070
|
-
Option A did not convert.
|
1071
|
-
Option C did not convert.
|
1072
|
-
Option D selected as the best alternative.
|
1104
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1105
|
+
There are 368 participants in this experiment.
|
1106
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1107
|
+
With 90% probability this result is statistically significant.
|
1108
|
+
Option B converted at 25.0%.
|
1109
|
+
Option A did not convert.
|
1110
|
+
Option C did not convert.
|
1111
|
+
Option D selected as the best alternative.
|
1073
1112
|
TEXT
|
1074
1113
|
end
|
1075
1114
|
|
@@ -1079,15 +1118,15 @@ Option D selected as the best alternative.
|
|
1079
1118
|
default :a
|
1080
1119
|
metrics :coolness
|
1081
1120
|
end
|
1082
|
-
fake :abcd, :
|
1121
|
+
fake :abcd, b: [180, 58], d: [188, 61]
|
1083
1122
|
|
1084
|
-
assert_equal
|
1085
|
-
There are 368 participants in this experiment.
|
1086
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1087
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1088
|
-
Option B converted at 32.2%.
|
1089
|
-
Option A did not convert.
|
1090
|
-
Option C did not convert.
|
1123
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1124
|
+
There are 368 participants in this experiment.
|
1125
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1126
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1127
|
+
Option B converted at 32.2%.
|
1128
|
+
Option A did not convert.
|
1129
|
+
Option C did not convert.
|
1091
1130
|
TEXT
|
1092
1131
|
end
|
1093
1132
|
|
@@ -1097,15 +1136,15 @@ Option C did not convert.
|
|
1097
1136
|
default :a
|
1098
1137
|
metrics :coolness
|
1099
1138
|
end
|
1100
|
-
fake :abcd, :
|
1139
|
+
fake :abcd, b: [186, 60], d: [188, 61]
|
1101
1140
|
|
1102
|
-
assert_equal
|
1103
|
-
There are 374 participants in this experiment.
|
1104
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1105
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1106
|
-
Option B converted at 32.3%.
|
1107
|
-
Option A did not convert.
|
1108
|
-
Option C did not convert.
|
1141
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1142
|
+
There are 374 participants in this experiment.
|
1143
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1144
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1145
|
+
Option B converted at 32.3%.
|
1146
|
+
Option A did not convert.
|
1147
|
+
Option C did not convert.
|
1109
1148
|
TEXT
|
1110
1149
|
end
|
1111
1150
|
|
@@ -1115,14 +1154,14 @@ Option C did not convert.
|
|
1115
1154
|
default :a
|
1116
1155
|
metrics :coolness
|
1117
1156
|
end
|
1118
|
-
fake :abcd, :
|
1157
|
+
fake :abcd, b: [188, 61], d: [188, 61]
|
1119
1158
|
|
1120
|
-
assert_equal
|
1121
|
-
There are 376 participants in this experiment.
|
1122
|
-
Option D converted at 32.4%.
|
1123
|
-
Option B converted at 32.4%.
|
1124
|
-
Option A did not convert.
|
1125
|
-
Option C did not convert.
|
1159
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1160
|
+
There are 376 participants in this experiment.
|
1161
|
+
Option D converted at 32.4%.
|
1162
|
+
Option B converted at 32.4%.
|
1163
|
+
Option A did not convert.
|
1164
|
+
Option C did not convert.
|
1126
1165
|
TEXT
|
1127
1166
|
end
|
1128
1167
|
|
@@ -1132,11 +1171,11 @@ Option C did not convert.
|
|
1132
1171
|
default :a
|
1133
1172
|
metrics :coolness
|
1134
1173
|
end
|
1135
|
-
fake :abcd, :
|
1174
|
+
fake :abcd, b: [180, 45]
|
1136
1175
|
|
1137
|
-
assert_equal
|
1138
|
-
There are 180 participants in this experiment.
|
1139
|
-
This experiment did not run long enough to find a clear winner.
|
1176
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1177
|
+
There are 180 participants in this experiment.
|
1178
|
+
This experiment did not run long enough to find a clear winner.
|
1140
1179
|
TEXT
|
1141
1180
|
end
|
1142
1181
|
|
@@ -1146,13 +1185,12 @@ This experiment did not run long enough to find a clear winner.
|
|
1146
1185
|
default :a
|
1147
1186
|
metrics :coolness
|
1148
1187
|
end
|
1149
|
-
assert_equal
|
1150
|
-
There are no participants in this experiment yet.
|
1151
|
-
This experiment did not run long enough to find a clear winner.
|
1188
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1189
|
+
There are no participants in this experiment yet.
|
1190
|
+
This experiment did not run long enough to find a clear winner.
|
1152
1191
|
TEXT
|
1153
1192
|
end
|
1154
1193
|
|
1155
|
-
|
1156
1194
|
# -- Completion --
|
1157
1195
|
|
1158
1196
|
def test_completion_if
|
@@ -1169,7 +1207,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1169
1207
|
def test_completion_if_fails
|
1170
1208
|
new_ab_test :simple do
|
1171
1209
|
identify { rand }
|
1172
|
-
complete_if {
|
1210
|
+
complete_if { raise "Testing complete_if raises exception" }
|
1173
1211
|
metrics :coolness
|
1174
1212
|
default false
|
1175
1213
|
end
|
@@ -1185,7 +1223,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1185
1223
|
metrics :coolness
|
1186
1224
|
default false
|
1187
1225
|
end
|
1188
|
-
99.times do |
|
1226
|
+
99.times do |_i|
|
1189
1227
|
experiment(:simple).choose
|
1190
1228
|
assert experiment(:simple).active?
|
1191
1229
|
end
|
@@ -1222,7 +1260,6 @@ This experiment did not run long enough to find a clear winner.
|
|
1222
1260
|
assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
|
1223
1261
|
end
|
1224
1262
|
|
1225
|
-
|
1226
1263
|
# -- Outcome --
|
1227
1264
|
|
1228
1265
|
def test_completion_outcome
|
@@ -1279,7 +1316,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1279
1316
|
|
1280
1317
|
def test_outcome_is_fails
|
1281
1318
|
new_ab_test :quick do
|
1282
|
-
outcome_is {
|
1319
|
+
outcome_is { raise "Testing outcome_is raising exception" }
|
1283
1320
|
metrics :coolness
|
1284
1321
|
default false
|
1285
1322
|
end
|
@@ -1292,7 +1329,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1292
1329
|
metrics :coolness
|
1293
1330
|
default false
|
1294
1331
|
end
|
1295
|
-
fake :quick, false=>[2,0], true=>10
|
1332
|
+
fake :quick, false => [2, 0], true => 10
|
1296
1333
|
experiment(:quick).complete!
|
1297
1334
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1298
1335
|
end
|
@@ -1302,7 +1339,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1302
1339
|
metrics :coolness
|
1303
1340
|
default false
|
1304
1341
|
end
|
1305
|
-
fake :quick, true=>2
|
1342
|
+
fake :quick, true => 2
|
1306
1343
|
experiment(:quick).complete!
|
1307
1344
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1308
1345
|
end
|
@@ -1312,12 +1349,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1312
1349
|
metrics :coolness
|
1313
1350
|
default false
|
1314
1351
|
end
|
1315
|
-
fake :quick, false=>8, true=>8
|
1352
|
+
fake :quick, false => 8, true => 8
|
1316
1353
|
experiment(:quick).complete!
|
1317
1354
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1318
1355
|
end
|
1319
1356
|
|
1320
|
-
|
1321
1357
|
# -- No collection --
|
1322
1358
|
|
1323
1359
|
def test_no_collection_does_not_track
|
@@ -1360,7 +1396,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1360
1396
|
identify { "1" }
|
1361
1397
|
end
|
1362
1398
|
val = experiment(:simple).choose.value
|
1363
|
-
alternative = experiment(:simple).alternatives.detect {|a| a.value != val }
|
1399
|
+
alternative = experiment(:simple).alternatives.detect { |a| a.value != val }
|
1364
1400
|
experiment(:simple).chooses(alternative.value)
|
1365
1401
|
assert_equal experiment(:simple).choose.value, alternative.value
|
1366
1402
|
experiment(:simple).chooses(val)
|
@@ -1441,7 +1477,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1441
1477
|
default :b
|
1442
1478
|
metrics :coolness
|
1443
1479
|
|
1444
|
-
reject do |
|
1480
|
+
reject do |_request, _identity|
|
1445
1481
|
true
|
1446
1482
|
end
|
1447
1483
|
end
|
@@ -1468,11 +1504,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1468
1504
|
end
|
1469
1505
|
|
1470
1506
|
def test_clears_outcome_and_completed_at
|
1471
|
-
|
1472
|
-
|
1473
|
-
|
1474
|
-
|
1475
|
-
|
1507
|
+
new_ab_test :simple do
|
1508
|
+
alternatives :a, :b, :c
|
1509
|
+
default :a
|
1510
|
+
metrics :coolness
|
1511
|
+
end
|
1476
1512
|
experiment(:simple).reset
|
1477
1513
|
assert_nil experiment(:simple).outcome
|
1478
1514
|
assert_nil experiment(:simple).completed_at
|
@@ -1491,7 +1527,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1491
1527
|
assert_not_nil experiment(:simple).completed_at
|
1492
1528
|
end
|
1493
1529
|
|
1494
|
-
def test_reset_clears_participants
|
1530
|
+
def test_reset_clears_participants # rubocop:todo Lint/DuplicateMethods
|
1495
1531
|
new_ab_test :simple do
|
1496
1532
|
alternatives :a, :b, :c
|
1497
1533
|
default :a
|
@@ -1503,7 +1539,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1503
1539
|
assert_equal experiment(:simple).alternatives[1].participants, 0
|
1504
1540
|
end
|
1505
1541
|
|
1506
|
-
def test_clears_outcome_and_completed_at
|
1542
|
+
def test_clears_outcome_and_completed_at # rubocop:todo Lint/DuplicateMethods
|
1507
1543
|
new_ab_test :simple do
|
1508
1544
|
alternatives :a, :b, :c
|
1509
1545
|
default :a
|
@@ -1514,11 +1550,9 @@ This experiment did not run long enough to find a clear winner.
|
|
1514
1550
|
assert_nil experiment(:simple).completed_at
|
1515
1551
|
end
|
1516
1552
|
|
1517
|
-
|
1518
1553
|
# -- Helper methods --
|
1519
1554
|
|
1520
1555
|
def fake(name, args)
|
1521
1556
|
experiment(name).instance_eval { fake args }
|
1522
1557
|
end
|
1523
|
-
|
1524
1558
|
end
|