vanity 3.1.0 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linting.yml +28 -0
- data/.github/workflows/test.yml +3 -6
- data/.rubocop.yml +114 -0
- data/.rubocop_todo.yml +67 -0
- data/Appraisals +9 -31
- data/CHANGELOG +5 -0
- data/Gemfile +7 -3
- data/Gemfile.lock +31 -3
- data/README.md +4 -9
- data/Rakefile +25 -24
- data/bin/vanity +1 -1
- data/doc/configuring.textile +1 -0
- data/gemfiles/rails52.gemfile +6 -3
- data/gemfiles/rails52.gemfile.lock +34 -9
- data/gemfiles/rails60.gemfile +6 -3
- data/gemfiles/rails60.gemfile.lock +34 -9
- data/gemfiles/rails61.gemfile +6 -3
- data/gemfiles/rails61.gemfile.lock +34 -9
- data/lib/generators/vanity/migration_generator.rb +5 -7
- data/lib/vanity/adapters/abstract_adapter.rb +43 -45
- data/lib/vanity/adapters/active_record_adapter.rb +30 -30
- data/lib/vanity/adapters/mock_adapter.rb +14 -18
- data/lib/vanity/adapters/mongodb_adapter.rb +73 -69
- data/lib/vanity/adapters/redis_adapter.rb +19 -27
- data/lib/vanity/adapters.rb +1 -1
- data/lib/vanity/autoconnect.rb +6 -7
- data/lib/vanity/commands/list.rb +7 -7
- data/lib/vanity/commands/report.rb +18 -22
- data/lib/vanity/configuration.rb +19 -19
- data/lib/vanity/connection.rb +12 -14
- data/lib/vanity/experiment/ab_test.rb +82 -70
- data/lib/vanity/experiment/alternative.rb +3 -5
- data/lib/vanity/experiment/base.rb +24 -19
- data/lib/vanity/experiment/bayesian_bandit_score.rb +7 -13
- data/lib/vanity/experiment/definition.rb +6 -6
- data/lib/vanity/frameworks/rails.rb +39 -39
- data/lib/vanity/frameworks.rb +2 -2
- data/lib/vanity/helpers.rb +1 -1
- data/lib/vanity/metric/active_record.rb +21 -19
- data/lib/vanity/metric/base.rb +22 -23
- data/lib/vanity/metric/google_analytics.rb +6 -9
- data/lib/vanity/metric/remote.rb +3 -5
- data/lib/vanity/playground.rb +3 -6
- data/lib/vanity/vanity.rb +8 -12
- data/lib/vanity/version.rb +1 -1
- data/test/adapters/active_record_adapter_test.rb +1 -5
- data/test/adapters/mock_adapter_test.rb +0 -2
- data/test/adapters/mongodb_adapter_test.rb +1 -5
- data/test/adapters/redis_adapter_test.rb +2 -3
- data/test/adapters/shared_tests.rb +9 -12
- data/test/autoconnect_test.rb +3 -3
- data/test/cli_test.rb +0 -1
- data/test/configuration_test.rb +18 -34
- data/test/connection_test.rb +3 -3
- data/test/dummy/Rakefile +1 -1
- data/test/dummy/app/controllers/use_vanity_controller.rb +12 -8
- data/test/dummy/app/mailers/vanity_mailer.rb +3 -3
- data/test/dummy/config/application.rb +1 -1
- data/test/dummy/config/boot.rb +3 -3
- data/test/dummy/config/environment.rb +1 -1
- data/test/dummy/config/environments/development.rb +0 -1
- data/test/dummy/config/environments/test.rb +1 -1
- data/test/dummy/config/initializers/session_store.rb +1 -1
- data/test/dummy/config.ru +1 -1
- data/test/dummy/script/rails +2 -2
- data/test/experiment/ab_test.rb +148 -154
- data/test/experiment/base_test.rb +48 -32
- data/test/frameworks/rails/action_controller_test.rb +25 -25
- data/test/frameworks/rails/action_mailer_test.rb +2 -2
- data/test/frameworks/rails/action_view_test.rb +5 -6
- data/test/frameworks/rails/rails_test.rb +147 -181
- data/test/helper_test.rb +2 -2
- data/test/metric/active_record_test.rb +174 -212
- data/test/metric/base_test.rb +21 -46
- data/test/metric/google_analytics_test.rb +17 -25
- data/test/metric/remote_test.rb +7 -10
- data/test/playground_test.rb +7 -14
- data/test/templates_test.rb +16 -20
- data/test/test_helper.rb +28 -29
- data/test/vanity_test.rb +4 -10
- data/test/web/rails/dashboard_test.rb +21 -21
- data/vanity.gemspec +8 -7
- metadata +28 -30
- data/gemfiles/rails42.gemfile +0 -33
- data/gemfiles/rails42.gemfile.lock +0 -265
- data/gemfiles/rails42_protected_attributes.gemfile +0 -34
- data/gemfiles/rails42_protected_attributes.gemfile.lock +0 -264
- data/gemfiles/rails51.gemfile +0 -33
- data/gemfiles/rails51.gemfile.lock +0 -285
data/test/experiment/ab_test.rb
CHANGED
@@ -6,24 +6,23 @@ class AbTestController < ActionController::Base
|
|
6
6
|
|
7
7
|
def test_render
|
8
8
|
text = Vanity.ab_test(:simple)
|
9
|
-
render :
|
9
|
+
render plain: text, text: text
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_view
|
13
|
-
render :
|
13
|
+
render inline: "<%= ab_test(:simple) %>"
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_capture
|
17
|
-
render :
|
17
|
+
render inline: "<%= ab_test(:simple) do |value| %><%= value %><% end %>"
|
18
18
|
end
|
19
19
|
|
20
20
|
def track
|
21
21
|
Vanity.track!(:coolness)
|
22
|
-
render :
|
22
|
+
render plain: "", text: ""
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
26
|
class AbTestTest < ActionController::TestCase
|
28
27
|
tests AbTestController
|
29
28
|
|
@@ -85,7 +84,7 @@ class AbTestTest < ActionController::TestCase
|
|
85
84
|
alternatives :a, :b
|
86
85
|
default :a
|
87
86
|
end
|
88
|
-
fingerprints = Vanity.playground.experiments.map { |
|
87
|
+
fingerprints = Vanity.playground.experiments.map { |_id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
89
88
|
assert_equal 4, fingerprints.uniq.size
|
90
89
|
end
|
91
90
|
|
@@ -97,7 +96,7 @@ class AbTestTest < ActionController::TestCase
|
|
97
96
|
end
|
98
97
|
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
99
98
|
fingerprints.each do |fingerprint|
|
100
|
-
assert_match
|
99
|
+
assert_match(/^[0-9a-f]{10}$/i, fingerprint)
|
101
100
|
end
|
102
101
|
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
103
102
|
end
|
@@ -183,25 +182,24 @@ class AbTestTest < ActionController::TestCase
|
|
183
182
|
assert_equal exp.default, exp.alternative(nil)
|
184
183
|
end
|
185
184
|
|
186
|
-
|
187
185
|
# -- Experiment Enabled/disabled --
|
188
186
|
|
189
187
|
# @example new test should be enabled regardless of collecting?
|
190
188
|
# regardless_of "Vanity.playground.collecting" do
|
191
189
|
# assert (new_ab_test :test).enabled?
|
192
190
|
# end
|
193
|
-
def regardless_of(attr_name
|
194
|
-
prev_val = eval "#{attr_name}?"
|
191
|
+
def regardless_of(attr_name)
|
192
|
+
prev_val = eval "#{attr_name}?" # rubocop:todo Lint/UselessAssignment, Style/EvalWithLocation, Security/Eval
|
195
193
|
|
196
|
-
eval "#{attr_name}=true"
|
197
|
-
|
194
|
+
eval "#{attr_name}=true" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
195
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
198
196
|
nuke_playground
|
199
197
|
|
200
|
-
eval "#{attr_name}=false"
|
201
|
-
|
198
|
+
eval "#{attr_name}=false" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
199
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
202
200
|
nuke_playground
|
203
201
|
|
204
|
-
eval "#{attr_name}=prev_val"
|
202
|
+
eval "#{attr_name}=prev_val" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
205
203
|
end
|
206
204
|
|
207
205
|
def test_new_test_is_disabled_when_experiments_start_enabled_is_false
|
@@ -228,7 +226,7 @@ class AbTestTest < ActionController::TestCase
|
|
228
226
|
metrics :coolness
|
229
227
|
default false
|
230
228
|
end
|
231
|
-
exp.complete! #active? => false
|
229
|
+
exp.complete! # active? => false
|
232
230
|
|
233
231
|
assert !exp.enabled?, "experiment should not be enabled but it is!"
|
234
232
|
end
|
@@ -263,7 +261,7 @@ class AbTestTest < ActionController::TestCase
|
|
263
261
|
metrics :coolness
|
264
262
|
default false
|
265
263
|
end
|
266
|
-
exp.complete! #active? => false
|
264
|
+
exp.complete! # active? => false
|
267
265
|
assert !exp.enabled?
|
268
266
|
exp.enabled = true
|
269
267
|
assert !exp.enabled?
|
@@ -283,65 +281,65 @@ class AbTestTest < ActionController::TestCase
|
|
283
281
|
def test_enabled_persists_across_definitions
|
284
282
|
Vanity.configuration.experiments_start_enabled = false
|
285
283
|
Vanity.playground.collecting = true
|
286
|
-
new_ab_test :test, :
|
284
|
+
new_ab_test :test, enable: false do
|
287
285
|
metrics :coolness
|
288
286
|
default false
|
289
287
|
end
|
290
|
-
assert !experiment(:test).enabled? #starts off false
|
288
|
+
assert !experiment(:test).enabled? # starts off false
|
291
289
|
|
292
290
|
new_playground
|
293
291
|
metric "Coolness"
|
294
292
|
|
295
|
-
new_ab_test :test, :
|
293
|
+
new_ab_test :test, enable: false do
|
296
294
|
metrics :coolness
|
297
295
|
default false
|
298
296
|
end
|
299
|
-
assert !experiment(:test).enabled? #still false
|
297
|
+
assert !experiment(:test).enabled? # still false
|
300
298
|
experiment(:test).enabled = true
|
301
|
-
assert experiment(:test).enabled? #now true
|
299
|
+
assert experiment(:test).enabled? # now true
|
302
300
|
|
303
301
|
new_playground
|
304
302
|
metric "Coolness"
|
305
303
|
|
306
|
-
new_ab_test :test, :
|
304
|
+
new_ab_test :test, enable: false do
|
307
305
|
metrics :coolness
|
308
306
|
default false
|
309
307
|
end
|
310
|
-
assert experiment(:test).enabled? #still true
|
308
|
+
assert experiment(:test).enabled? # still true
|
311
309
|
experiment(:test).enabled = false
|
312
|
-
assert !experiment(:test).enabled? #now false again
|
310
|
+
assert !experiment(:test).enabled? # now false again
|
313
311
|
end
|
314
312
|
|
315
313
|
def test_enabled_persists_across_definitions_when_starting_enabled
|
316
314
|
Vanity.configuration.experiments_start_enabled = true
|
317
315
|
Vanity.playground.collecting = true
|
318
|
-
new_ab_test :test, :
|
316
|
+
new_ab_test :test, enable: false do
|
319
317
|
metrics :coolness
|
320
318
|
default false
|
321
319
|
end
|
322
|
-
assert experiment(:test).enabled? #starts off true
|
320
|
+
assert experiment(:test).enabled? # starts off true
|
323
321
|
|
324
322
|
new_playground
|
325
323
|
metric "Coolness"
|
326
324
|
|
327
|
-
new_ab_test :test, :
|
325
|
+
new_ab_test :test, enable: false do
|
328
326
|
metrics :coolness
|
329
327
|
default false
|
330
328
|
end
|
331
|
-
assert experiment(:test).enabled? #still true
|
329
|
+
assert experiment(:test).enabled? # still true
|
332
330
|
experiment(:test).enabled = false
|
333
|
-
assert !experiment(:test).enabled? #now false
|
331
|
+
assert !experiment(:test).enabled? # now false
|
334
332
|
|
335
333
|
new_playground
|
336
334
|
metric "Coolness"
|
337
335
|
|
338
|
-
new_ab_test :test, :
|
336
|
+
new_ab_test :test, enable: false do
|
339
337
|
metrics :coolness
|
340
338
|
default false
|
341
339
|
end
|
342
|
-
assert !experiment(:test).enabled? #still false
|
340
|
+
assert !experiment(:test).enabled? # still false
|
343
341
|
experiment(:test).enabled = true
|
344
|
-
assert experiment(:test).enabled? #now true again
|
342
|
+
assert experiment(:test).enabled? # now true again
|
345
343
|
end
|
346
344
|
|
347
345
|
def test_choose_random_when_enabled
|
@@ -378,7 +376,7 @@ class AbTestTest < ActionController::TestCase
|
|
378
376
|
def test_choose_outcome_when_finished
|
379
377
|
exp = new_ab_test :test do
|
380
378
|
metrics :coolness
|
381
|
-
alternatives 0,1,2,3,4,5
|
379
|
+
alternatives 0, 1, 2, 3, 4, 5
|
382
380
|
default 3
|
383
381
|
outcome_is { alternative(5) }
|
384
382
|
end
|
@@ -429,7 +427,7 @@ class AbTestTest < ActionController::TestCase
|
|
429
427
|
assert_equal 0, experiment(:foobar).alternatives.sum(&:converted)
|
430
428
|
experiment(:foobar).track!(:coolness, Time.now, 1)
|
431
429
|
assert_equal 1, experiment(:foobar).alternatives.sum(&:converted)
|
432
|
-
experiment(:foobar).track!(:coolness, Time.now, 1, :
|
430
|
+
experiment(:foobar).track!(:coolness, Time.now, 1, identity: "quux")
|
433
431
|
assert_equal 2, experiment(:foobar).alternatives.sum(&:converted)
|
434
432
|
end
|
435
433
|
|
@@ -458,7 +456,7 @@ class AbTestTest < ActionController::TestCase
|
|
458
456
|
alternatives "foo", "bar"
|
459
457
|
identify { "6e98ec" }
|
460
458
|
metrics :coolness
|
461
|
-
on_assignment { on_assignment_called_times
|
459
|
+
on_assignment { on_assignment_called_times += 1 }
|
462
460
|
end
|
463
461
|
2.times { experiment(:foobar).choose }
|
464
462
|
assert_equal 1, on_assignment_called_times
|
@@ -471,7 +469,7 @@ class AbTestTest < ActionController::TestCase
|
|
471
469
|
alternatives "foo", "bar"
|
472
470
|
identify { "6e98ec" }
|
473
471
|
metrics :coolness
|
474
|
-
on_assignment { on_assignment_called_times
|
472
|
+
on_assignment { on_assignment_called_times += 1 }
|
475
473
|
end
|
476
474
|
experiment(:foobar).choose(dummy_request)
|
477
475
|
assert_equal 1, on_assignment_called_times
|
@@ -484,7 +482,7 @@ class AbTestTest < ActionController::TestCase
|
|
484
482
|
alternatives "foo", "bar"
|
485
483
|
identify { "6e98ec" }
|
486
484
|
metrics :coolness
|
487
|
-
on_assignment { on_assignment_called_times
|
485
|
+
on_assignment { on_assignment_called_times += 1 }
|
488
486
|
end
|
489
487
|
request = dummy_request
|
490
488
|
request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
|
@@ -499,7 +497,7 @@ class AbTestTest < ActionController::TestCase
|
|
499
497
|
default "foo"
|
500
498
|
identify { "6e98ec" }
|
501
499
|
metrics :coolness
|
502
|
-
on_assignment { on_assignment_called_times
|
500
|
+
on_assignment { on_assignment_called_times += 1 }
|
503
501
|
end
|
504
502
|
2.times { experiment(:foobar).chooses("foo") }
|
505
503
|
assert_equal 1, on_assignment_called_times
|
@@ -510,11 +508,11 @@ class AbTestTest < ActionController::TestCase
|
|
510
508
|
alternatives "foo", "bar"
|
511
509
|
default "foo"
|
512
510
|
identify { "6e98ec" }
|
513
|
-
on_assignment {}
|
511
|
+
on_assignment {} # rubocop:todo Lint/EmptyBlock
|
514
512
|
metrics :coolness
|
515
513
|
end
|
516
514
|
assert value = experiment(:foobar).choose.value
|
517
|
-
assert_match
|
515
|
+
assert_match(/foo|bar/, value)
|
518
516
|
1000.times do
|
519
517
|
assert_equal value, experiment(:foobar).choose.value
|
520
518
|
end
|
@@ -575,7 +573,7 @@ class AbTestTest < ActionController::TestCase
|
|
575
573
|
end
|
576
574
|
|
577
575
|
def test_ab_assigned_object
|
578
|
-
identity = { :
|
576
|
+
identity = { a: :b }
|
579
577
|
new_ab_test :foobar do
|
580
578
|
alternatives "foo", "bar"
|
581
579
|
default "foo"
|
@@ -599,7 +597,7 @@ class AbTestTest < ActionController::TestCase
|
|
599
597
|
end
|
600
598
|
value = experiment(:foobar).choose.value
|
601
599
|
assert value
|
602
|
-
assert_match
|
600
|
+
assert_match(/foo|bar/, value)
|
603
601
|
100.times do
|
604
602
|
assert_equal value, experiment(:foobar).choose.value
|
605
603
|
end
|
@@ -611,7 +609,10 @@ class AbTestTest < ActionController::TestCase
|
|
611
609
|
identify { rand }
|
612
610
|
metrics :coolness
|
613
611
|
end
|
614
|
-
alts = Array.new(10_000) { experiment(:foobar).choose.value }.
|
612
|
+
alts = Array.new(10_000) { experiment(:foobar).choose.value }.each_with_object({}) do |k, h|
|
613
|
+
h[k] ||= 0
|
614
|
+
h[k] += 1
|
615
|
+
end
|
615
616
|
assert_equal %w{bar foo}, alts.keys.sort
|
616
617
|
assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
|
617
618
|
end
|
@@ -625,8 +626,8 @@ class AbTestTest < ActionController::TestCase
|
|
625
626
|
metrics :coolness
|
626
627
|
end
|
627
628
|
altered_alts = experiment(:foobar).alternatives
|
628
|
-
altered_alts[0].probability=30
|
629
|
-
altered_alts[1].probability=70
|
629
|
+
altered_alts[0].probability = 30
|
630
|
+
altered_alts[1].probability = 70
|
630
631
|
experiment(:foobar).set_alternative_probabilities altered_alts
|
631
632
|
alts = Array.new(600) { experiment(:foobar).choose.value }
|
632
633
|
assert_equal %w{bar foo}, alts.uniq.sort
|
@@ -643,10 +644,11 @@ class AbTestTest < ActionController::TestCase
|
|
643
644
|
rebalance_frequency 12
|
644
645
|
metrics :coolness
|
645
646
|
end
|
646
|
-
class <<experiment(:foobar)
|
647
|
+
class << experiment(:foobar)
|
647
648
|
def times_called
|
648
649
|
@times_called ||= 0
|
649
650
|
end
|
651
|
+
|
650
652
|
def rebalance!
|
651
653
|
@times_called = times_called + 1
|
652
654
|
end
|
@@ -669,20 +671,16 @@ class AbTestTest < ActionController::TestCase
|
|
669
671
|
end
|
670
672
|
corresponding_probabilities = [[experiment(:foobar).alternatives[0], 0.3], [experiment(:foobar).alternatives[1], 0.6], [experiment(:foobar).alternatives[2], 1.0]]
|
671
673
|
|
672
|
-
class <<experiment(:foobar)
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
def bayes_bandit_score(probability=90)
|
674
|
+
class << experiment(:foobar)
|
675
|
+
attr_reader :was_called, :use_probabilities
|
676
|
+
|
677
|
+
def bayes_bandit_score(_probability = 90)
|
677
678
|
@was_called = true
|
678
679
|
altered_alts = Vanity.playground.experiment(:foobar).alternatives
|
679
|
-
altered_alts[0].probability=30
|
680
|
-
altered_alts[1].probability=30
|
681
|
-
altered_alts[2].probability=40
|
682
|
-
Struct.new(:alts
|
683
|
-
end
|
684
|
-
def use_probabilities
|
685
|
-
@use_probabilities
|
680
|
+
altered_alts[0].probability = 30
|
681
|
+
altered_alts[1].probability = 30
|
682
|
+
altered_alts[2].probability = 40
|
683
|
+
Struct.new(:alts, :method).new(altered_alts, :bayes_bandit_score) # rubocop:todo Lint/StructNewOverride
|
686
684
|
end
|
687
685
|
end
|
688
686
|
experiment(:foobar).rebalance!
|
@@ -700,7 +698,7 @@ class AbTestTest < ActionController::TestCase
|
|
700
698
|
metrics :coolness
|
701
699
|
end
|
702
700
|
assert value = experiment(:foobar).choose.value
|
703
|
-
assert_match
|
701
|
+
assert_match(/foo|bar/, value)
|
704
702
|
1000.times do
|
705
703
|
assert_equal value, experiment(:foobar).choose.value
|
706
704
|
end
|
@@ -752,7 +750,7 @@ class AbTestTest < ActionController::TestCase
|
|
752
750
|
end
|
753
751
|
|
754
752
|
def test_records_each_converted_participant_only_once
|
755
|
-
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
753
|
+
ids = ((1..100).map { |i| [i, i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
756
754
|
new_ab_test :foobar do
|
757
755
|
alternatives "foo", "bar"
|
758
756
|
default "foo"
|
@@ -768,7 +766,7 @@ class AbTestTest < ActionController::TestCase
|
|
768
766
|
end
|
769
767
|
|
770
768
|
def test_records_conversion_only_for_participants
|
771
|
-
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
769
|
+
ids = ((1..100).map { |i| [-i, i, i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
772
770
|
new_ab_test :foobar do
|
773
771
|
alternatives "foo", "bar"
|
774
772
|
default "foo"
|
@@ -830,7 +828,6 @@ class AbTestTest < ActionController::TestCase
|
|
830
828
|
assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
|
831
829
|
end
|
832
830
|
|
833
|
-
|
834
831
|
# -- A/B helper methods --
|
835
832
|
|
836
833
|
def test_fail_if_no_experiment
|
@@ -846,7 +843,8 @@ class AbTestTest < ActionController::TestCase
|
|
846
843
|
default false
|
847
844
|
end
|
848
845
|
responses = Array.new(100) do
|
849
|
-
@controller = nil
|
846
|
+
@controller = nil
|
847
|
+
setup_controller_request_and_response
|
850
848
|
get :test_render
|
851
849
|
@response.body
|
852
850
|
end
|
@@ -860,7 +858,8 @@ class AbTestTest < ActionController::TestCase
|
|
860
858
|
default false
|
861
859
|
end
|
862
860
|
responses = Array.new(100) do
|
863
|
-
@controller = nil
|
861
|
+
@controller = nil
|
862
|
+
setup_controller_request_and_response
|
864
863
|
get :test_view
|
865
864
|
@response.body
|
866
865
|
end
|
@@ -874,7 +873,8 @@ class AbTestTest < ActionController::TestCase
|
|
874
873
|
default false
|
875
874
|
end
|
876
875
|
responses = Array.new(100) do
|
877
|
-
@controller = nil
|
876
|
+
@controller = nil
|
877
|
+
setup_controller_request_and_response
|
878
878
|
get :test_capture
|
879
879
|
@response.body
|
880
880
|
end
|
@@ -886,14 +886,13 @@ class AbTestTest < ActionController::TestCase
|
|
886
886
|
metrics :coolness
|
887
887
|
default false
|
888
888
|
end
|
889
|
-
responses = Array.new(100) do
|
890
|
-
@controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
|
889
|
+
responses = Array.new(100) do # rubocop:todo Lint/UselessAssignment
|
890
|
+
@controller.send(:cookies).each { |cookie| @controller.send(:cookies).delete(cookie.first) }
|
891
891
|
get :track
|
892
892
|
@response.body
|
893
893
|
end
|
894
894
|
end
|
895
895
|
|
896
|
-
|
897
896
|
# -- Testing with tests --
|
898
897
|
|
899
898
|
def test_with_given_choice
|
@@ -902,8 +901,9 @@ class AbTestTest < ActionController::TestCase
|
|
902
901
|
default :a
|
903
902
|
metrics :coolness
|
904
903
|
end
|
905
|
-
100.times do |
|
906
|
-
@controller = nil
|
904
|
+
100.times do |_i|
|
905
|
+
@controller = nil
|
906
|
+
setup_controller_request_and_response
|
907
907
|
experiment(:simple).chooses(:b)
|
908
908
|
get :test_render
|
909
909
|
assert "b", @response.body
|
@@ -927,8 +927,9 @@ class AbTestTest < ActionController::TestCase
|
|
927
927
|
default :a
|
928
928
|
metrics :coolness
|
929
929
|
end
|
930
|
-
responses = Array.new(100) do |
|
931
|
-
@controller = nil
|
930
|
+
responses = Array.new(100) do |_i|
|
931
|
+
@controller = nil
|
932
|
+
setup_controller_request_and_response
|
932
933
|
experiment(:simple).chooses(:b)
|
933
934
|
experiment(:simple).chooses(nil)
|
934
935
|
get :test_render
|
@@ -937,7 +938,6 @@ class AbTestTest < ActionController::TestCase
|
|
937
938
|
assert responses.uniq.size == 3
|
938
939
|
end
|
939
940
|
|
940
|
-
|
941
941
|
# -- Scoring --
|
942
942
|
|
943
943
|
def test_calculate_score
|
@@ -970,9 +970,9 @@ class AbTestTest < ActionController::TestCase
|
|
970
970
|
# Treatment A: 180 45 25.00% 1.33
|
971
971
|
# treatment B: 189 28 14.81% -1.13
|
972
972
|
# treatment C: 188 61 32.45% 2.94
|
973
|
-
fake :abcd, :
|
973
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
974
974
|
|
975
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
975
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score } # rubocop:todo Style/FormatString
|
976
976
|
assert_equal %w{-1.33 0.00 -2.46 1.58}, z_scores
|
977
977
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
978
978
|
assert_equal [90, 0, 99, 90], probabilities
|
@@ -997,11 +997,11 @@ class AbTestTest < ActionController::TestCase
|
|
997
997
|
# Treatment A: 180 45 25.00% 1.33
|
998
998
|
# treatment B: 189 28 14.81% -1.13
|
999
999
|
# treatment C: 188 61 32.45% 2.94
|
1000
|
-
fake :abcd, :
|
1000
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1001
1001
|
|
1002
1002
|
score_result = experiment(:abcd).bayes_bandit_score
|
1003
|
-
probabilities = score_result.alts.map{|a| a.probability.round}
|
1004
|
-
assert_equal [0,0,6,94], probabilities
|
1003
|
+
probabilities = score_result.alts.map { |a| a.probability.round }
|
1004
|
+
assert_equal [0, 0, 6, 94], probabilities
|
1005
1005
|
end
|
1006
1006
|
|
1007
1007
|
def test_scoring_with_no_performers
|
@@ -1024,13 +1024,13 @@ class AbTestTest < ActionController::TestCase
|
|
1024
1024
|
default :a
|
1025
1025
|
metrics :coolness
|
1026
1026
|
end
|
1027
|
-
fake :abcd, :
|
1027
|
+
fake :abcd, b: [10, 8]
|
1028
1028
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
1029
1029
|
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
1030
1030
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
1031
1031
|
assert_equal 1, experiment(:abcd).score.best.id
|
1032
1032
|
assert_nil experiment(:abcd).score.choice
|
1033
|
-
assert_includes [0,2,3], experiment(:abcd).score.base.id
|
1033
|
+
assert_includes [0, 2, 3], experiment(:abcd).score.base.id
|
1034
1034
|
assert_equal 1, experiment(:abcd).score.least.id
|
1035
1035
|
end
|
1036
1036
|
|
@@ -1040,9 +1040,9 @@ class AbTestTest < ActionController::TestCase
|
|
1040
1040
|
default :a
|
1041
1041
|
metrics :coolness
|
1042
1042
|
end
|
1043
|
-
fake :abcd, :
|
1043
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1044
1044
|
|
1045
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase)
|
1045
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase) # rubocop:todo Style/FormatString
|
1046
1046
|
assert_equal %w{nan 2.01 nan 0.00}, z_scores
|
1047
1047
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
1048
1048
|
assert_equal [0, 95, 0, 0], probabilities
|
@@ -1060,14 +1060,13 @@ class AbTestTest < ActionController::TestCase
|
|
1060
1060
|
default :a
|
1061
1061
|
metrics :coolness
|
1062
1062
|
end
|
1063
|
-
fake :abcd, :
|
1063
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1064
1064
|
|
1065
1065
|
assert_equal 1, experiment(:abcd).score(90).choice.id
|
1066
1066
|
assert_equal 1, experiment(:abcd).score(95).choice.id
|
1067
1067
|
assert_nil experiment(:abcd).score(99).choice
|
1068
1068
|
end
|
1069
1069
|
|
1070
|
-
|
1071
1070
|
# -- Conclusion --
|
1072
1071
|
|
1073
1072
|
def test_conclusion
|
@@ -1081,16 +1080,16 @@ class AbTestTest < ActionController::TestCase
|
|
1081
1080
|
# Treatment A: 180 45 25.00% 1.33
|
1082
1081
|
# treatment B: 189 28 14.81% -1.13
|
1083
1082
|
# treatment C: 188 61 32.45% 2.94
|
1084
|
-
fake :abcd, :
|
1085
|
-
|
1086
|
-
assert_equal
|
1087
|
-
There are 739 participants in this experiment.
|
1088
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1089
|
-
With 90% probability this result is statistically significant.
|
1090
|
-
Option B converted at 25.0%.
|
1091
|
-
Option A converted at 19.2%.
|
1092
|
-
Option C converted at 14.8%.
|
1093
|
-
Option D selected as the best alternative.
|
1083
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1084
|
+
|
1085
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1086
|
+
There are 739 participants in this experiment.
|
1087
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1088
|
+
With 90% probability this result is statistically significant.
|
1089
|
+
Option B converted at 25.0%.
|
1090
|
+
Option A converted at 19.2%.
|
1091
|
+
Option C converted at 14.8%.
|
1092
|
+
Option D selected as the best alternative.
|
1094
1093
|
TEXT
|
1095
1094
|
end
|
1096
1095
|
|
@@ -1100,16 +1099,16 @@ Option D selected as the best alternative.
|
|
1100
1099
|
default :a
|
1101
1100
|
metrics :coolness
|
1102
1101
|
end
|
1103
|
-
fake :abcd, :
|
1102
|
+
fake :abcd, b: [180, 45], d: [188, 61]
|
1104
1103
|
|
1105
|
-
assert_equal
|
1106
|
-
There are 368 participants in this experiment.
|
1107
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1108
|
-
With 90% probability this result is statistically significant.
|
1109
|
-
Option B converted at 25.0%.
|
1110
|
-
Option A did not convert.
|
1111
|
-
Option C did not convert.
|
1112
|
-
Option D selected as the best alternative.
|
1104
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1105
|
+
There are 368 participants in this experiment.
|
1106
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1107
|
+
With 90% probability this result is statistically significant.
|
1108
|
+
Option B converted at 25.0%.
|
1109
|
+
Option A did not convert.
|
1110
|
+
Option C did not convert.
|
1111
|
+
Option D selected as the best alternative.
|
1113
1112
|
TEXT
|
1114
1113
|
end
|
1115
1114
|
|
@@ -1119,15 +1118,15 @@ Option D selected as the best alternative.
|
|
1119
1118
|
default :a
|
1120
1119
|
metrics :coolness
|
1121
1120
|
end
|
1122
|
-
fake :abcd, :
|
1121
|
+
fake :abcd, b: [180, 58], d: [188, 61]
|
1123
1122
|
|
1124
|
-
assert_equal
|
1125
|
-
There are 368 participants in this experiment.
|
1126
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1127
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1128
|
-
Option B converted at 32.2%.
|
1129
|
-
Option A did not convert.
|
1130
|
-
Option C did not convert.
|
1123
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1124
|
+
There are 368 participants in this experiment.
|
1125
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1126
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1127
|
+
Option B converted at 32.2%.
|
1128
|
+
Option A did not convert.
|
1129
|
+
Option C did not convert.
|
1131
1130
|
TEXT
|
1132
1131
|
end
|
1133
1132
|
|
@@ -1137,15 +1136,15 @@ Option C did not convert.
|
|
1137
1136
|
default :a
|
1138
1137
|
metrics :coolness
|
1139
1138
|
end
|
1140
|
-
fake :abcd, :
|
1139
|
+
fake :abcd, b: [186, 60], d: [188, 61]
|
1141
1140
|
|
1142
|
-
assert_equal
|
1143
|
-
There are 374 participants in this experiment.
|
1144
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1145
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1146
|
-
Option B converted at 32.3%.
|
1147
|
-
Option A did not convert.
|
1148
|
-
Option C did not convert.
|
1141
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1142
|
+
There are 374 participants in this experiment.
|
1143
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1144
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1145
|
+
Option B converted at 32.3%.
|
1146
|
+
Option A did not convert.
|
1147
|
+
Option C did not convert.
|
1149
1148
|
TEXT
|
1150
1149
|
end
|
1151
1150
|
|
@@ -1155,14 +1154,14 @@ Option C did not convert.
|
|
1155
1154
|
default :a
|
1156
1155
|
metrics :coolness
|
1157
1156
|
end
|
1158
|
-
fake :abcd, :
|
1157
|
+
fake :abcd, b: [188, 61], d: [188, 61]
|
1159
1158
|
|
1160
|
-
assert_equal
|
1161
|
-
There are 376 participants in this experiment.
|
1162
|
-
Option D converted at 32.4%.
|
1163
|
-
Option B converted at 32.4%.
|
1164
|
-
Option A did not convert.
|
1165
|
-
Option C did not convert.
|
1159
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1160
|
+
There are 376 participants in this experiment.
|
1161
|
+
Option D converted at 32.4%.
|
1162
|
+
Option B converted at 32.4%.
|
1163
|
+
Option A did not convert.
|
1164
|
+
Option C did not convert.
|
1166
1165
|
TEXT
|
1167
1166
|
end
|
1168
1167
|
|
@@ -1172,11 +1171,11 @@ Option C did not convert.
|
|
1172
1171
|
default :a
|
1173
1172
|
metrics :coolness
|
1174
1173
|
end
|
1175
|
-
fake :abcd, :
|
1174
|
+
fake :abcd, b: [180, 45]
|
1176
1175
|
|
1177
|
-
assert_equal
|
1178
|
-
There are 180 participants in this experiment.
|
1179
|
-
This experiment did not run long enough to find a clear winner.
|
1176
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1177
|
+
There are 180 participants in this experiment.
|
1178
|
+
This experiment did not run long enough to find a clear winner.
|
1180
1179
|
TEXT
|
1181
1180
|
end
|
1182
1181
|
|
@@ -1186,13 +1185,12 @@ This experiment did not run long enough to find a clear winner.
|
|
1186
1185
|
default :a
|
1187
1186
|
metrics :coolness
|
1188
1187
|
end
|
1189
|
-
assert_equal
|
1190
|
-
There are no participants in this experiment yet.
|
1191
|
-
This experiment did not run long enough to find a clear winner.
|
1188
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1189
|
+
There are no participants in this experiment yet.
|
1190
|
+
This experiment did not run long enough to find a clear winner.
|
1192
1191
|
TEXT
|
1193
1192
|
end
|
1194
1193
|
|
1195
|
-
|
1196
1194
|
# -- Completion --
|
1197
1195
|
|
1198
1196
|
def test_completion_if
|
@@ -1209,7 +1207,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1209
1207
|
def test_completion_if_fails
|
1210
1208
|
new_ab_test :simple do
|
1211
1209
|
identify { rand }
|
1212
|
-
complete_if {
|
1210
|
+
complete_if { raise "Testing complete_if raises exception" }
|
1213
1211
|
metrics :coolness
|
1214
1212
|
default false
|
1215
1213
|
end
|
@@ -1225,7 +1223,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1225
1223
|
metrics :coolness
|
1226
1224
|
default false
|
1227
1225
|
end
|
1228
|
-
99.times do |
|
1226
|
+
99.times do |_i|
|
1229
1227
|
experiment(:simple).choose
|
1230
1228
|
assert experiment(:simple).active?
|
1231
1229
|
end
|
@@ -1262,7 +1260,6 @@ This experiment did not run long enough to find a clear winner.
|
|
1262
1260
|
assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
|
1263
1261
|
end
|
1264
1262
|
|
1265
|
-
|
1266
1263
|
# -- Outcome --
|
1267
1264
|
|
1268
1265
|
def test_completion_outcome
|
@@ -1319,7 +1316,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1319
1316
|
|
1320
1317
|
def test_outcome_is_fails
|
1321
1318
|
new_ab_test :quick do
|
1322
|
-
outcome_is {
|
1319
|
+
outcome_is { raise "Testing outcome_is raising exception" }
|
1323
1320
|
metrics :coolness
|
1324
1321
|
default false
|
1325
1322
|
end
|
@@ -1332,7 +1329,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1332
1329
|
metrics :coolness
|
1333
1330
|
default false
|
1334
1331
|
end
|
1335
|
-
fake :quick, false=>[2,0], true=>10
|
1332
|
+
fake :quick, false => [2, 0], true => 10
|
1336
1333
|
experiment(:quick).complete!
|
1337
1334
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1338
1335
|
end
|
@@ -1342,7 +1339,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1342
1339
|
metrics :coolness
|
1343
1340
|
default false
|
1344
1341
|
end
|
1345
|
-
fake :quick, true=>2
|
1342
|
+
fake :quick, true => 2
|
1346
1343
|
experiment(:quick).complete!
|
1347
1344
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1348
1345
|
end
|
@@ -1352,12 +1349,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1352
1349
|
metrics :coolness
|
1353
1350
|
default false
|
1354
1351
|
end
|
1355
|
-
fake :quick, false=>8, true=>8
|
1352
|
+
fake :quick, false => 8, true => 8
|
1356
1353
|
experiment(:quick).complete!
|
1357
1354
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1358
1355
|
end
|
1359
1356
|
|
1360
|
-
|
1361
1357
|
# -- No collection --
|
1362
1358
|
|
1363
1359
|
def test_no_collection_does_not_track
|
@@ -1400,7 +1396,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1400
1396
|
identify { "1" }
|
1401
1397
|
end
|
1402
1398
|
val = experiment(:simple).choose.value
|
1403
|
-
alternative = experiment(:simple).alternatives.detect {|a| a.value != val }
|
1399
|
+
alternative = experiment(:simple).alternatives.detect { |a| a.value != val }
|
1404
1400
|
experiment(:simple).chooses(alternative.value)
|
1405
1401
|
assert_equal experiment(:simple).choose.value, alternative.value
|
1406
1402
|
experiment(:simple).chooses(val)
|
@@ -1481,7 +1477,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1481
1477
|
default :b
|
1482
1478
|
metrics :coolness
|
1483
1479
|
|
1484
|
-
reject do |
|
1480
|
+
reject do |_request, _identity|
|
1485
1481
|
true
|
1486
1482
|
end
|
1487
1483
|
end
|
@@ -1508,11 +1504,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1508
1504
|
end
|
1509
1505
|
|
1510
1506
|
def test_clears_outcome_and_completed_at
|
1511
|
-
|
1512
|
-
|
1513
|
-
|
1514
|
-
|
1515
|
-
|
1507
|
+
new_ab_test :simple do
|
1508
|
+
alternatives :a, :b, :c
|
1509
|
+
default :a
|
1510
|
+
metrics :coolness
|
1511
|
+
end
|
1516
1512
|
experiment(:simple).reset
|
1517
1513
|
assert_nil experiment(:simple).outcome
|
1518
1514
|
assert_nil experiment(:simple).completed_at
|
@@ -1531,7 +1527,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1531
1527
|
assert_not_nil experiment(:simple).completed_at
|
1532
1528
|
end
|
1533
1529
|
|
1534
|
-
def test_reset_clears_participants
|
1530
|
+
def test_reset_clears_participants # rubocop:todo Lint/DuplicateMethods
|
1535
1531
|
new_ab_test :simple do
|
1536
1532
|
alternatives :a, :b, :c
|
1537
1533
|
default :a
|
@@ -1543,7 +1539,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1543
1539
|
assert_equal experiment(:simple).alternatives[1].participants, 0
|
1544
1540
|
end
|
1545
1541
|
|
1546
|
-
def test_clears_outcome_and_completed_at
|
1542
|
+
def test_clears_outcome_and_completed_at # rubocop:todo Lint/DuplicateMethods
|
1547
1543
|
new_ab_test :simple do
|
1548
1544
|
alternatives :a, :b, :c
|
1549
1545
|
default :a
|
@@ -1554,11 +1550,9 @@ This experiment did not run long enough to find a clear winner.
|
|
1554
1550
|
assert_nil experiment(:simple).completed_at
|
1555
1551
|
end
|
1556
1552
|
|
1557
|
-
|
1558
1553
|
# -- Helper methods --
|
1559
1554
|
|
1560
1555
|
def fake(name, args)
|
1561
1556
|
experiment(name).instance_eval { fake args }
|
1562
1557
|
end
|
1563
|
-
|
1564
1558
|
end
|