vanity 3.1.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linting.yml +28 -0
- data/.github/workflows/test.yml +3 -6
- data/.rubocop.yml +114 -0
- data/.rubocop_todo.yml +67 -0
- data/Appraisals +9 -31
- data/CHANGELOG +5 -0
- data/Gemfile +7 -3
- data/Gemfile.lock +31 -3
- data/README.md +4 -9
- data/Rakefile +25 -24
- data/bin/vanity +1 -1
- data/doc/configuring.textile +1 -0
- data/gemfiles/rails52.gemfile +6 -3
- data/gemfiles/rails52.gemfile.lock +34 -9
- data/gemfiles/rails60.gemfile +6 -3
- data/gemfiles/rails60.gemfile.lock +34 -9
- data/gemfiles/rails61.gemfile +6 -3
- data/gemfiles/rails61.gemfile.lock +34 -9
- data/lib/generators/vanity/migration_generator.rb +5 -7
- data/lib/vanity/adapters/abstract_adapter.rb +43 -45
- data/lib/vanity/adapters/active_record_adapter.rb +30 -30
- data/lib/vanity/adapters/mock_adapter.rb +14 -18
- data/lib/vanity/adapters/mongodb_adapter.rb +73 -69
- data/lib/vanity/adapters/redis_adapter.rb +19 -27
- data/lib/vanity/adapters.rb +1 -1
- data/lib/vanity/autoconnect.rb +6 -7
- data/lib/vanity/commands/list.rb +7 -7
- data/lib/vanity/commands/report.rb +18 -22
- data/lib/vanity/configuration.rb +19 -19
- data/lib/vanity/connection.rb +12 -14
- data/lib/vanity/experiment/ab_test.rb +82 -70
- data/lib/vanity/experiment/alternative.rb +3 -5
- data/lib/vanity/experiment/base.rb +24 -19
- data/lib/vanity/experiment/bayesian_bandit_score.rb +7 -13
- data/lib/vanity/experiment/definition.rb +6 -6
- data/lib/vanity/frameworks/rails.rb +39 -39
- data/lib/vanity/frameworks.rb +2 -2
- data/lib/vanity/helpers.rb +1 -1
- data/lib/vanity/metric/active_record.rb +21 -19
- data/lib/vanity/metric/base.rb +22 -23
- data/lib/vanity/metric/google_analytics.rb +6 -9
- data/lib/vanity/metric/remote.rb +3 -5
- data/lib/vanity/playground.rb +3 -6
- data/lib/vanity/vanity.rb +8 -12
- data/lib/vanity/version.rb +1 -1
- data/test/adapters/active_record_adapter_test.rb +1 -5
- data/test/adapters/mock_adapter_test.rb +0 -2
- data/test/adapters/mongodb_adapter_test.rb +1 -5
- data/test/adapters/redis_adapter_test.rb +2 -3
- data/test/adapters/shared_tests.rb +9 -12
- data/test/autoconnect_test.rb +3 -3
- data/test/cli_test.rb +0 -1
- data/test/configuration_test.rb +18 -34
- data/test/connection_test.rb +3 -3
- data/test/dummy/Rakefile +1 -1
- data/test/dummy/app/controllers/use_vanity_controller.rb +12 -8
- data/test/dummy/app/mailers/vanity_mailer.rb +3 -3
- data/test/dummy/config/application.rb +1 -1
- data/test/dummy/config/boot.rb +3 -3
- data/test/dummy/config/environment.rb +1 -1
- data/test/dummy/config/environments/development.rb +0 -1
- data/test/dummy/config/environments/test.rb +1 -1
- data/test/dummy/config/initializers/session_store.rb +1 -1
- data/test/dummy/config.ru +1 -1
- data/test/dummy/script/rails +2 -2
- data/test/experiment/ab_test.rb +148 -154
- data/test/experiment/base_test.rb +48 -32
- data/test/frameworks/rails/action_controller_test.rb +25 -25
- data/test/frameworks/rails/action_mailer_test.rb +2 -2
- data/test/frameworks/rails/action_view_test.rb +5 -6
- data/test/frameworks/rails/rails_test.rb +147 -181
- data/test/helper_test.rb +2 -2
- data/test/metric/active_record_test.rb +174 -212
- data/test/metric/base_test.rb +21 -46
- data/test/metric/google_analytics_test.rb +17 -25
- data/test/metric/remote_test.rb +7 -10
- data/test/playground_test.rb +7 -14
- data/test/templates_test.rb +16 -20
- data/test/test_helper.rb +28 -29
- data/test/vanity_test.rb +4 -10
- data/test/web/rails/dashboard_test.rb +21 -21
- data/vanity.gemspec +8 -7
- metadata +28 -30
- data/gemfiles/rails42.gemfile +0 -33
- data/gemfiles/rails42.gemfile.lock +0 -265
- data/gemfiles/rails42_protected_attributes.gemfile +0 -34
- data/gemfiles/rails42_protected_attributes.gemfile.lock +0 -264
- data/gemfiles/rails51.gemfile +0 -33
- data/gemfiles/rails51.gemfile.lock +0 -285
data/test/experiment/ab_test.rb
CHANGED
@@ -6,24 +6,23 @@ class AbTestController < ActionController::Base
|
|
6
6
|
|
7
7
|
def test_render
|
8
8
|
text = Vanity.ab_test(:simple)
|
9
|
-
render :
|
9
|
+
render plain: text, text: text
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_view
|
13
|
-
render :
|
13
|
+
render inline: "<%= ab_test(:simple) %>"
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_capture
|
17
|
-
render :
|
17
|
+
render inline: "<%= ab_test(:simple) do |value| %><%= value %><% end %>"
|
18
18
|
end
|
19
19
|
|
20
20
|
def track
|
21
21
|
Vanity.track!(:coolness)
|
22
|
-
render :
|
22
|
+
render plain: "", text: ""
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
26
|
class AbTestTest < ActionController::TestCase
|
28
27
|
tests AbTestController
|
29
28
|
|
@@ -85,7 +84,7 @@ class AbTestTest < ActionController::TestCase
|
|
85
84
|
alternatives :a, :b
|
86
85
|
default :a
|
87
86
|
end
|
88
|
-
fingerprints = Vanity.playground.experiments.map { |
|
87
|
+
fingerprints = Vanity.playground.experiments.map { |_id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
89
88
|
assert_equal 4, fingerprints.uniq.size
|
90
89
|
end
|
91
90
|
|
@@ -97,7 +96,7 @@ class AbTestTest < ActionController::TestCase
|
|
97
96
|
end
|
98
97
|
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
99
98
|
fingerprints.each do |fingerprint|
|
100
|
-
assert_match
|
99
|
+
assert_match(/^[0-9a-f]{10}$/i, fingerprint)
|
101
100
|
end
|
102
101
|
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
103
102
|
end
|
@@ -183,25 +182,24 @@ class AbTestTest < ActionController::TestCase
|
|
183
182
|
assert_equal exp.default, exp.alternative(nil)
|
184
183
|
end
|
185
184
|
|
186
|
-
|
187
185
|
# -- Experiment Enabled/disabled --
|
188
186
|
|
189
187
|
# @example new test should be enabled regardless of collecting?
|
190
188
|
# regardless_of "Vanity.playground.collecting" do
|
191
189
|
# assert (new_ab_test :test).enabled?
|
192
190
|
# end
|
193
|
-
def regardless_of(attr_name
|
194
|
-
prev_val = eval "#{attr_name}?"
|
191
|
+
def regardless_of(attr_name)
|
192
|
+
prev_val = eval "#{attr_name}?" # rubocop:todo Lint/UselessAssignment, Style/EvalWithLocation, Security/Eval
|
195
193
|
|
196
|
-
eval "#{attr_name}=true"
|
197
|
-
|
194
|
+
eval "#{attr_name}=true" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
195
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
198
196
|
nuke_playground
|
199
197
|
|
200
|
-
eval "#{attr_name}=false"
|
201
|
-
|
198
|
+
eval "#{attr_name}=false" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
199
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
202
200
|
nuke_playground
|
203
201
|
|
204
|
-
eval "#{attr_name}=prev_val"
|
202
|
+
eval "#{attr_name}=prev_val" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
205
203
|
end
|
206
204
|
|
207
205
|
def test_new_test_is_disabled_when_experiments_start_enabled_is_false
|
@@ -228,7 +226,7 @@ class AbTestTest < ActionController::TestCase
|
|
228
226
|
metrics :coolness
|
229
227
|
default false
|
230
228
|
end
|
231
|
-
exp.complete! #active? => false
|
229
|
+
exp.complete! # active? => false
|
232
230
|
|
233
231
|
assert !exp.enabled?, "experiment should not be enabled but it is!"
|
234
232
|
end
|
@@ -263,7 +261,7 @@ class AbTestTest < ActionController::TestCase
|
|
263
261
|
metrics :coolness
|
264
262
|
default false
|
265
263
|
end
|
266
|
-
exp.complete! #active? => false
|
264
|
+
exp.complete! # active? => false
|
267
265
|
assert !exp.enabled?
|
268
266
|
exp.enabled = true
|
269
267
|
assert !exp.enabled?
|
@@ -283,65 +281,65 @@ class AbTestTest < ActionController::TestCase
|
|
283
281
|
def test_enabled_persists_across_definitions
|
284
282
|
Vanity.configuration.experiments_start_enabled = false
|
285
283
|
Vanity.playground.collecting = true
|
286
|
-
new_ab_test :test, :
|
284
|
+
new_ab_test :test, enable: false do
|
287
285
|
metrics :coolness
|
288
286
|
default false
|
289
287
|
end
|
290
|
-
assert !experiment(:test).enabled? #starts off false
|
288
|
+
assert !experiment(:test).enabled? # starts off false
|
291
289
|
|
292
290
|
new_playground
|
293
291
|
metric "Coolness"
|
294
292
|
|
295
|
-
new_ab_test :test, :
|
293
|
+
new_ab_test :test, enable: false do
|
296
294
|
metrics :coolness
|
297
295
|
default false
|
298
296
|
end
|
299
|
-
assert !experiment(:test).enabled? #still false
|
297
|
+
assert !experiment(:test).enabled? # still false
|
300
298
|
experiment(:test).enabled = true
|
301
|
-
assert experiment(:test).enabled? #now true
|
299
|
+
assert experiment(:test).enabled? # now true
|
302
300
|
|
303
301
|
new_playground
|
304
302
|
metric "Coolness"
|
305
303
|
|
306
|
-
new_ab_test :test, :
|
304
|
+
new_ab_test :test, enable: false do
|
307
305
|
metrics :coolness
|
308
306
|
default false
|
309
307
|
end
|
310
|
-
assert experiment(:test).enabled? #still true
|
308
|
+
assert experiment(:test).enabled? # still true
|
311
309
|
experiment(:test).enabled = false
|
312
|
-
assert !experiment(:test).enabled? #now false again
|
310
|
+
assert !experiment(:test).enabled? # now false again
|
313
311
|
end
|
314
312
|
|
315
313
|
def test_enabled_persists_across_definitions_when_starting_enabled
|
316
314
|
Vanity.configuration.experiments_start_enabled = true
|
317
315
|
Vanity.playground.collecting = true
|
318
|
-
new_ab_test :test, :
|
316
|
+
new_ab_test :test, enable: false do
|
319
317
|
metrics :coolness
|
320
318
|
default false
|
321
319
|
end
|
322
|
-
assert experiment(:test).enabled? #starts off true
|
320
|
+
assert experiment(:test).enabled? # starts off true
|
323
321
|
|
324
322
|
new_playground
|
325
323
|
metric "Coolness"
|
326
324
|
|
327
|
-
new_ab_test :test, :
|
325
|
+
new_ab_test :test, enable: false do
|
328
326
|
metrics :coolness
|
329
327
|
default false
|
330
328
|
end
|
331
|
-
assert experiment(:test).enabled? #still true
|
329
|
+
assert experiment(:test).enabled? # still true
|
332
330
|
experiment(:test).enabled = false
|
333
|
-
assert !experiment(:test).enabled? #now false
|
331
|
+
assert !experiment(:test).enabled? # now false
|
334
332
|
|
335
333
|
new_playground
|
336
334
|
metric "Coolness"
|
337
335
|
|
338
|
-
new_ab_test :test, :
|
336
|
+
new_ab_test :test, enable: false do
|
339
337
|
metrics :coolness
|
340
338
|
default false
|
341
339
|
end
|
342
|
-
assert !experiment(:test).enabled? #still false
|
340
|
+
assert !experiment(:test).enabled? # still false
|
343
341
|
experiment(:test).enabled = true
|
344
|
-
assert experiment(:test).enabled? #now true again
|
342
|
+
assert experiment(:test).enabled? # now true again
|
345
343
|
end
|
346
344
|
|
347
345
|
def test_choose_random_when_enabled
|
@@ -378,7 +376,7 @@ class AbTestTest < ActionController::TestCase
|
|
378
376
|
def test_choose_outcome_when_finished
|
379
377
|
exp = new_ab_test :test do
|
380
378
|
metrics :coolness
|
381
|
-
alternatives 0,1,2,3,4,5
|
379
|
+
alternatives 0, 1, 2, 3, 4, 5
|
382
380
|
default 3
|
383
381
|
outcome_is { alternative(5) }
|
384
382
|
end
|
@@ -429,7 +427,7 @@ class AbTestTest < ActionController::TestCase
|
|
429
427
|
assert_equal 0, experiment(:foobar).alternatives.sum(&:converted)
|
430
428
|
experiment(:foobar).track!(:coolness, Time.now, 1)
|
431
429
|
assert_equal 1, experiment(:foobar).alternatives.sum(&:converted)
|
432
|
-
experiment(:foobar).track!(:coolness, Time.now, 1, :
|
430
|
+
experiment(:foobar).track!(:coolness, Time.now, 1, identity: "quux")
|
433
431
|
assert_equal 2, experiment(:foobar).alternatives.sum(&:converted)
|
434
432
|
end
|
435
433
|
|
@@ -458,7 +456,7 @@ class AbTestTest < ActionController::TestCase
|
|
458
456
|
alternatives "foo", "bar"
|
459
457
|
identify { "6e98ec" }
|
460
458
|
metrics :coolness
|
461
|
-
on_assignment { on_assignment_called_times
|
459
|
+
on_assignment { on_assignment_called_times += 1 }
|
462
460
|
end
|
463
461
|
2.times { experiment(:foobar).choose }
|
464
462
|
assert_equal 1, on_assignment_called_times
|
@@ -471,7 +469,7 @@ class AbTestTest < ActionController::TestCase
|
|
471
469
|
alternatives "foo", "bar"
|
472
470
|
identify { "6e98ec" }
|
473
471
|
metrics :coolness
|
474
|
-
on_assignment { on_assignment_called_times
|
472
|
+
on_assignment { on_assignment_called_times += 1 }
|
475
473
|
end
|
476
474
|
experiment(:foobar).choose(dummy_request)
|
477
475
|
assert_equal 1, on_assignment_called_times
|
@@ -484,7 +482,7 @@ class AbTestTest < ActionController::TestCase
|
|
484
482
|
alternatives "foo", "bar"
|
485
483
|
identify { "6e98ec" }
|
486
484
|
metrics :coolness
|
487
|
-
on_assignment { on_assignment_called_times
|
485
|
+
on_assignment { on_assignment_called_times += 1 }
|
488
486
|
end
|
489
487
|
request = dummy_request
|
490
488
|
request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
|
@@ -499,7 +497,7 @@ class AbTestTest < ActionController::TestCase
|
|
499
497
|
default "foo"
|
500
498
|
identify { "6e98ec" }
|
501
499
|
metrics :coolness
|
502
|
-
on_assignment { on_assignment_called_times
|
500
|
+
on_assignment { on_assignment_called_times += 1 }
|
503
501
|
end
|
504
502
|
2.times { experiment(:foobar).chooses("foo") }
|
505
503
|
assert_equal 1, on_assignment_called_times
|
@@ -510,11 +508,11 @@ class AbTestTest < ActionController::TestCase
|
|
510
508
|
alternatives "foo", "bar"
|
511
509
|
default "foo"
|
512
510
|
identify { "6e98ec" }
|
513
|
-
on_assignment {}
|
511
|
+
on_assignment {} # rubocop:todo Lint/EmptyBlock
|
514
512
|
metrics :coolness
|
515
513
|
end
|
516
514
|
assert value = experiment(:foobar).choose.value
|
517
|
-
assert_match
|
515
|
+
assert_match(/foo|bar/, value)
|
518
516
|
1000.times do
|
519
517
|
assert_equal value, experiment(:foobar).choose.value
|
520
518
|
end
|
@@ -575,7 +573,7 @@ class AbTestTest < ActionController::TestCase
|
|
575
573
|
end
|
576
574
|
|
577
575
|
def test_ab_assigned_object
|
578
|
-
identity = { :
|
576
|
+
identity = { a: :b }
|
579
577
|
new_ab_test :foobar do
|
580
578
|
alternatives "foo", "bar"
|
581
579
|
default "foo"
|
@@ -599,7 +597,7 @@ class AbTestTest < ActionController::TestCase
|
|
599
597
|
end
|
600
598
|
value = experiment(:foobar).choose.value
|
601
599
|
assert value
|
602
|
-
assert_match
|
600
|
+
assert_match(/foo|bar/, value)
|
603
601
|
100.times do
|
604
602
|
assert_equal value, experiment(:foobar).choose.value
|
605
603
|
end
|
@@ -611,7 +609,10 @@ class AbTestTest < ActionController::TestCase
|
|
611
609
|
identify { rand }
|
612
610
|
metrics :coolness
|
613
611
|
end
|
614
|
-
alts = Array.new(10_000) { experiment(:foobar).choose.value }.
|
612
|
+
alts = Array.new(10_000) { experiment(:foobar).choose.value }.each_with_object({}) do |k, h|
|
613
|
+
h[k] ||= 0
|
614
|
+
h[k] += 1
|
615
|
+
end
|
615
616
|
assert_equal %w{bar foo}, alts.keys.sort
|
616
617
|
assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
|
617
618
|
end
|
@@ -625,8 +626,8 @@ class AbTestTest < ActionController::TestCase
|
|
625
626
|
metrics :coolness
|
626
627
|
end
|
627
628
|
altered_alts = experiment(:foobar).alternatives
|
628
|
-
altered_alts[0].probability=30
|
629
|
-
altered_alts[1].probability=70
|
629
|
+
altered_alts[0].probability = 30
|
630
|
+
altered_alts[1].probability = 70
|
630
631
|
experiment(:foobar).set_alternative_probabilities altered_alts
|
631
632
|
alts = Array.new(600) { experiment(:foobar).choose.value }
|
632
633
|
assert_equal %w{bar foo}, alts.uniq.sort
|
@@ -643,10 +644,11 @@ class AbTestTest < ActionController::TestCase
|
|
643
644
|
rebalance_frequency 12
|
644
645
|
metrics :coolness
|
645
646
|
end
|
646
|
-
class <<experiment(:foobar)
|
647
|
+
class << experiment(:foobar)
|
647
648
|
def times_called
|
648
649
|
@times_called ||= 0
|
649
650
|
end
|
651
|
+
|
650
652
|
def rebalance!
|
651
653
|
@times_called = times_called + 1
|
652
654
|
end
|
@@ -669,20 +671,16 @@ class AbTestTest < ActionController::TestCase
|
|
669
671
|
end
|
670
672
|
corresponding_probabilities = [[experiment(:foobar).alternatives[0], 0.3], [experiment(:foobar).alternatives[1], 0.6], [experiment(:foobar).alternatives[2], 1.0]]
|
671
673
|
|
672
|
-
class <<experiment(:foobar)
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
def bayes_bandit_score(probability=90)
|
674
|
+
class << experiment(:foobar)
|
675
|
+
attr_reader :was_called, :use_probabilities
|
676
|
+
|
677
|
+
def bayes_bandit_score(_probability = 90)
|
677
678
|
@was_called = true
|
678
679
|
altered_alts = Vanity.playground.experiment(:foobar).alternatives
|
679
|
-
altered_alts[0].probability=30
|
680
|
-
altered_alts[1].probability=30
|
681
|
-
altered_alts[2].probability=40
|
682
|
-
Struct.new(:alts
|
683
|
-
end
|
684
|
-
def use_probabilities
|
685
|
-
@use_probabilities
|
680
|
+
altered_alts[0].probability = 30
|
681
|
+
altered_alts[1].probability = 30
|
682
|
+
altered_alts[2].probability = 40
|
683
|
+
Struct.new(:alts, :method).new(altered_alts, :bayes_bandit_score) # rubocop:todo Lint/StructNewOverride
|
686
684
|
end
|
687
685
|
end
|
688
686
|
experiment(:foobar).rebalance!
|
@@ -700,7 +698,7 @@ class AbTestTest < ActionController::TestCase
|
|
700
698
|
metrics :coolness
|
701
699
|
end
|
702
700
|
assert value = experiment(:foobar).choose.value
|
703
|
-
assert_match
|
701
|
+
assert_match(/foo|bar/, value)
|
704
702
|
1000.times do
|
705
703
|
assert_equal value, experiment(:foobar).choose.value
|
706
704
|
end
|
@@ -752,7 +750,7 @@ class AbTestTest < ActionController::TestCase
|
|
752
750
|
end
|
753
751
|
|
754
752
|
def test_records_each_converted_participant_only_once
|
755
|
-
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
753
|
+
ids = ((1..100).map { |i| [i, i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
756
754
|
new_ab_test :foobar do
|
757
755
|
alternatives "foo", "bar"
|
758
756
|
default "foo"
|
@@ -768,7 +766,7 @@ class AbTestTest < ActionController::TestCase
|
|
768
766
|
end
|
769
767
|
|
770
768
|
def test_records_conversion_only_for_participants
|
771
|
-
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
769
|
+
ids = ((1..100).map { |i| [-i, i, i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
772
770
|
new_ab_test :foobar do
|
773
771
|
alternatives "foo", "bar"
|
774
772
|
default "foo"
|
@@ -830,7 +828,6 @@ class AbTestTest < ActionController::TestCase
|
|
830
828
|
assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
|
831
829
|
end
|
832
830
|
|
833
|
-
|
834
831
|
# -- A/B helper methods --
|
835
832
|
|
836
833
|
def test_fail_if_no_experiment
|
@@ -846,7 +843,8 @@ class AbTestTest < ActionController::TestCase
|
|
846
843
|
default false
|
847
844
|
end
|
848
845
|
responses = Array.new(100) do
|
849
|
-
@controller = nil
|
846
|
+
@controller = nil
|
847
|
+
setup_controller_request_and_response
|
850
848
|
get :test_render
|
851
849
|
@response.body
|
852
850
|
end
|
@@ -860,7 +858,8 @@ class AbTestTest < ActionController::TestCase
|
|
860
858
|
default false
|
861
859
|
end
|
862
860
|
responses = Array.new(100) do
|
863
|
-
@controller = nil
|
861
|
+
@controller = nil
|
862
|
+
setup_controller_request_and_response
|
864
863
|
get :test_view
|
865
864
|
@response.body
|
866
865
|
end
|
@@ -874,7 +873,8 @@ class AbTestTest < ActionController::TestCase
|
|
874
873
|
default false
|
875
874
|
end
|
876
875
|
responses = Array.new(100) do
|
877
|
-
@controller = nil
|
876
|
+
@controller = nil
|
877
|
+
setup_controller_request_and_response
|
878
878
|
get :test_capture
|
879
879
|
@response.body
|
880
880
|
end
|
@@ -886,14 +886,13 @@ class AbTestTest < ActionController::TestCase
|
|
886
886
|
metrics :coolness
|
887
887
|
default false
|
888
888
|
end
|
889
|
-
responses = Array.new(100) do
|
890
|
-
@controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
|
889
|
+
responses = Array.new(100) do # rubocop:todo Lint/UselessAssignment
|
890
|
+
@controller.send(:cookies).each { |cookie| @controller.send(:cookies).delete(cookie.first) }
|
891
891
|
get :track
|
892
892
|
@response.body
|
893
893
|
end
|
894
894
|
end
|
895
895
|
|
896
|
-
|
897
896
|
# -- Testing with tests --
|
898
897
|
|
899
898
|
def test_with_given_choice
|
@@ -902,8 +901,9 @@ class AbTestTest < ActionController::TestCase
|
|
902
901
|
default :a
|
903
902
|
metrics :coolness
|
904
903
|
end
|
905
|
-
100.times do |
|
906
|
-
@controller = nil
|
904
|
+
100.times do |_i|
|
905
|
+
@controller = nil
|
906
|
+
setup_controller_request_and_response
|
907
907
|
experiment(:simple).chooses(:b)
|
908
908
|
get :test_render
|
909
909
|
assert "b", @response.body
|
@@ -927,8 +927,9 @@ class AbTestTest < ActionController::TestCase
|
|
927
927
|
default :a
|
928
928
|
metrics :coolness
|
929
929
|
end
|
930
|
-
responses = Array.new(100) do |
|
931
|
-
@controller = nil
|
930
|
+
responses = Array.new(100) do |_i|
|
931
|
+
@controller = nil
|
932
|
+
setup_controller_request_and_response
|
932
933
|
experiment(:simple).chooses(:b)
|
933
934
|
experiment(:simple).chooses(nil)
|
934
935
|
get :test_render
|
@@ -937,7 +938,6 @@ class AbTestTest < ActionController::TestCase
|
|
937
938
|
assert responses.uniq.size == 3
|
938
939
|
end
|
939
940
|
|
940
|
-
|
941
941
|
# -- Scoring --
|
942
942
|
|
943
943
|
def test_calculate_score
|
@@ -970,9 +970,9 @@ class AbTestTest < ActionController::TestCase
|
|
970
970
|
# Treatment A: 180 45 25.00% 1.33
|
971
971
|
# treatment B: 189 28 14.81% -1.13
|
972
972
|
# treatment C: 188 61 32.45% 2.94
|
973
|
-
fake :abcd, :
|
973
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
974
974
|
|
975
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
975
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score } # rubocop:todo Style/FormatString
|
976
976
|
assert_equal %w{-1.33 0.00 -2.46 1.58}, z_scores
|
977
977
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
978
978
|
assert_equal [90, 0, 99, 90], probabilities
|
@@ -997,11 +997,11 @@ class AbTestTest < ActionController::TestCase
|
|
997
997
|
# Treatment A: 180 45 25.00% 1.33
|
998
998
|
# treatment B: 189 28 14.81% -1.13
|
999
999
|
# treatment C: 188 61 32.45% 2.94
|
1000
|
-
fake :abcd, :
|
1000
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1001
1001
|
|
1002
1002
|
score_result = experiment(:abcd).bayes_bandit_score
|
1003
|
-
probabilities = score_result.alts.map{|a| a.probability.round}
|
1004
|
-
assert_equal [0,0,6,94], probabilities
|
1003
|
+
probabilities = score_result.alts.map { |a| a.probability.round }
|
1004
|
+
assert_equal [0, 0, 6, 94], probabilities
|
1005
1005
|
end
|
1006
1006
|
|
1007
1007
|
def test_scoring_with_no_performers
|
@@ -1024,13 +1024,13 @@ class AbTestTest < ActionController::TestCase
|
|
1024
1024
|
default :a
|
1025
1025
|
metrics :coolness
|
1026
1026
|
end
|
1027
|
-
fake :abcd, :
|
1027
|
+
fake :abcd, b: [10, 8]
|
1028
1028
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
1029
1029
|
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
1030
1030
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
1031
1031
|
assert_equal 1, experiment(:abcd).score.best.id
|
1032
1032
|
assert_nil experiment(:abcd).score.choice
|
1033
|
-
assert_includes [0,2,3], experiment(:abcd).score.base.id
|
1033
|
+
assert_includes [0, 2, 3], experiment(:abcd).score.base.id
|
1034
1034
|
assert_equal 1, experiment(:abcd).score.least.id
|
1035
1035
|
end
|
1036
1036
|
|
@@ -1040,9 +1040,9 @@ class AbTestTest < ActionController::TestCase
|
|
1040
1040
|
default :a
|
1041
1041
|
metrics :coolness
|
1042
1042
|
end
|
1043
|
-
fake :abcd, :
|
1043
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1044
1044
|
|
1045
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase)
|
1045
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase) # rubocop:todo Style/FormatString
|
1046
1046
|
assert_equal %w{nan 2.01 nan 0.00}, z_scores
|
1047
1047
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
1048
1048
|
assert_equal [0, 95, 0, 0], probabilities
|
@@ -1060,14 +1060,13 @@ class AbTestTest < ActionController::TestCase
|
|
1060
1060
|
default :a
|
1061
1061
|
metrics :coolness
|
1062
1062
|
end
|
1063
|
-
fake :abcd, :
|
1063
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1064
1064
|
|
1065
1065
|
assert_equal 1, experiment(:abcd).score(90).choice.id
|
1066
1066
|
assert_equal 1, experiment(:abcd).score(95).choice.id
|
1067
1067
|
assert_nil experiment(:abcd).score(99).choice
|
1068
1068
|
end
|
1069
1069
|
|
1070
|
-
|
1071
1070
|
# -- Conclusion --
|
1072
1071
|
|
1073
1072
|
def test_conclusion
|
@@ -1081,16 +1080,16 @@ class AbTestTest < ActionController::TestCase
|
|
1081
1080
|
# Treatment A: 180 45 25.00% 1.33
|
1082
1081
|
# treatment B: 189 28 14.81% -1.13
|
1083
1082
|
# treatment C: 188 61 32.45% 2.94
|
1084
|
-
fake :abcd, :
|
1085
|
-
|
1086
|
-
assert_equal
|
1087
|
-
There are 739 participants in this experiment.
|
1088
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1089
|
-
With 90% probability this result is statistically significant.
|
1090
|
-
Option B converted at 25.0%.
|
1091
|
-
Option A converted at 19.2%.
|
1092
|
-
Option C converted at 14.8%.
|
1093
|
-
Option D selected as the best alternative.
|
1083
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1084
|
+
|
1085
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1086
|
+
There are 739 participants in this experiment.
|
1087
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1088
|
+
With 90% probability this result is statistically significant.
|
1089
|
+
Option B converted at 25.0%.
|
1090
|
+
Option A converted at 19.2%.
|
1091
|
+
Option C converted at 14.8%.
|
1092
|
+
Option D selected as the best alternative.
|
1094
1093
|
TEXT
|
1095
1094
|
end
|
1096
1095
|
|
@@ -1100,16 +1099,16 @@ Option D selected as the best alternative.
|
|
1100
1099
|
default :a
|
1101
1100
|
metrics :coolness
|
1102
1101
|
end
|
1103
|
-
fake :abcd, :
|
1102
|
+
fake :abcd, b: [180, 45], d: [188, 61]
|
1104
1103
|
|
1105
|
-
assert_equal
|
1106
|
-
There are 368 participants in this experiment.
|
1107
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1108
|
-
With 90% probability this result is statistically significant.
|
1109
|
-
Option B converted at 25.0%.
|
1110
|
-
Option A did not convert.
|
1111
|
-
Option C did not convert.
|
1112
|
-
Option D selected as the best alternative.
|
1104
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1105
|
+
There are 368 participants in this experiment.
|
1106
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1107
|
+
With 90% probability this result is statistically significant.
|
1108
|
+
Option B converted at 25.0%.
|
1109
|
+
Option A did not convert.
|
1110
|
+
Option C did not convert.
|
1111
|
+
Option D selected as the best alternative.
|
1113
1112
|
TEXT
|
1114
1113
|
end
|
1115
1114
|
|
@@ -1119,15 +1118,15 @@ Option D selected as the best alternative.
|
|
1119
1118
|
default :a
|
1120
1119
|
metrics :coolness
|
1121
1120
|
end
|
1122
|
-
fake :abcd, :
|
1121
|
+
fake :abcd, b: [180, 58], d: [188, 61]
|
1123
1122
|
|
1124
|
-
assert_equal
|
1125
|
-
There are 368 participants in this experiment.
|
1126
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1127
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1128
|
-
Option B converted at 32.2%.
|
1129
|
-
Option A did not convert.
|
1130
|
-
Option C did not convert.
|
1123
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1124
|
+
There are 368 participants in this experiment.
|
1125
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1126
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1127
|
+
Option B converted at 32.2%.
|
1128
|
+
Option A did not convert.
|
1129
|
+
Option C did not convert.
|
1131
1130
|
TEXT
|
1132
1131
|
end
|
1133
1132
|
|
@@ -1137,15 +1136,15 @@ Option C did not convert.
|
|
1137
1136
|
default :a
|
1138
1137
|
metrics :coolness
|
1139
1138
|
end
|
1140
|
-
fake :abcd, :
|
1139
|
+
fake :abcd, b: [186, 60], d: [188, 61]
|
1141
1140
|
|
1142
|
-
assert_equal
|
1143
|
-
There are 374 participants in this experiment.
|
1144
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1145
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1146
|
-
Option B converted at 32.3%.
|
1147
|
-
Option A did not convert.
|
1148
|
-
Option C did not convert.
|
1141
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1142
|
+
There are 374 participants in this experiment.
|
1143
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1144
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1145
|
+
Option B converted at 32.3%.
|
1146
|
+
Option A did not convert.
|
1147
|
+
Option C did not convert.
|
1149
1148
|
TEXT
|
1150
1149
|
end
|
1151
1150
|
|
@@ -1155,14 +1154,14 @@ Option C did not convert.
|
|
1155
1154
|
default :a
|
1156
1155
|
metrics :coolness
|
1157
1156
|
end
|
1158
|
-
fake :abcd, :
|
1157
|
+
fake :abcd, b: [188, 61], d: [188, 61]
|
1159
1158
|
|
1160
|
-
assert_equal
|
1161
|
-
There are 376 participants in this experiment.
|
1162
|
-
Option D converted at 32.4%.
|
1163
|
-
Option B converted at 32.4%.
|
1164
|
-
Option A did not convert.
|
1165
|
-
Option C did not convert.
|
1159
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1160
|
+
There are 376 participants in this experiment.
|
1161
|
+
Option D converted at 32.4%.
|
1162
|
+
Option B converted at 32.4%.
|
1163
|
+
Option A did not convert.
|
1164
|
+
Option C did not convert.
|
1166
1165
|
TEXT
|
1167
1166
|
end
|
1168
1167
|
|
@@ -1172,11 +1171,11 @@ Option C did not convert.
|
|
1172
1171
|
default :a
|
1173
1172
|
metrics :coolness
|
1174
1173
|
end
|
1175
|
-
fake :abcd, :
|
1174
|
+
fake :abcd, b: [180, 45]
|
1176
1175
|
|
1177
|
-
assert_equal
|
1178
|
-
There are 180 participants in this experiment.
|
1179
|
-
This experiment did not run long enough to find a clear winner.
|
1176
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1177
|
+
There are 180 participants in this experiment.
|
1178
|
+
This experiment did not run long enough to find a clear winner.
|
1180
1179
|
TEXT
|
1181
1180
|
end
|
1182
1181
|
|
@@ -1186,13 +1185,12 @@ This experiment did not run long enough to find a clear winner.
|
|
1186
1185
|
default :a
|
1187
1186
|
metrics :coolness
|
1188
1187
|
end
|
1189
|
-
assert_equal
|
1190
|
-
There are no participants in this experiment yet.
|
1191
|
-
This experiment did not run long enough to find a clear winner.
|
1188
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1189
|
+
There are no participants in this experiment yet.
|
1190
|
+
This experiment did not run long enough to find a clear winner.
|
1192
1191
|
TEXT
|
1193
1192
|
end
|
1194
1193
|
|
1195
|
-
|
1196
1194
|
# -- Completion --
|
1197
1195
|
|
1198
1196
|
def test_completion_if
|
@@ -1209,7 +1207,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1209
1207
|
def test_completion_if_fails
|
1210
1208
|
new_ab_test :simple do
|
1211
1209
|
identify { rand }
|
1212
|
-
complete_if {
|
1210
|
+
complete_if { raise "Testing complete_if raises exception" }
|
1213
1211
|
metrics :coolness
|
1214
1212
|
default false
|
1215
1213
|
end
|
@@ -1225,7 +1223,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1225
1223
|
metrics :coolness
|
1226
1224
|
default false
|
1227
1225
|
end
|
1228
|
-
99.times do |
|
1226
|
+
99.times do |_i|
|
1229
1227
|
experiment(:simple).choose
|
1230
1228
|
assert experiment(:simple).active?
|
1231
1229
|
end
|
@@ -1262,7 +1260,6 @@ This experiment did not run long enough to find a clear winner.
|
|
1262
1260
|
assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
|
1263
1261
|
end
|
1264
1262
|
|
1265
|
-
|
1266
1263
|
# -- Outcome --
|
1267
1264
|
|
1268
1265
|
def test_completion_outcome
|
@@ -1319,7 +1316,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1319
1316
|
|
1320
1317
|
def test_outcome_is_fails
|
1321
1318
|
new_ab_test :quick do
|
1322
|
-
outcome_is {
|
1319
|
+
outcome_is { raise "Testing outcome_is raising exception" }
|
1323
1320
|
metrics :coolness
|
1324
1321
|
default false
|
1325
1322
|
end
|
@@ -1332,7 +1329,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1332
1329
|
metrics :coolness
|
1333
1330
|
default false
|
1334
1331
|
end
|
1335
|
-
fake :quick, false=>[2,0], true=>10
|
1332
|
+
fake :quick, false => [2, 0], true => 10
|
1336
1333
|
experiment(:quick).complete!
|
1337
1334
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1338
1335
|
end
|
@@ -1342,7 +1339,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1342
1339
|
metrics :coolness
|
1343
1340
|
default false
|
1344
1341
|
end
|
1345
|
-
fake :quick, true=>2
|
1342
|
+
fake :quick, true => 2
|
1346
1343
|
experiment(:quick).complete!
|
1347
1344
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1348
1345
|
end
|
@@ -1352,12 +1349,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1352
1349
|
metrics :coolness
|
1353
1350
|
default false
|
1354
1351
|
end
|
1355
|
-
fake :quick, false=>8, true=>8
|
1352
|
+
fake :quick, false => 8, true => 8
|
1356
1353
|
experiment(:quick).complete!
|
1357
1354
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1358
1355
|
end
|
1359
1356
|
|
1360
|
-
|
1361
1357
|
# -- No collection --
|
1362
1358
|
|
1363
1359
|
def test_no_collection_does_not_track
|
@@ -1400,7 +1396,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1400
1396
|
identify { "1" }
|
1401
1397
|
end
|
1402
1398
|
val = experiment(:simple).choose.value
|
1403
|
-
alternative = experiment(:simple).alternatives.detect {|a| a.value != val }
|
1399
|
+
alternative = experiment(:simple).alternatives.detect { |a| a.value != val }
|
1404
1400
|
experiment(:simple).chooses(alternative.value)
|
1405
1401
|
assert_equal experiment(:simple).choose.value, alternative.value
|
1406
1402
|
experiment(:simple).chooses(val)
|
@@ -1481,7 +1477,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1481
1477
|
default :b
|
1482
1478
|
metrics :coolness
|
1483
1479
|
|
1484
|
-
reject do |
|
1480
|
+
reject do |_request, _identity|
|
1485
1481
|
true
|
1486
1482
|
end
|
1487
1483
|
end
|
@@ -1508,11 +1504,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1508
1504
|
end
|
1509
1505
|
|
1510
1506
|
def test_clears_outcome_and_completed_at
|
1511
|
-
|
1512
|
-
|
1513
|
-
|
1514
|
-
|
1515
|
-
|
1507
|
+
new_ab_test :simple do
|
1508
|
+
alternatives :a, :b, :c
|
1509
|
+
default :a
|
1510
|
+
metrics :coolness
|
1511
|
+
end
|
1516
1512
|
experiment(:simple).reset
|
1517
1513
|
assert_nil experiment(:simple).outcome
|
1518
1514
|
assert_nil experiment(:simple).completed_at
|
@@ -1531,7 +1527,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1531
1527
|
assert_not_nil experiment(:simple).completed_at
|
1532
1528
|
end
|
1533
1529
|
|
1534
|
-
def test_reset_clears_participants
|
1530
|
+
def test_reset_clears_participants # rubocop:todo Lint/DuplicateMethods
|
1535
1531
|
new_ab_test :simple do
|
1536
1532
|
alternatives :a, :b, :c
|
1537
1533
|
default :a
|
@@ -1543,7 +1539,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1543
1539
|
assert_equal experiment(:simple).alternatives[1].participants, 0
|
1544
1540
|
end
|
1545
1541
|
|
1546
|
-
def test_clears_outcome_and_completed_at
|
1542
|
+
def test_clears_outcome_and_completed_at # rubocop:todo Lint/DuplicateMethods
|
1547
1543
|
new_ab_test :simple do
|
1548
1544
|
alternatives :a, :b, :c
|
1549
1545
|
default :a
|
@@ -1554,11 +1550,9 @@ This experiment did not run long enough to find a clear winner.
|
|
1554
1550
|
assert_nil experiment(:simple).completed_at
|
1555
1551
|
end
|
1556
1552
|
|
1557
|
-
|
1558
1553
|
# -- Helper methods --
|
1559
1554
|
|
1560
1555
|
def fake(name, args)
|
1561
1556
|
experiment(name).instance_eval { fake args }
|
1562
1557
|
end
|
1563
|
-
|
1564
1558
|
end
|