vanity 3.0.2 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linting.yml +28 -0
- data/.github/workflows/test.yml +3 -6
- data/.rubocop.yml +114 -0
- data/.rubocop_todo.yml +67 -0
- data/Appraisals +9 -31
- data/CHANGELOG +13 -0
- data/Gemfile +7 -3
- data/Gemfile.lock +32 -4
- data/README.md +4 -9
- data/Rakefile +25 -24
- data/bin/vanity +1 -1
- data/doc/configuring.textile +1 -0
- data/gemfiles/rails52.gemfile +6 -3
- data/gemfiles/rails52.gemfile.lock +34 -9
- data/gemfiles/rails60.gemfile +6 -3
- data/gemfiles/rails60.gemfile.lock +34 -9
- data/gemfiles/rails61.gemfile +6 -3
- data/gemfiles/rails61.gemfile.lock +34 -9
- data/lib/generators/vanity/migration_generator.rb +5 -7
- data/lib/vanity/adapters/abstract_adapter.rb +43 -45
- data/lib/vanity/adapters/active_record_adapter.rb +30 -30
- data/lib/vanity/adapters/mock_adapter.rb +14 -18
- data/lib/vanity/adapters/mongodb_adapter.rb +73 -69
- data/lib/vanity/adapters/redis_adapter.rb +19 -27
- data/lib/vanity/adapters.rb +1 -1
- data/lib/vanity/autoconnect.rb +6 -7
- data/lib/vanity/commands/list.rb +7 -7
- data/lib/vanity/commands/report.rb +18 -22
- data/lib/vanity/configuration.rb +23 -19
- data/lib/vanity/connection.rb +12 -14
- data/lib/vanity/experiment/ab_test.rb +95 -79
- data/lib/vanity/experiment/alternative.rb +3 -5
- data/lib/vanity/experiment/base.rb +24 -19
- data/lib/vanity/experiment/bayesian_bandit_score.rb +7 -13
- data/lib/vanity/experiment/definition.rb +6 -6
- data/lib/vanity/frameworks/rails.rb +39 -39
- data/lib/vanity/frameworks.rb +2 -2
- data/lib/vanity/helpers.rb +1 -1
- data/lib/vanity/metric/active_record.rb +21 -19
- data/lib/vanity/metric/base.rb +22 -23
- data/lib/vanity/metric/google_analytics.rb +6 -9
- data/lib/vanity/metric/remote.rb +3 -5
- data/lib/vanity/playground.rb +3 -6
- data/lib/vanity/vanity.rb +8 -12
- data/lib/vanity/version.rb +1 -1
- data/test/adapters/active_record_adapter_test.rb +1 -5
- data/test/adapters/mock_adapter_test.rb +0 -2
- data/test/adapters/mongodb_adapter_test.rb +1 -5
- data/test/adapters/redis_adapter_test.rb +2 -3
- data/test/adapters/shared_tests.rb +9 -12
- data/test/autoconnect_test.rb +3 -3
- data/test/cli_test.rb +0 -1
- data/test/configuration_test.rb +18 -34
- data/test/connection_test.rb +3 -3
- data/test/dummy/Rakefile +1 -1
- data/test/dummy/app/controllers/use_vanity_controller.rb +12 -8
- data/test/dummy/app/mailers/vanity_mailer.rb +3 -3
- data/test/dummy/config/application.rb +1 -1
- data/test/dummy/config/boot.rb +3 -3
- data/test/dummy/config/environment.rb +1 -1
- data/test/dummy/config/environments/development.rb +0 -1
- data/test/dummy/config/environments/test.rb +1 -1
- data/test/dummy/config/initializers/session_store.rb +1 -1
- data/test/dummy/config/initializers/vanity.rb +3 -0
- data/test/dummy/config/vanity.yml +7 -0
- data/test/dummy/config.ru +1 -1
- data/test/dummy/script/rails +2 -2
- data/test/experiment/ab_test.rb +188 -154
- data/test/experiment/base_test.rb +48 -32
- data/test/frameworks/rails/action_controller_test.rb +25 -25
- data/test/frameworks/rails/action_mailer_test.rb +2 -2
- data/test/frameworks/rails/action_view_test.rb +5 -6
- data/test/frameworks/rails/rails_test.rb +147 -181
- data/test/helper_test.rb +2 -2
- data/test/metric/active_record_test.rb +174 -212
- data/test/metric/base_test.rb +21 -46
- data/test/metric/google_analytics_test.rb +17 -25
- data/test/metric/remote_test.rb +7 -10
- data/test/playground_test.rb +7 -15
- data/test/templates_test.rb +16 -20
- data/test/test_helper.rb +28 -29
- data/test/vanity_test.rb +4 -10
- data/test/web/rails/dashboard_test.rb +21 -21
- data/vanity.gemspec +8 -7
- metadata +32 -30
- data/gemfiles/rails42.gemfile +0 -33
- data/gemfiles/rails42.gemfile.lock +0 -265
- data/gemfiles/rails42_protected_attributes.gemfile +0 -34
- data/gemfiles/rails42_protected_attributes.gemfile.lock +0 -264
- data/gemfiles/rails51.gemfile +0 -33
- data/gemfiles/rails51.gemfile.lock +0 -285
data/test/experiment/ab_test.rb
CHANGED
@@ -6,24 +6,23 @@ class AbTestController < ActionController::Base
|
|
6
6
|
|
7
7
|
def test_render
|
8
8
|
text = Vanity.ab_test(:simple)
|
9
|
-
render :
|
9
|
+
render plain: text, text: text
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_view
|
13
|
-
render :
|
13
|
+
render inline: "<%= ab_test(:simple) %>"
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_capture
|
17
|
-
render :
|
17
|
+
render inline: "<%= ab_test(:simple) do |value| %><%= value %><% end %>"
|
18
18
|
end
|
19
19
|
|
20
20
|
def track
|
21
21
|
Vanity.track!(:coolness)
|
22
|
-
render :
|
22
|
+
render plain: "", text: ""
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
26
|
class AbTestTest < ActionController::TestCase
|
28
27
|
tests AbTestController
|
29
28
|
|
@@ -85,7 +84,7 @@ class AbTestTest < ActionController::TestCase
|
|
85
84
|
alternatives :a, :b
|
86
85
|
default :a
|
87
86
|
end
|
88
|
-
fingerprints = Vanity.playground.experiments.map { |
|
87
|
+
fingerprints = Vanity.playground.experiments.map { |_id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
89
88
|
assert_equal 4, fingerprints.uniq.size
|
90
89
|
end
|
91
90
|
|
@@ -97,7 +96,7 @@ class AbTestTest < ActionController::TestCase
|
|
97
96
|
end
|
98
97
|
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
99
98
|
fingerprints.each do |fingerprint|
|
100
|
-
assert_match
|
99
|
+
assert_match(/^[0-9a-f]{10}$/i, fingerprint)
|
101
100
|
end
|
102
101
|
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
103
102
|
end
|
@@ -183,25 +182,24 @@ class AbTestTest < ActionController::TestCase
|
|
183
182
|
assert_equal exp.default, exp.alternative(nil)
|
184
183
|
end
|
185
184
|
|
186
|
-
|
187
185
|
# -- Experiment Enabled/disabled --
|
188
186
|
|
189
187
|
# @example new test should be enabled regardless of collecting?
|
190
188
|
# regardless_of "Vanity.playground.collecting" do
|
191
189
|
# assert (new_ab_test :test).enabled?
|
192
190
|
# end
|
193
|
-
def regardless_of(attr_name
|
194
|
-
prev_val = eval "#{attr_name}?"
|
191
|
+
def regardless_of(attr_name)
|
192
|
+
prev_val = eval "#{attr_name}?" # rubocop:todo Lint/UselessAssignment, Style/EvalWithLocation, Security/Eval
|
195
193
|
|
196
|
-
eval "#{attr_name}=true"
|
197
|
-
|
194
|
+
eval "#{attr_name}=true" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
195
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
198
196
|
nuke_playground
|
199
197
|
|
200
|
-
eval "#{attr_name}=false"
|
201
|
-
|
198
|
+
eval "#{attr_name}=false" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
199
|
+
yield(eval("#{attr_name}?")) # rubocop:todo Style/EvalWithLocation, Security/Eval
|
202
200
|
nuke_playground
|
203
201
|
|
204
|
-
eval "#{attr_name}=prev_val"
|
202
|
+
eval "#{attr_name}=prev_val" # rubocop:todo Style/EvalWithLocation, Security/Eval
|
205
203
|
end
|
206
204
|
|
207
205
|
def test_new_test_is_disabled_when_experiments_start_enabled_is_false
|
@@ -228,7 +226,7 @@ class AbTestTest < ActionController::TestCase
|
|
228
226
|
metrics :coolness
|
229
227
|
default false
|
230
228
|
end
|
231
|
-
exp.complete! #active? => false
|
229
|
+
exp.complete! # active? => false
|
232
230
|
|
233
231
|
assert !exp.enabled?, "experiment should not be enabled but it is!"
|
234
232
|
end
|
@@ -263,7 +261,7 @@ class AbTestTest < ActionController::TestCase
|
|
263
261
|
metrics :coolness
|
264
262
|
default false
|
265
263
|
end
|
266
|
-
exp.complete! #active? => false
|
264
|
+
exp.complete! # active? => false
|
267
265
|
assert !exp.enabled?
|
268
266
|
exp.enabled = true
|
269
267
|
assert !exp.enabled?
|
@@ -283,65 +281,65 @@ class AbTestTest < ActionController::TestCase
|
|
283
281
|
def test_enabled_persists_across_definitions
|
284
282
|
Vanity.configuration.experiments_start_enabled = false
|
285
283
|
Vanity.playground.collecting = true
|
286
|
-
new_ab_test :test, :
|
284
|
+
new_ab_test :test, enable: false do
|
287
285
|
metrics :coolness
|
288
286
|
default false
|
289
287
|
end
|
290
|
-
assert !experiment(:test).enabled? #starts off false
|
288
|
+
assert !experiment(:test).enabled? # starts off false
|
291
289
|
|
292
290
|
new_playground
|
293
291
|
metric "Coolness"
|
294
292
|
|
295
|
-
new_ab_test :test, :
|
293
|
+
new_ab_test :test, enable: false do
|
296
294
|
metrics :coolness
|
297
295
|
default false
|
298
296
|
end
|
299
|
-
assert !experiment(:test).enabled? #still false
|
297
|
+
assert !experiment(:test).enabled? # still false
|
300
298
|
experiment(:test).enabled = true
|
301
|
-
assert experiment(:test).enabled? #now true
|
299
|
+
assert experiment(:test).enabled? # now true
|
302
300
|
|
303
301
|
new_playground
|
304
302
|
metric "Coolness"
|
305
303
|
|
306
|
-
new_ab_test :test, :
|
304
|
+
new_ab_test :test, enable: false do
|
307
305
|
metrics :coolness
|
308
306
|
default false
|
309
307
|
end
|
310
|
-
assert experiment(:test).enabled? #still true
|
308
|
+
assert experiment(:test).enabled? # still true
|
311
309
|
experiment(:test).enabled = false
|
312
|
-
assert !experiment(:test).enabled? #now false again
|
310
|
+
assert !experiment(:test).enabled? # now false again
|
313
311
|
end
|
314
312
|
|
315
313
|
def test_enabled_persists_across_definitions_when_starting_enabled
|
316
314
|
Vanity.configuration.experiments_start_enabled = true
|
317
315
|
Vanity.playground.collecting = true
|
318
|
-
new_ab_test :test, :
|
316
|
+
new_ab_test :test, enable: false do
|
319
317
|
metrics :coolness
|
320
318
|
default false
|
321
319
|
end
|
322
|
-
assert experiment(:test).enabled? #starts off true
|
320
|
+
assert experiment(:test).enabled? # starts off true
|
323
321
|
|
324
322
|
new_playground
|
325
323
|
metric "Coolness"
|
326
324
|
|
327
|
-
new_ab_test :test, :
|
325
|
+
new_ab_test :test, enable: false do
|
328
326
|
metrics :coolness
|
329
327
|
default false
|
330
328
|
end
|
331
|
-
assert experiment(:test).enabled? #still true
|
329
|
+
assert experiment(:test).enabled? # still true
|
332
330
|
experiment(:test).enabled = false
|
333
|
-
assert !experiment(:test).enabled? #now false
|
331
|
+
assert !experiment(:test).enabled? # now false
|
334
332
|
|
335
333
|
new_playground
|
336
334
|
metric "Coolness"
|
337
335
|
|
338
|
-
new_ab_test :test, :
|
336
|
+
new_ab_test :test, enable: false do
|
339
337
|
metrics :coolness
|
340
338
|
default false
|
341
339
|
end
|
342
|
-
assert !experiment(:test).enabled? #still false
|
340
|
+
assert !experiment(:test).enabled? # still false
|
343
341
|
experiment(:test).enabled = true
|
344
|
-
assert experiment(:test).enabled? #now true again
|
342
|
+
assert experiment(:test).enabled? # now true again
|
345
343
|
end
|
346
344
|
|
347
345
|
def test_choose_random_when_enabled
|
@@ -378,7 +376,7 @@ class AbTestTest < ActionController::TestCase
|
|
378
376
|
def test_choose_outcome_when_finished
|
379
377
|
exp = new_ab_test :test do
|
380
378
|
metrics :coolness
|
381
|
-
alternatives 0,1,2,3,4,5
|
379
|
+
alternatives 0, 1, 2, 3, 4, 5
|
382
380
|
default 3
|
383
381
|
outcome_is { alternative(5) }
|
384
382
|
end
|
@@ -429,7 +427,7 @@ class AbTestTest < ActionController::TestCase
|
|
429
427
|
assert_equal 0, experiment(:foobar).alternatives.sum(&:converted)
|
430
428
|
experiment(:foobar).track!(:coolness, Time.now, 1)
|
431
429
|
assert_equal 1, experiment(:foobar).alternatives.sum(&:converted)
|
432
|
-
experiment(:foobar).track!(:coolness, Time.now, 1, :
|
430
|
+
experiment(:foobar).track!(:coolness, Time.now, 1, identity: "quux")
|
433
431
|
assert_equal 2, experiment(:foobar).alternatives.sum(&:converted)
|
434
432
|
end
|
435
433
|
|
@@ -458,7 +456,7 @@ class AbTestTest < ActionController::TestCase
|
|
458
456
|
alternatives "foo", "bar"
|
459
457
|
identify { "6e98ec" }
|
460
458
|
metrics :coolness
|
461
|
-
on_assignment { on_assignment_called_times
|
459
|
+
on_assignment { on_assignment_called_times += 1 }
|
462
460
|
end
|
463
461
|
2.times { experiment(:foobar).choose }
|
464
462
|
assert_equal 1, on_assignment_called_times
|
@@ -471,7 +469,7 @@ class AbTestTest < ActionController::TestCase
|
|
471
469
|
alternatives "foo", "bar"
|
472
470
|
identify { "6e98ec" }
|
473
471
|
metrics :coolness
|
474
|
-
on_assignment { on_assignment_called_times
|
472
|
+
on_assignment { on_assignment_called_times += 1 }
|
475
473
|
end
|
476
474
|
experiment(:foobar).choose(dummy_request)
|
477
475
|
assert_equal 1, on_assignment_called_times
|
@@ -484,7 +482,7 @@ class AbTestTest < ActionController::TestCase
|
|
484
482
|
alternatives "foo", "bar"
|
485
483
|
identify { "6e98ec" }
|
486
484
|
metrics :coolness
|
487
|
-
on_assignment { on_assignment_called_times
|
485
|
+
on_assignment { on_assignment_called_times += 1 }
|
488
486
|
end
|
489
487
|
request = dummy_request
|
490
488
|
request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
|
@@ -499,7 +497,7 @@ class AbTestTest < ActionController::TestCase
|
|
499
497
|
default "foo"
|
500
498
|
identify { "6e98ec" }
|
501
499
|
metrics :coolness
|
502
|
-
on_assignment { on_assignment_called_times
|
500
|
+
on_assignment { on_assignment_called_times += 1 }
|
503
501
|
end
|
504
502
|
2.times { experiment(:foobar).chooses("foo") }
|
505
503
|
assert_equal 1, on_assignment_called_times
|
@@ -510,16 +508,56 @@ class AbTestTest < ActionController::TestCase
|
|
510
508
|
alternatives "foo", "bar"
|
511
509
|
default "foo"
|
512
510
|
identify { "6e98ec" }
|
513
|
-
on_assignment {}
|
511
|
+
on_assignment {} # rubocop:todo Lint/EmptyBlock
|
514
512
|
metrics :coolness
|
515
513
|
end
|
516
514
|
assert value = experiment(:foobar).choose.value
|
517
|
-
assert_match
|
515
|
+
assert_match(/foo|bar/, value)
|
518
516
|
1000.times do
|
519
517
|
assert_equal value, experiment(:foobar).choose.value
|
520
518
|
end
|
521
519
|
end
|
522
520
|
|
521
|
+
def test_calls_default_on_assignment
|
522
|
+
on_assignment_called_times = 0
|
523
|
+
|
524
|
+
Vanity.configuration.on_assignment = proc do |_controller, _identity, _assignment|
|
525
|
+
on_assignment_called_times += 1
|
526
|
+
end
|
527
|
+
|
528
|
+
new_ab_test :foobar do
|
529
|
+
alternatives "foo", "bar"
|
530
|
+
default "foo"
|
531
|
+
identify { "6e98ec" }
|
532
|
+
metrics :coolness
|
533
|
+
end
|
534
|
+
|
535
|
+
2.times { experiment(:foobar).chooses("foo") }
|
536
|
+
assert_equal 1, on_assignment_called_times
|
537
|
+
end
|
538
|
+
|
539
|
+
def test_calls_on_assignment_defined_in_experiment
|
540
|
+
expected_value = 0
|
541
|
+
|
542
|
+
Vanity.configuration.on_assignment = proc do |_controller, _identity, _assignment|
|
543
|
+
expected_value = 1
|
544
|
+
end
|
545
|
+
|
546
|
+
new_ab_test :foobar do
|
547
|
+
alternatives "foo", "bar"
|
548
|
+
default "foo"
|
549
|
+
identify { "6e98ec" }
|
550
|
+
metrics :coolness
|
551
|
+
|
552
|
+
on_assignment do |_controller, _identity, _assignment|
|
553
|
+
expected_value = 20
|
554
|
+
end
|
555
|
+
end
|
556
|
+
|
557
|
+
2.times { experiment(:foobar).chooses("foo") }
|
558
|
+
assert_equal 20, expected_value
|
559
|
+
end
|
560
|
+
|
523
561
|
# -- ab_assigned --
|
524
562
|
|
525
563
|
def test_ab_assigned
|
@@ -535,7 +573,7 @@ class AbTestTest < ActionController::TestCase
|
|
535
573
|
end
|
536
574
|
|
537
575
|
def test_ab_assigned_object
|
538
|
-
identity = { :
|
576
|
+
identity = { a: :b }
|
539
577
|
new_ab_test :foobar do
|
540
578
|
alternatives "foo", "bar"
|
541
579
|
default "foo"
|
@@ -559,7 +597,7 @@ class AbTestTest < ActionController::TestCase
|
|
559
597
|
end
|
560
598
|
value = experiment(:foobar).choose.value
|
561
599
|
assert value
|
562
|
-
assert_match
|
600
|
+
assert_match(/foo|bar/, value)
|
563
601
|
100.times do
|
564
602
|
assert_equal value, experiment(:foobar).choose.value
|
565
603
|
end
|
@@ -571,7 +609,10 @@ class AbTestTest < ActionController::TestCase
|
|
571
609
|
identify { rand }
|
572
610
|
metrics :coolness
|
573
611
|
end
|
574
|
-
alts = Array.new(10_000) { experiment(:foobar).choose.value }.
|
612
|
+
alts = Array.new(10_000) { experiment(:foobar).choose.value }.each_with_object({}) do |k, h|
|
613
|
+
h[k] ||= 0
|
614
|
+
h[k] += 1
|
615
|
+
end
|
575
616
|
assert_equal %w{bar foo}, alts.keys.sort
|
576
617
|
assert_in_delta 3333, alts["foo"], 200 # this may fail, such is propability
|
577
618
|
end
|
@@ -585,8 +626,8 @@ class AbTestTest < ActionController::TestCase
|
|
585
626
|
metrics :coolness
|
586
627
|
end
|
587
628
|
altered_alts = experiment(:foobar).alternatives
|
588
|
-
altered_alts[0].probability=30
|
589
|
-
altered_alts[1].probability=70
|
629
|
+
altered_alts[0].probability = 30
|
630
|
+
altered_alts[1].probability = 70
|
590
631
|
experiment(:foobar).set_alternative_probabilities altered_alts
|
591
632
|
alts = Array.new(600) { experiment(:foobar).choose.value }
|
592
633
|
assert_equal %w{bar foo}, alts.uniq.sort
|
@@ -603,10 +644,11 @@ class AbTestTest < ActionController::TestCase
|
|
603
644
|
rebalance_frequency 12
|
604
645
|
metrics :coolness
|
605
646
|
end
|
606
|
-
class <<experiment(:foobar)
|
647
|
+
class << experiment(:foobar)
|
607
648
|
def times_called
|
608
649
|
@times_called ||= 0
|
609
650
|
end
|
651
|
+
|
610
652
|
def rebalance!
|
611
653
|
@times_called = times_called + 1
|
612
654
|
end
|
@@ -629,20 +671,16 @@ class AbTestTest < ActionController::TestCase
|
|
629
671
|
end
|
630
672
|
corresponding_probabilities = [[experiment(:foobar).alternatives[0], 0.3], [experiment(:foobar).alternatives[1], 0.6], [experiment(:foobar).alternatives[2], 1.0]]
|
631
673
|
|
632
|
-
class <<experiment(:foobar)
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
def bayes_bandit_score(probability=90)
|
674
|
+
class << experiment(:foobar)
|
675
|
+
attr_reader :was_called, :use_probabilities
|
676
|
+
|
677
|
+
def bayes_bandit_score(_probability = 90)
|
637
678
|
@was_called = true
|
638
679
|
altered_alts = Vanity.playground.experiment(:foobar).alternatives
|
639
|
-
altered_alts[0].probability=30
|
640
|
-
altered_alts[1].probability=30
|
641
|
-
altered_alts[2].probability=40
|
642
|
-
Struct.new(:alts
|
643
|
-
end
|
644
|
-
def use_probabilities
|
645
|
-
@use_probabilities
|
680
|
+
altered_alts[0].probability = 30
|
681
|
+
altered_alts[1].probability = 30
|
682
|
+
altered_alts[2].probability = 40
|
683
|
+
Struct.new(:alts, :method).new(altered_alts, :bayes_bandit_score) # rubocop:todo Lint/StructNewOverride
|
646
684
|
end
|
647
685
|
end
|
648
686
|
experiment(:foobar).rebalance!
|
@@ -660,7 +698,7 @@ class AbTestTest < ActionController::TestCase
|
|
660
698
|
metrics :coolness
|
661
699
|
end
|
662
700
|
assert value = experiment(:foobar).choose.value
|
663
|
-
assert_match
|
701
|
+
assert_match(/foo|bar/, value)
|
664
702
|
1000.times do
|
665
703
|
assert_equal value, experiment(:foobar).choose.value
|
666
704
|
end
|
@@ -712,7 +750,7 @@ class AbTestTest < ActionController::TestCase
|
|
712
750
|
end
|
713
751
|
|
714
752
|
def test_records_each_converted_participant_only_once
|
715
|
-
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
753
|
+
ids = ((1..100).map { |i| [i, i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
716
754
|
new_ab_test :foobar do
|
717
755
|
alternatives "foo", "bar"
|
718
756
|
default "foo"
|
@@ -728,7 +766,7 @@ class AbTestTest < ActionController::TestCase
|
|
728
766
|
end
|
729
767
|
|
730
768
|
def test_records_conversion_only_for_participants
|
731
|
-
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
769
|
+
ids = ((1..100).map { |i| [-i, i, i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
732
770
|
new_ab_test :foobar do
|
733
771
|
alternatives "foo", "bar"
|
734
772
|
default "foo"
|
@@ -790,7 +828,6 @@ class AbTestTest < ActionController::TestCase
|
|
790
828
|
assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
|
791
829
|
end
|
792
830
|
|
793
|
-
|
794
831
|
# -- A/B helper methods --
|
795
832
|
|
796
833
|
def test_fail_if_no_experiment
|
@@ -806,7 +843,8 @@ class AbTestTest < ActionController::TestCase
|
|
806
843
|
default false
|
807
844
|
end
|
808
845
|
responses = Array.new(100) do
|
809
|
-
@controller = nil
|
846
|
+
@controller = nil
|
847
|
+
setup_controller_request_and_response
|
810
848
|
get :test_render
|
811
849
|
@response.body
|
812
850
|
end
|
@@ -820,7 +858,8 @@ class AbTestTest < ActionController::TestCase
|
|
820
858
|
default false
|
821
859
|
end
|
822
860
|
responses = Array.new(100) do
|
823
|
-
@controller = nil
|
861
|
+
@controller = nil
|
862
|
+
setup_controller_request_and_response
|
824
863
|
get :test_view
|
825
864
|
@response.body
|
826
865
|
end
|
@@ -834,7 +873,8 @@ class AbTestTest < ActionController::TestCase
|
|
834
873
|
default false
|
835
874
|
end
|
836
875
|
responses = Array.new(100) do
|
837
|
-
@controller = nil
|
876
|
+
@controller = nil
|
877
|
+
setup_controller_request_and_response
|
838
878
|
get :test_capture
|
839
879
|
@response.body
|
840
880
|
end
|
@@ -846,14 +886,13 @@ class AbTestTest < ActionController::TestCase
|
|
846
886
|
metrics :coolness
|
847
887
|
default false
|
848
888
|
end
|
849
|
-
responses = Array.new(100) do
|
850
|
-
@controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
|
889
|
+
responses = Array.new(100) do # rubocop:todo Lint/UselessAssignment
|
890
|
+
@controller.send(:cookies).each { |cookie| @controller.send(:cookies).delete(cookie.first) }
|
851
891
|
get :track
|
852
892
|
@response.body
|
853
893
|
end
|
854
894
|
end
|
855
895
|
|
856
|
-
|
857
896
|
# -- Testing with tests --
|
858
897
|
|
859
898
|
def test_with_given_choice
|
@@ -862,8 +901,9 @@ class AbTestTest < ActionController::TestCase
|
|
862
901
|
default :a
|
863
902
|
metrics :coolness
|
864
903
|
end
|
865
|
-
100.times do |
|
866
|
-
@controller = nil
|
904
|
+
100.times do |_i|
|
905
|
+
@controller = nil
|
906
|
+
setup_controller_request_and_response
|
867
907
|
experiment(:simple).chooses(:b)
|
868
908
|
get :test_render
|
869
909
|
assert "b", @response.body
|
@@ -887,8 +927,9 @@ class AbTestTest < ActionController::TestCase
|
|
887
927
|
default :a
|
888
928
|
metrics :coolness
|
889
929
|
end
|
890
|
-
responses = Array.new(100) do |
|
891
|
-
@controller = nil
|
930
|
+
responses = Array.new(100) do |_i|
|
931
|
+
@controller = nil
|
932
|
+
setup_controller_request_and_response
|
892
933
|
experiment(:simple).chooses(:b)
|
893
934
|
experiment(:simple).chooses(nil)
|
894
935
|
get :test_render
|
@@ -897,7 +938,6 @@ class AbTestTest < ActionController::TestCase
|
|
897
938
|
assert responses.uniq.size == 3
|
898
939
|
end
|
899
940
|
|
900
|
-
|
901
941
|
# -- Scoring --
|
902
942
|
|
903
943
|
def test_calculate_score
|
@@ -930,9 +970,9 @@ class AbTestTest < ActionController::TestCase
|
|
930
970
|
# Treatment A: 180 45 25.00% 1.33
|
931
971
|
# treatment B: 189 28 14.81% -1.13
|
932
972
|
# treatment C: 188 61 32.45% 2.94
|
933
|
-
fake :abcd, :
|
973
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
934
974
|
|
935
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
975
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score } # rubocop:todo Style/FormatString
|
936
976
|
assert_equal %w{-1.33 0.00 -2.46 1.58}, z_scores
|
937
977
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
938
978
|
assert_equal [90, 0, 99, 90], probabilities
|
@@ -957,11 +997,11 @@ class AbTestTest < ActionController::TestCase
|
|
957
997
|
# Treatment A: 180 45 25.00% 1.33
|
958
998
|
# treatment B: 189 28 14.81% -1.13
|
959
999
|
# treatment C: 188 61 32.45% 2.94
|
960
|
-
fake :abcd, :
|
1000
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
961
1001
|
|
962
1002
|
score_result = experiment(:abcd).bayes_bandit_score
|
963
|
-
probabilities = score_result.alts.map{|a| a.probability.round}
|
964
|
-
assert_equal [0,0,6,94], probabilities
|
1003
|
+
probabilities = score_result.alts.map { |a| a.probability.round }
|
1004
|
+
assert_equal [0, 0, 6, 94], probabilities
|
965
1005
|
end
|
966
1006
|
|
967
1007
|
def test_scoring_with_no_performers
|
@@ -984,13 +1024,13 @@ class AbTestTest < ActionController::TestCase
|
|
984
1024
|
default :a
|
985
1025
|
metrics :coolness
|
986
1026
|
end
|
987
|
-
fake :abcd, :
|
1027
|
+
fake :abcd, b: [10, 8]
|
988
1028
|
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
989
1029
|
assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
|
990
1030
|
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
991
1031
|
assert_equal 1, experiment(:abcd).score.best.id
|
992
1032
|
assert_nil experiment(:abcd).score.choice
|
993
|
-
assert_includes [0,2,3], experiment(:abcd).score.base.id
|
1033
|
+
assert_includes [0, 2, 3], experiment(:abcd).score.base.id
|
994
1034
|
assert_equal 1, experiment(:abcd).score.least.id
|
995
1035
|
end
|
996
1036
|
|
@@ -1000,9 +1040,9 @@ class AbTestTest < ActionController::TestCase
|
|
1000
1040
|
default :a
|
1001
1041
|
metrics :coolness
|
1002
1042
|
end
|
1003
|
-
fake :abcd, :
|
1043
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1004
1044
|
|
1005
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase)
|
1045
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase) # rubocop:todo Style/FormatString
|
1006
1046
|
assert_equal %w{nan 2.01 nan 0.00}, z_scores
|
1007
1047
|
probabilities = experiment(:abcd).score.alts.map(&:probability)
|
1008
1048
|
assert_equal [0, 95, 0, 0], probabilities
|
@@ -1020,14 +1060,13 @@ class AbTestTest < ActionController::TestCase
|
|
1020
1060
|
default :a
|
1021
1061
|
metrics :coolness
|
1022
1062
|
end
|
1023
|
-
fake :abcd, :
|
1063
|
+
fake :abcd, b: [10, 8], d: [12, 5]
|
1024
1064
|
|
1025
1065
|
assert_equal 1, experiment(:abcd).score(90).choice.id
|
1026
1066
|
assert_equal 1, experiment(:abcd).score(95).choice.id
|
1027
1067
|
assert_nil experiment(:abcd).score(99).choice
|
1028
1068
|
end
|
1029
1069
|
|
1030
|
-
|
1031
1070
|
# -- Conclusion --
|
1032
1071
|
|
1033
1072
|
def test_conclusion
|
@@ -1041,16 +1080,16 @@ class AbTestTest < ActionController::TestCase
|
|
1041
1080
|
# Treatment A: 180 45 25.00% 1.33
|
1042
1081
|
# treatment B: 189 28 14.81% -1.13
|
1043
1082
|
# treatment C: 188 61 32.45% 2.94
|
1044
|
-
fake :abcd, :
|
1045
|
-
|
1046
|
-
assert_equal
|
1047
|
-
There are 739 participants in this experiment.
|
1048
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1049
|
-
With 90% probability this result is statistically significant.
|
1050
|
-
Option B converted at 25.0%.
|
1051
|
-
Option A converted at 19.2%.
|
1052
|
-
Option C converted at 14.8%.
|
1053
|
-
Option D selected as the best alternative.
|
1083
|
+
fake :abcd, a: [182, 35], b: [180, 45], c: [189, 28], d: [188, 61]
|
1084
|
+
|
1085
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1086
|
+
There are 739 participants in this experiment.
|
1087
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1088
|
+
With 90% probability this result is statistically significant.
|
1089
|
+
Option B converted at 25.0%.
|
1090
|
+
Option A converted at 19.2%.
|
1091
|
+
Option C converted at 14.8%.
|
1092
|
+
Option D selected as the best alternative.
|
1054
1093
|
TEXT
|
1055
1094
|
end
|
1056
1095
|
|
@@ -1060,16 +1099,16 @@ Option D selected as the best alternative.
|
|
1060
1099
|
default :a
|
1061
1100
|
metrics :coolness
|
1062
1101
|
end
|
1063
|
-
fake :abcd, :
|
1102
|
+
fake :abcd, b: [180, 45], d: [188, 61]
|
1064
1103
|
|
1065
|
-
assert_equal
|
1066
|
-
There are 368 participants in this experiment.
|
1067
|
-
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1068
|
-
With 90% probability this result is statistically significant.
|
1069
|
-
Option B converted at 25.0%.
|
1070
|
-
Option A did not convert.
|
1071
|
-
Option C did not convert.
|
1072
|
-
Option D selected as the best alternative.
|
1104
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1105
|
+
There are 368 participants in this experiment.
|
1106
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
1107
|
+
With 90% probability this result is statistically significant.
|
1108
|
+
Option B converted at 25.0%.
|
1109
|
+
Option A did not convert.
|
1110
|
+
Option C did not convert.
|
1111
|
+
Option D selected as the best alternative.
|
1073
1112
|
TEXT
|
1074
1113
|
end
|
1075
1114
|
|
@@ -1079,15 +1118,15 @@ Option D selected as the best alternative.
|
|
1079
1118
|
default :a
|
1080
1119
|
metrics :coolness
|
1081
1120
|
end
|
1082
|
-
fake :abcd, :
|
1121
|
+
fake :abcd, b: [180, 58], d: [188, 61]
|
1083
1122
|
|
1084
|
-
assert_equal
|
1085
|
-
There are 368 participants in this experiment.
|
1086
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1087
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1088
|
-
Option B converted at 32.2%.
|
1089
|
-
Option A did not convert.
|
1090
|
-
Option C did not convert.
|
1123
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1124
|
+
There are 368 participants in this experiment.
|
1125
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1126
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1127
|
+
Option B converted at 32.2%.
|
1128
|
+
Option A did not convert.
|
1129
|
+
Option C did not convert.
|
1091
1130
|
TEXT
|
1092
1131
|
end
|
1093
1132
|
|
@@ -1097,15 +1136,15 @@ Option C did not convert.
|
|
1097
1136
|
default :a
|
1098
1137
|
metrics :coolness
|
1099
1138
|
end
|
1100
|
-
fake :abcd, :
|
1139
|
+
fake :abcd, b: [186, 60], d: [188, 61]
|
1101
1140
|
|
1102
|
-
assert_equal
|
1103
|
-
There are 374 participants in this experiment.
|
1104
|
-
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1105
|
-
This result is not statistically significant, suggest you continue this experiment.
|
1106
|
-
Option B converted at 32.3%.
|
1107
|
-
Option A did not convert.
|
1108
|
-
Option C did not convert.
|
1141
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1142
|
+
There are 374 participants in this experiment.
|
1143
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
1144
|
+
This result is not statistically significant, suggest you continue this experiment.
|
1145
|
+
Option B converted at 32.3%.
|
1146
|
+
Option A did not convert.
|
1147
|
+
Option C did not convert.
|
1109
1148
|
TEXT
|
1110
1149
|
end
|
1111
1150
|
|
@@ -1115,14 +1154,14 @@ Option C did not convert.
|
|
1115
1154
|
default :a
|
1116
1155
|
metrics :coolness
|
1117
1156
|
end
|
1118
|
-
fake :abcd, :
|
1157
|
+
fake :abcd, b: [188, 61], d: [188, 61]
|
1119
1158
|
|
1120
|
-
assert_equal
|
1121
|
-
There are 376 participants in this experiment.
|
1122
|
-
Option D converted at 32.4%.
|
1123
|
-
Option B converted at 32.4%.
|
1124
|
-
Option A did not convert.
|
1125
|
-
Option C did not convert.
|
1159
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1160
|
+
There are 376 participants in this experiment.
|
1161
|
+
Option D converted at 32.4%.
|
1162
|
+
Option B converted at 32.4%.
|
1163
|
+
Option A did not convert.
|
1164
|
+
Option C did not convert.
|
1126
1165
|
TEXT
|
1127
1166
|
end
|
1128
1167
|
|
@@ -1132,11 +1171,11 @@ Option C did not convert.
|
|
1132
1171
|
default :a
|
1133
1172
|
metrics :coolness
|
1134
1173
|
end
|
1135
|
-
fake :abcd, :
|
1174
|
+
fake :abcd, b: [180, 45]
|
1136
1175
|
|
1137
|
-
assert_equal
|
1138
|
-
There are 180 participants in this experiment.
|
1139
|
-
This experiment did not run long enough to find a clear winner.
|
1176
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1177
|
+
There are 180 participants in this experiment.
|
1178
|
+
This experiment did not run long enough to find a clear winner.
|
1140
1179
|
TEXT
|
1141
1180
|
end
|
1142
1181
|
|
@@ -1146,13 +1185,12 @@ This experiment did not run long enough to find a clear winner.
|
|
1146
1185
|
default :a
|
1147
1186
|
metrics :coolness
|
1148
1187
|
end
|
1149
|
-
assert_equal
|
1150
|
-
There are no participants in this experiment yet.
|
1151
|
-
This experiment did not run long enough to find a clear winner.
|
1188
|
+
assert_equal <<~TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
1189
|
+
There are no participants in this experiment yet.
|
1190
|
+
This experiment did not run long enough to find a clear winner.
|
1152
1191
|
TEXT
|
1153
1192
|
end
|
1154
1193
|
|
1155
|
-
|
1156
1194
|
# -- Completion --
|
1157
1195
|
|
1158
1196
|
def test_completion_if
|
@@ -1169,7 +1207,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1169
1207
|
def test_completion_if_fails
|
1170
1208
|
new_ab_test :simple do
|
1171
1209
|
identify { rand }
|
1172
|
-
complete_if {
|
1210
|
+
complete_if { raise "Testing complete_if raises exception" }
|
1173
1211
|
metrics :coolness
|
1174
1212
|
default false
|
1175
1213
|
end
|
@@ -1185,7 +1223,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1185
1223
|
metrics :coolness
|
1186
1224
|
default false
|
1187
1225
|
end
|
1188
|
-
99.times do |
|
1226
|
+
99.times do |_i|
|
1189
1227
|
experiment(:simple).choose
|
1190
1228
|
assert experiment(:simple).active?
|
1191
1229
|
end
|
@@ -1222,7 +1260,6 @@ This experiment did not run long enough to find a clear winner.
|
|
1222
1260
|
assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
|
1223
1261
|
end
|
1224
1262
|
|
1225
|
-
|
1226
1263
|
# -- Outcome --
|
1227
1264
|
|
1228
1265
|
def test_completion_outcome
|
@@ -1279,7 +1316,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1279
1316
|
|
1280
1317
|
def test_outcome_is_fails
|
1281
1318
|
new_ab_test :quick do
|
1282
|
-
outcome_is {
|
1319
|
+
outcome_is { raise "Testing outcome_is raising exception" }
|
1283
1320
|
metrics :coolness
|
1284
1321
|
default false
|
1285
1322
|
end
|
@@ -1292,7 +1329,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1292
1329
|
metrics :coolness
|
1293
1330
|
default false
|
1294
1331
|
end
|
1295
|
-
fake :quick, false=>[2,0], true=>10
|
1332
|
+
fake :quick, false => [2, 0], true => 10
|
1296
1333
|
experiment(:quick).complete!
|
1297
1334
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1298
1335
|
end
|
@@ -1302,7 +1339,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1302
1339
|
metrics :coolness
|
1303
1340
|
default false
|
1304
1341
|
end
|
1305
|
-
fake :quick, true=>2
|
1342
|
+
fake :quick, true => 2
|
1306
1343
|
experiment(:quick).complete!
|
1307
1344
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1308
1345
|
end
|
@@ -1312,12 +1349,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1312
1349
|
metrics :coolness
|
1313
1350
|
default false
|
1314
1351
|
end
|
1315
|
-
fake :quick, false=>8, true=>8
|
1352
|
+
fake :quick, false => 8, true => 8
|
1316
1353
|
experiment(:quick).complete!
|
1317
1354
|
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
1318
1355
|
end
|
1319
1356
|
|
1320
|
-
|
1321
1357
|
# -- No collection --
|
1322
1358
|
|
1323
1359
|
def test_no_collection_does_not_track
|
@@ -1360,7 +1396,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1360
1396
|
identify { "1" }
|
1361
1397
|
end
|
1362
1398
|
val = experiment(:simple).choose.value
|
1363
|
-
alternative = experiment(:simple).alternatives.detect {|a| a.value != val }
|
1399
|
+
alternative = experiment(:simple).alternatives.detect { |a| a.value != val }
|
1364
1400
|
experiment(:simple).chooses(alternative.value)
|
1365
1401
|
assert_equal experiment(:simple).choose.value, alternative.value
|
1366
1402
|
experiment(:simple).chooses(val)
|
@@ -1441,7 +1477,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1441
1477
|
default :b
|
1442
1478
|
metrics :coolness
|
1443
1479
|
|
1444
|
-
reject do |
|
1480
|
+
reject do |_request, _identity|
|
1445
1481
|
true
|
1446
1482
|
end
|
1447
1483
|
end
|
@@ -1468,11 +1504,11 @@ This experiment did not run long enough to find a clear winner.
|
|
1468
1504
|
end
|
1469
1505
|
|
1470
1506
|
def test_clears_outcome_and_completed_at
|
1471
|
-
|
1472
|
-
|
1473
|
-
|
1474
|
-
|
1475
|
-
|
1507
|
+
new_ab_test :simple do
|
1508
|
+
alternatives :a, :b, :c
|
1509
|
+
default :a
|
1510
|
+
metrics :coolness
|
1511
|
+
end
|
1476
1512
|
experiment(:simple).reset
|
1477
1513
|
assert_nil experiment(:simple).outcome
|
1478
1514
|
assert_nil experiment(:simple).completed_at
|
@@ -1491,7 +1527,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1491
1527
|
assert_not_nil experiment(:simple).completed_at
|
1492
1528
|
end
|
1493
1529
|
|
1494
|
-
def test_reset_clears_participants
|
1530
|
+
def test_reset_clears_participants # rubocop:todo Lint/DuplicateMethods
|
1495
1531
|
new_ab_test :simple do
|
1496
1532
|
alternatives :a, :b, :c
|
1497
1533
|
default :a
|
@@ -1503,7 +1539,7 @@ This experiment did not run long enough to find a clear winner.
|
|
1503
1539
|
assert_equal experiment(:simple).alternatives[1].participants, 0
|
1504
1540
|
end
|
1505
1541
|
|
1506
|
-
def test_clears_outcome_and_completed_at
|
1542
|
+
def test_clears_outcome_and_completed_at # rubocop:todo Lint/DuplicateMethods
|
1507
1543
|
new_ab_test :simple do
|
1508
1544
|
alternatives :a, :b, :c
|
1509
1545
|
default :a
|
@@ -1514,11 +1550,9 @@ This experiment did not run long enough to find a clear winner.
|
|
1514
1550
|
assert_nil experiment(:simple).completed_at
|
1515
1551
|
end
|
1516
1552
|
|
1517
|
-
|
1518
1553
|
# -- Helper methods --
|
1519
1554
|
|
1520
1555
|
def fake(name, args)
|
1521
1556
|
experiment(name).instance_eval { fake args }
|
1522
1557
|
end
|
1523
|
-
|
1524
1558
|
end
|