bananasplit 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +102 -0
- data/MIT-LICENSE +20 -0
- data/README.md +129 -0
- data/Rakefile +24 -0
- data/bananasplit.gemspec +20 -0
- data/generators/bananasplit_migration/bananasplit_migration_generator.rb +24 -0
- data/generators/bananasplit_migration/templates/abingo_migration.rb +31 -0
- data/lib/bananasplit.rb +325 -0
- data/lib/bananasplit/alternative.rb +22 -0
- data/lib/bananasplit/controller/dashboard.rb +25 -0
- data/lib/bananasplit/conversion_rate.rb +9 -0
- data/lib/bananasplit/experiment.rb +103 -0
- data/lib/bananasplit/rails/controller/dashboard.rb +13 -0
- data/lib/bananasplit/statistics.rb +90 -0
- data/lib/bananasplit/version.rb +3 -0
- data/lib/bananasplit/views/dashboard/_experiment.erb +43 -0
- data/lib/bananasplit/views/dashboard/index.erb +20 -0
- data/lib/bananasplit_sugar.rb +48 -0
- data/lib/bananasplit_view_helper.rb +40 -0
- data/test/abingo_test.rb +220 -0
- data/test/test_helper.rb +32 -0
- metadata +108 -0
@@ -0,0 +1,22 @@
|
|
1
|
+
require "active_record"
|
2
|
+
require_relative "conversion_rate"
|
3
|
+
class BananaSplit::Alternative < ActiveRecord::Base
|
4
|
+
include BananaSplit::ConversionRate
|
5
|
+
|
6
|
+
belongs_to :experiment, :class_name => "BananaSplit::Experiment"
|
7
|
+
attr_accessible :content, :weight, :lookup
|
8
|
+
serialize :content
|
9
|
+
|
10
|
+
def self.calculate_lookup(test_name, alternative_name)
|
11
|
+
Digest::MD5.hexdigest(BananaSplit.salt + test_name + alternative_name.to_s)
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.score_conversion(test_name, viewed_alternative)
|
15
|
+
self.update_all("conversions = conversions + 1", :lookup => self.calculate_lookup(test_name, viewed_alternative))
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.score_participation(test_name, viewed_alternative)
|
19
|
+
self.update_all("participants = participants + 1", :lookup => self.calculate_lookup(test_name, viewed_alternative))
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'action_controller'
|
2
|
+
class BananaSplit
|
3
|
+
module Controller
|
4
|
+
module Dashboard
|
5
|
+
ActionController::Base.prepend_view_path File.join(File.dirname(__FILE__), "../views")
|
6
|
+
|
7
|
+
def index
|
8
|
+
@experiments = BananaSplit::Experiment.all
|
9
|
+
render :template => 'dashboard/index'
|
10
|
+
end
|
11
|
+
|
12
|
+
def end_experiment
|
13
|
+
@alternative = BananaSplit::Alternative.find(params[:id])
|
14
|
+
@experiment = BananaSplit::Experiment.find(@alternative.experiment_id)
|
15
|
+
if (@experiment.status != "Completed")
|
16
|
+
@experiment.end_experiment!(@alternative.content)
|
17
|
+
flash[:notice] = "Experiment marked as ended. All users will now see the chosen alternative."
|
18
|
+
else
|
19
|
+
flash[:notice] = "This experiment is already ended."
|
20
|
+
end
|
21
|
+
redirect_to :action => "index"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require_relative "statistics"
|
2
|
+
require_relative "conversion_rate"
|
3
|
+
class BananaSplit::Experiment < ActiveRecord::Base
|
4
|
+
include BananaSplit::Statistics
|
5
|
+
include BananaSplit::ConversionRate
|
6
|
+
|
7
|
+
has_many :alternatives, :dependent => :destroy, :class_name => "BananaSplit::Alternative"
|
8
|
+
validates_uniqueness_of :test_name
|
9
|
+
attr_accessible :test_name
|
10
|
+
before_destroy :cleanup_cache
|
11
|
+
|
12
|
+
def cache_keys
|
13
|
+
["BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_"),
|
14
|
+
"BananaSplit::Experiment::#{test_name}::alternatives".gsub(" ","_"),
|
15
|
+
"BananaSplit::Experiment::short_circuit(#{test_name})".gsub(" ", "_")
|
16
|
+
]
|
17
|
+
end
|
18
|
+
|
19
|
+
def cleanup_cache
|
20
|
+
cache_keys.each do |key|
|
21
|
+
BananaSplit.cache.delete key
|
22
|
+
end
|
23
|
+
true
|
24
|
+
end
|
25
|
+
|
26
|
+
def participants
|
27
|
+
alternatives.sum("participants")
|
28
|
+
end
|
29
|
+
|
30
|
+
def conversions
|
31
|
+
alternatives.sum("conversions")
|
32
|
+
end
|
33
|
+
|
34
|
+
def best_alternative
|
35
|
+
alternatives.max do |a,b|
|
36
|
+
a.conversion_rate <=> b.conversion_rate
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.exists?(test_name)
|
41
|
+
cache_key = "BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_")
|
42
|
+
ret = BananaSplit.cache.fetch(cache_key) do
|
43
|
+
count = BananaSplit::Experiment.count(:conditions => {:test_name => test_name})
|
44
|
+
count > 0 ? count : nil
|
45
|
+
end
|
46
|
+
(!ret.nil?)
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.alternatives_for_test(test_name)
|
50
|
+
cache_key = "BananaSplit::#{test_name}::alternatives".gsub(" ","_")
|
51
|
+
BananaSplit.cache.fetch(cache_key) do
|
52
|
+
experiment = BananaSplit::Experiment.find_by_test_name(test_name)
|
53
|
+
alternatives_array = BananaSplit.cache.fetch(cache_key) do
|
54
|
+
tmp_array = experiment.alternatives.map do |alt|
|
55
|
+
[alt.content, alt.weight]
|
56
|
+
end
|
57
|
+
tmp_hash = tmp_array.inject({}) {|hash, couplet| hash[couplet[0]] = couplet[1]; hash}
|
58
|
+
BananaSplit.parse_alternatives(tmp_hash)
|
59
|
+
end
|
60
|
+
alternatives_array
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.start_experiment!(test_name, alternatives_array, conversion_name = nil)
|
65
|
+
conversion_name ||= test_name
|
66
|
+
conversion_name.gsub!(" ", "_")
|
67
|
+
cloned_alternatives_array = alternatives_array.clone
|
68
|
+
ActiveRecord::Base.transaction do
|
69
|
+
experiment = BananaSplit::Experiment.find_or_create_by_test_name(test_name)
|
70
|
+
experiment.alternatives.destroy_all #Blows away alternatives for pre-existing experiments.
|
71
|
+
while (cloned_alternatives_array.size > 0)
|
72
|
+
alt = cloned_alternatives_array[0]
|
73
|
+
weight = cloned_alternatives_array.size - (cloned_alternatives_array - [alt]).size
|
74
|
+
experiment.alternatives.build(:content => alt, :weight => weight,
|
75
|
+
:lookup => BananaSplit::Alternative.calculate_lookup(test_name, alt))
|
76
|
+
cloned_alternatives_array -= [alt]
|
77
|
+
end
|
78
|
+
experiment.status = "Live"
|
79
|
+
experiment.save(:validate => false)
|
80
|
+
BananaSplit.cache.write("BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_"), 1)
|
81
|
+
|
82
|
+
#This might have issues in very, very high concurrency environments...
|
83
|
+
|
84
|
+
tests_listening_to_conversion = BananaSplit.cache.read("BananaSplit::tests_listening_to_conversion#{conversion_name}") || []
|
85
|
+
tests_listening_to_conversion += [test_name] unless tests_listening_to_conversion.include? test_name
|
86
|
+
BananaSplit.cache.write("BananaSplit::tests_listening_to_conversion#{conversion_name}", tests_listening_to_conversion)
|
87
|
+
experiment
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def end_experiment!(final_alternative, conversion_name = nil)
|
92
|
+
conversion_name ||= test_name
|
93
|
+
ActiveRecord::Base.transaction do
|
94
|
+
alternatives.each do |alternative|
|
95
|
+
alternative.lookup = "Experiment completed. #{alternative.id}"
|
96
|
+
alternative.save!
|
97
|
+
end
|
98
|
+
update_attribute(:status, "Finished")
|
99
|
+
BananaSplit.cache.write("BananaSplit::Experiment::short_circuit(#{test_name})".gsub(" ", "_"), final_alternative)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
#Designed to be included into BananaSplit::Experiment, but you can feel free to adapt this
|
2
|
+
#to anything you want.
|
3
|
+
|
4
|
+
module BananaSplit::Statistics
|
5
|
+
|
6
|
+
HANDY_Z_SCORE_CHEATSHEET = [[0.10, 1.29], [0.05, 1.65], [0.01, 2.33], [0.001, 3.08]]
|
7
|
+
|
8
|
+
PERCENTAGES = {0.10 => '90%', 0.05 => '95%', 0.01 => '99%', 0.001 => '99.9%'}
|
9
|
+
|
10
|
+
DESCRIPTION_IN_WORDS = {0.10 => 'fairly confident', 0.05 => 'confident',
|
11
|
+
0.01 => 'very confident', 0.001 => 'extremely confident'}
|
12
|
+
def zscore
|
13
|
+
if alternatives.size != 2
|
14
|
+
raise "Sorry, can't currently automatically calculate statistics for A/B tests with > 2 alternatives."
|
15
|
+
end
|
16
|
+
|
17
|
+
if (alternatives[0].participants == 0) || (alternatives[1].participants == 0)
|
18
|
+
raise "Can't calculate the z score if either of the alternatives lacks participants."
|
19
|
+
end
|
20
|
+
|
21
|
+
cr1 = alternatives[0].conversion_rate
|
22
|
+
cr2 = alternatives[1].conversion_rate
|
23
|
+
|
24
|
+
n1 = alternatives[0].participants
|
25
|
+
n2 = alternatives[1].participants
|
26
|
+
|
27
|
+
numerator = cr1 - cr2
|
28
|
+
frac1 = cr1 * (1 - cr1) / n1
|
29
|
+
frac2 = cr2 * (1 - cr2) / n2
|
30
|
+
|
31
|
+
numerator / ((frac1 + frac2) ** 0.5)
|
32
|
+
end
|
33
|
+
|
34
|
+
def p_value
|
35
|
+
index = 0
|
36
|
+
z = zscore
|
37
|
+
z = z.abs
|
38
|
+
found_p = nil
|
39
|
+
while index < HANDY_Z_SCORE_CHEATSHEET.size do
|
40
|
+
if (z > HANDY_Z_SCORE_CHEATSHEET[index][1])
|
41
|
+
found_p = HANDY_Z_SCORE_CHEATSHEET[index][0]
|
42
|
+
end
|
43
|
+
index += 1
|
44
|
+
end
|
45
|
+
found_p
|
46
|
+
end
|
47
|
+
|
48
|
+
def is_statistically_significant?(p = 0.05)
|
49
|
+
p_value <= p
|
50
|
+
end
|
51
|
+
|
52
|
+
def pretty_conversion_rate
|
53
|
+
sprintf("%4.2f%%", conversion_rate * 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
def describe_result_in_words
|
57
|
+
begin
|
58
|
+
z = zscore
|
59
|
+
rescue
|
60
|
+
return "Could not execute the significance test because one or more of the alternatives has not been seen yet."
|
61
|
+
end
|
62
|
+
p = p_value
|
63
|
+
|
64
|
+
words = ""
|
65
|
+
if (alternatives[0].participants < 10) || (alternatives[1].participants < 10)
|
66
|
+
words += "Take these results with a grain of salt since your samples are so small: "
|
67
|
+
end
|
68
|
+
|
69
|
+
alts = alternatives - [best_alternative]
|
70
|
+
worst_alternative = alts.first
|
71
|
+
|
72
|
+
words += "The best alternative you have is: [#{best_alternative.content}], which had "
|
73
|
+
words += "#{best_alternative.conversions} conversions from #{best_alternative.participants} participants "
|
74
|
+
words += "(#{best_alternative.pretty_conversion_rate}). The other alternative was [#{worst_alternative.content}], "
|
75
|
+
words += "which had #{worst_alternative.conversions} conversions from #{worst_alternative.participants} participants "
|
76
|
+
words += "(#{worst_alternative.pretty_conversion_rate}). "
|
77
|
+
|
78
|
+
if (p.nil?)
|
79
|
+
words += "However, this difference is not statistically significant."
|
80
|
+
else
|
81
|
+
words += "This difference is #{PERCENTAGES[p]} likely to be statistically significant, which means you can be "
|
82
|
+
words += "#{DESCRIPTION_IN_WORDS[p]} that it is the result of your alternatives actually mattering, rather than "
|
83
|
+
words += "being due to random chance. However, this statistical test can't measure how likely the currently "
|
84
|
+
words += "observed magnitude of the difference is to be accurate or not. It only says \"better\", not \"better "
|
85
|
+
words += "by so much\"."
|
86
|
+
end
|
87
|
+
words
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
<h3><%= experiment.test_name.titleize %> <%= %Q|<i>(Test completed)</i>| if experiment.status != "Live" %> </h3>
|
2
|
+
<% short_circuit = BananaSplit.cache.read("BananaSplit::Experiment::short_circuit(#{experiment.test_name})".gsub(" ", "")) %>
|
3
|
+
<table class="experiment" style="border: 1px black;">
|
4
|
+
<tr class="header_row">
|
5
|
+
<th>Name</th>
|
6
|
+
<th>Participants</th>
|
7
|
+
<th>Conversions</th>
|
8
|
+
<th>Notes</th>
|
9
|
+
</tr>
|
10
|
+
<tr class="experiment_row">
|
11
|
+
<td>Experiment Total: </td>
|
12
|
+
<td><%= experiment.participants %> </td>
|
13
|
+
<td><%= experiment.conversions %> (<%= experiment.pretty_conversion_rate %>)</td>
|
14
|
+
<td></td>
|
15
|
+
</tr>
|
16
|
+
<% experiment.alternatives.each do |alternative| %>
|
17
|
+
<tr class="alternative_row">
|
18
|
+
<td>
|
19
|
+
<%= h alternative.content %>
|
20
|
+
</td>
|
21
|
+
<td><%= alternative.participants %></td>
|
22
|
+
<td><%= alternative.conversions %> (<%= alternative.pretty_conversion_rate %>)</td>
|
23
|
+
<td>
|
24
|
+
<% unless experiment.status != "Live" %>
|
25
|
+
<%= link_to("End experiment, picking this.", url_for(:id => alternative.id,
|
26
|
+
:action => "end_experiment"),
|
27
|
+
:method => :post,
|
28
|
+
:confirm => "Are you sure you want to terminate this experiment? This is not reversible."
|
29
|
+
) %>
|
30
|
+
<% else %>
|
31
|
+
<% if alternative.content == short_circuit %>
|
32
|
+
<b>(All users seeing this.)</b>
|
33
|
+
<% end %>
|
34
|
+
<% end %>
|
35
|
+
</td>
|
36
|
+
</tr>
|
37
|
+
<% end %>
|
38
|
+
<tr>
|
39
|
+
<td colspan="4">
|
40
|
+
<b>Significance test results: </b><%= experiment.describe_result_in_words %>
|
41
|
+
</td>
|
42
|
+
</tr>
|
43
|
+
</table>
|
@@ -0,0 +1,20 @@
|
|
1
|
+
<div id="abingo_dashboard">
|
2
|
+
<p><h1>Welcome to your A/Bingo dashboard!</h1>
|
3
|
+
|
4
|
+
See <a href="http://www.bingocardcreator.com/abingo">the official site</a> for documentation.
|
5
|
+
I encourage you to customize this page to fit your needs. See /vendor/plugins/abingo/views for
|
6
|
+
the view templates. Hack them to pieces -- please!
|
7
|
+
</p>
|
8
|
+
|
9
|
+
<p>
|
10
|
+
<% if flash[:notice] %>
|
11
|
+
<span style="color: green;"><%= flash[:notice] %> </span>
|
12
|
+
<% end %>
|
13
|
+
<h2>All Experiments</h2>
|
14
|
+
|
15
|
+
<% @experiments.each do |experiment| %>
|
16
|
+
<%= render :partial => "dashboard/experiment", :locals => {:experiment => experiment} %>
|
17
|
+
<br/>
|
18
|
+
<% end %>
|
19
|
+
</p>
|
20
|
+
</div>
|
@@ -0,0 +1,48 @@
|
|
1
|
+
#This module exists entirely to save finger strain for programmers.
|
2
|
+
#It is designed to be included in your ApplicationController.
|
3
|
+
#
|
4
|
+
#See abingo.rb for descriptions of what these do.
|
5
|
+
|
6
|
+
module BananaSplitSugar
|
7
|
+
|
8
|
+
def ab_test(abingo, test_name, alternatives = nil, options = {})
|
9
|
+
if (BananaSplit.options[:enable_specification] && !params[test_name].nil?)
|
10
|
+
choice = params[test_name]
|
11
|
+
elsif (BananaSplit.options[:enable_override_in_session] && !session[test_name].nil?)
|
12
|
+
choice = session[test_name]
|
13
|
+
elsif (alternatives.nil?)
|
14
|
+
choice = abingo.flip(test_name)
|
15
|
+
else
|
16
|
+
choice = abingo.test(test_name, alternatives, options)
|
17
|
+
end
|
18
|
+
|
19
|
+
if block_given?
|
20
|
+
yield(choice)
|
21
|
+
else
|
22
|
+
choice
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def bingo!(abingo, test_name, options = {})
|
27
|
+
abingo.bingo!(test_name, options)
|
28
|
+
end
|
29
|
+
|
30
|
+
#Mark the user as a human.
|
31
|
+
def abingo_mark_human(abingo)
|
32
|
+
textual_result = "1"
|
33
|
+
begin
|
34
|
+
a = params[:a].to_i
|
35
|
+
b = params[:b].to_i
|
36
|
+
c = params[:c].to_i
|
37
|
+
if (request.method == :post && (a + b == c))
|
38
|
+
abingo.human!
|
39
|
+
else
|
40
|
+
textual_result = "0"
|
41
|
+
end
|
42
|
+
rescue #If a bot doesn't pass a, b, or c, to_i will fail. This scarfs up the exception, to save it from polluting our logs.
|
43
|
+
textual_result = "0"
|
44
|
+
end
|
45
|
+
render :text => textual_result, :layout => false #Not actually used by browser
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#Gives you easy syntax to use ABingo in your views.
|
2
|
+
|
3
|
+
module BananaSplitViewHelper
|
4
|
+
|
5
|
+
def ab_test(abingo, test_name, alternatives = nil, options = {}, &block)
|
6
|
+
|
7
|
+
if (BananaSplit.options[:enable_specification] && !params[test_name].nil?)
|
8
|
+
choice = params[test_name]
|
9
|
+
elsif (BananaSplit.options[:enable_override_in_session] && !session[test_name].nil?)
|
10
|
+
choice = session[test_name]
|
11
|
+
elsif (alternatives.nil?)
|
12
|
+
choice = abingo.flip(test_name)
|
13
|
+
else
|
14
|
+
choice = abingo.test(test_name, alternatives, options)
|
15
|
+
end
|
16
|
+
|
17
|
+
if block
|
18
|
+
content_tag = capture(choice, &block)
|
19
|
+
else
|
20
|
+
choice
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def bingo!(abingo, test_name, options = {})
|
25
|
+
abingo.bingo!(test_name, options)
|
26
|
+
end
|
27
|
+
|
28
|
+
#This causes an AJAX post against the URL. That URL should call BananaSplit.human!
|
29
|
+
#This guarantees that anyone calling BananaSplit.human! is capable of at least minimal Javascript execution, and thus is (probably) not a robot.
|
30
|
+
def include_humanizing_javascript(url = "/abingo_mark_human", style = :prototype)
|
31
|
+
script = nil
|
32
|
+
if (style == :prototype)
|
33
|
+
script = "var a=Math.floor(Math.random()*11); var b=Math.floor(Math.random()*11);var x=new Ajax.Request('#{url}', {parameters:{a: a, b: b, c: a+b}})"
|
34
|
+
elsif (style == :jquery)
|
35
|
+
script = "var a=Math.floor(Math.random()*11); var b=Math.floor(Math.random()*11);var x=jQuery.post('#{url}', {a: a, b: b, c: a+b})"
|
36
|
+
end
|
37
|
+
script.nil? ? "" : %Q|<script type="text/javascript">#{script}</script>|
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
data/test/abingo_test.rb
ADDED
@@ -0,0 +1,220 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
class BananaSplitTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
setup do
|
6
|
+
BananaSplit.options = {}
|
7
|
+
end
|
8
|
+
|
9
|
+
teardown do
|
10
|
+
BananaSplit.cache.clear
|
11
|
+
BananaSplit::Experiment.delete_all
|
12
|
+
BananaSplit::Alternative.delete_all
|
13
|
+
end
|
14
|
+
|
15
|
+
test "identity automatically assigned" do
|
16
|
+
split = BananaSplit.identify
|
17
|
+
assert split.identity != nil
|
18
|
+
end
|
19
|
+
|
20
|
+
test "alternative parsing" do
|
21
|
+
array = %w{a b c}
|
22
|
+
assert_equal array, BananaSplit.parse_alternatives(array)
|
23
|
+
assert_equal 65, BananaSplit.parse_alternatives(65).size
|
24
|
+
assert_equal 4, BananaSplit.parse_alternatives(2..5).size
|
25
|
+
assert !(BananaSplit.parse_alternatives(2..5).include? 1)
|
26
|
+
end
|
27
|
+
|
28
|
+
test "experiment creation" do
|
29
|
+
assert_equal 0, BananaSplit::Experiment.count
|
30
|
+
assert_equal 0, BananaSplit::Alternative.count
|
31
|
+
alternatives = %w{A B}
|
32
|
+
split = BananaSplit.identify
|
33
|
+
alternative_selected = split.test("unit_test_sample_A", alternatives)
|
34
|
+
assert_equal 1, BananaSplit::Experiment.count
|
35
|
+
assert_equal 2, BananaSplit::Alternative.count
|
36
|
+
assert alternatives.include?(alternative_selected)
|
37
|
+
end
|
38
|
+
|
39
|
+
test "exists works right" do
|
40
|
+
split = BananaSplit.identify
|
41
|
+
split.test("exist works right", %w{does does_not})
|
42
|
+
assert BananaSplit::Experiment.exists?("exist works right")
|
43
|
+
end
|
44
|
+
|
45
|
+
test "alternatives picked consistently" do
|
46
|
+
split = BananaSplit.identify
|
47
|
+
alternative_picked = split.test("consistency_test", 1..100)
|
48
|
+
100.times do
|
49
|
+
assert_equal alternative_picked, split.test("consistency_test", 1..100)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
test "participation works" do
|
54
|
+
new_tests = %w{participationA participationB participationC}
|
55
|
+
split = BananaSplit.identify
|
56
|
+
new_tests.map do |test_name|
|
57
|
+
split.test(test_name, 1..5)
|
58
|
+
end
|
59
|
+
|
60
|
+
participating_tests = BananaSplit.cache.read("BananaSplit::participating_tests::#{split.identity}") || []
|
61
|
+
|
62
|
+
new_tests.map do |test_name|
|
63
|
+
assert participating_tests.include? test_name
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
test "participants counted" do
|
68
|
+
test_name = "participants_counted_test"
|
69
|
+
split = BananaSplit.identify
|
70
|
+
alternative = split.test(test_name, %w{a b c})
|
71
|
+
|
72
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
73
|
+
lookup = BananaSplit::Alternative.calculate_lookup(test_name, alternative)
|
74
|
+
chosen_alt = BananaSplit::Alternative.find_by_lookup(lookup)
|
75
|
+
assert_equal 1, ex.participants
|
76
|
+
assert_equal 1, chosen_alt.participants
|
77
|
+
end
|
78
|
+
|
79
|
+
test "conversion tracking by test name" do
|
80
|
+
test_name = "conversion_test_by_name"
|
81
|
+
split = BananaSplit.identify
|
82
|
+
alternative = split.test(test_name, %w{a b c})
|
83
|
+
split.bingo!(test_name)
|
84
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
85
|
+
lookup = BananaSplit::Alternative.calculate_lookup(test_name, alternative)
|
86
|
+
chosen_alt = BananaSplit::Alternative.find_by_lookup(lookup)
|
87
|
+
assert_equal 1, ex.conversions
|
88
|
+
assert_equal 1, chosen_alt.conversions
|
89
|
+
split.bingo!(test_name)
|
90
|
+
|
91
|
+
#Should still only have one because this conversion should not be double counted.
|
92
|
+
#We haven't specified that in the test options.
|
93
|
+
assert_equal 1, BananaSplit::Experiment.find_by_test_name(test_name).conversions
|
94
|
+
end
|
95
|
+
|
96
|
+
test "conversion tracking by conversion name" do
|
97
|
+
split = BananaSplit.identify
|
98
|
+
conversion_name = "purchase"
|
99
|
+
tests = %w{conversionTrackingByConversionNameA conversionTrackingByConversionNameB conversionTrackingByConversionNameC}
|
100
|
+
tests.map do |test_name|
|
101
|
+
split.test(test_name, %w{A B}, :conversion => conversion_name)
|
102
|
+
end
|
103
|
+
|
104
|
+
split.bingo!(conversion_name)
|
105
|
+
tests.map do |test_name|
|
106
|
+
assert_equal 1, BananaSplit::Experiment.find_by_test_name(test_name).conversions
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
test "short circuiting works" do
|
111
|
+
conversion_name = "purchase"
|
112
|
+
test_name = "short circuit test"
|
113
|
+
split = BananaSplit.identify
|
114
|
+
alt_picked = split.test(test_name, %w{A B}, :conversion => conversion_name)
|
115
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
116
|
+
alt_not_picked = (%w{A B} - [alt_picked]).first
|
117
|
+
|
118
|
+
ex.end_experiment!(alt_not_picked, conversion_name)
|
119
|
+
|
120
|
+
ex.reload
|
121
|
+
assert_equal "Finished", ex.status
|
122
|
+
|
123
|
+
split.bingo!(test_name) #Should not be counted, test is over.
|
124
|
+
assert_equal 0, ex.conversions
|
125
|
+
|
126
|
+
new_bingo = BananaSplit.identify("shortCircuitTestNewIdentity")
|
127
|
+
new_bingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
128
|
+
ex.reload
|
129
|
+
assert_equal 1, ex.participants #Original identity counted, new identity not counted b/c test stopped
|
130
|
+
end
|
131
|
+
|
132
|
+
test "proper experiment creation in high concurrency" do
|
133
|
+
conversion_name = "purchase"
|
134
|
+
test_name = "high_concurrency_test"
|
135
|
+
alternatives = %w{foo bar}
|
136
|
+
|
137
|
+
threads = []
|
138
|
+
5.times do
|
139
|
+
threads << Thread.new do
|
140
|
+
split = BananaSplit.identify
|
141
|
+
split.test(test_name, alternatives, :conversion => conversion_name)
|
142
|
+
ActiveRecord::Base.connection.close
|
143
|
+
end
|
144
|
+
end
|
145
|
+
threads.each(&:join)
|
146
|
+
assert_equal 1, BananaSplit::Experiment.count_by_sql(["select count(id) from experiments where test_name = ?", test_name])
|
147
|
+
end
|
148
|
+
|
149
|
+
test "proper conversions with concurrency" do
|
150
|
+
test_name = "conversion_concurrency_test"
|
151
|
+
alternatives = %w{foo bar}
|
152
|
+
threads = []
|
153
|
+
alternatives.size.times do |i|
|
154
|
+
threads << Thread.new do
|
155
|
+
split = BananaSplit.identify(i)
|
156
|
+
split.test(test_name, alternatives)
|
157
|
+
split.bingo!(test_name)
|
158
|
+
sleep(0.3) if i == 0
|
159
|
+
ActiveRecord::Base.connection.close
|
160
|
+
end
|
161
|
+
end
|
162
|
+
threads.each(&:join)
|
163
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
164
|
+
ex.alternatives.each do |alternative|
|
165
|
+
assert_equal 1, alternative.conversions
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
test "non-humans are ignored for participation and conversions if not explicitly counted" do
|
170
|
+
BananaSplit.options[:count_humans_only] = true
|
171
|
+
BananaSplit.options[:expires_in] = 1.hour
|
172
|
+
BananaSplit.options[:expires_in_for_bots] = 3.seconds
|
173
|
+
first_identity = BananaSplit.identify("unsure_if_human#{Time.now.to_i}")
|
174
|
+
test_name = "are_you_a_human"
|
175
|
+
first_identity.test(test_name, %w{does_not matter})
|
176
|
+
|
177
|
+
assert !first_identity.is_human?, "Identity not marked as human yet."
|
178
|
+
|
179
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
180
|
+
first_identity.bingo!(test_name)
|
181
|
+
assert_equal 0, ex.participants, "Not human yet, so should have no participants."
|
182
|
+
assert_equal 0, ex.conversions, "Not human yet, so should have no conversions."
|
183
|
+
|
184
|
+
first_identity.human!
|
185
|
+
|
186
|
+
#Setting up second participant who doesn't convert.
|
187
|
+
second_identity = BananaSplit.identify("unsure_if_human_2_#{Time.now.to_i}")
|
188
|
+
second_identity.test(test_name, %w{does_not matter})
|
189
|
+
second_identity.human!
|
190
|
+
|
191
|
+
ex = BananaSplit::Experiment.find_by_test_name(test_name)
|
192
|
+
assert_equal 2, ex.participants, "Now that we're human, our participation should matter."
|
193
|
+
assert_equal 1, ex.conversions, "Now that we're human, our conversions should matter, but only one of us converted."
|
194
|
+
end
|
195
|
+
|
196
|
+
test "Participating tests for a given identity" do
|
197
|
+
split = BananaSplit.identify("test_participant")
|
198
|
+
test_names = (1..3).map {|t| "participating_test_test_name #{t}"}
|
199
|
+
test_alternatives = %w{yes no}
|
200
|
+
test_names.each {|test_name| split.test(test_name, test_alternatives)}
|
201
|
+
ex = BananaSplit::Experiment.last
|
202
|
+
ex.end_experiment!("no") #End final of 3 tests, leaving 2 presently running
|
203
|
+
|
204
|
+
assert_equal 2, split.participating_tests.size #Pairs for two tests
|
205
|
+
split.participating_tests.each do |key, value|
|
206
|
+
assert test_names.include? key
|
207
|
+
assert test_alternatives.include? value
|
208
|
+
end
|
209
|
+
|
210
|
+
assert_equal 3, split.participating_tests(false).size #pairs for three tests
|
211
|
+
split.participating_tests(false).each do |key, value|
|
212
|
+
assert test_names.include? key
|
213
|
+
assert test_alternatives.include? value
|
214
|
+
end
|
215
|
+
|
216
|
+
non_participant = BananaSplit.identify("test_nonparticipant")
|
217
|
+
assert_equal({}, non_participant.participating_tests)
|
218
|
+
end
|
219
|
+
|
220
|
+
end
|