vanity 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +14 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +8 -109
- data/bin/vanity +12 -5
- data/lib/vanity.rb +5 -5
- data/lib/vanity/commands.rb +1 -1
- data/lib/vanity/commands/report.rb +20 -4
- data/lib/vanity/experiment/ab_test.rb +151 -132
- data/lib/vanity/experiment/base.rb +5 -5
- data/lib/vanity/playground.rb +24 -16
- data/lib/vanity/rails.rb +4 -3
- data/lib/vanity/rails/console.rb +14 -0
- data/lib/vanity/rails/helpers.rb +3 -1
- data/lib/vanity/rails/testing.rb +1 -1
- data/lib/vanity/templates/_ab_test.erb +25 -0
- data/lib/vanity/templates/_experiments.erb +12 -0
- data/lib/vanity/templates/_report.erb +16 -0
- data/lib/vanity/templates/_vanity.css +13 -0
- data/test/ab_test_test.rb +119 -134
- data/test/experiments/age_and_zipcode.rb +3 -0
- data/test/experiments/null_abc.rb +2 -2
- data/test/test_helper.rb +1 -3
- data/vanity.gemspec +5 -5
- metadata +13 -7
- data/lib/vanity/report.erb +0 -22
data/CHANGELOG
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
0.3.0 (2009-11-13)
|
2
|
+
* Added: score now includes least performing alternatives, names and values.
|
3
|
+
* Added: shiny reports.
|
4
|
+
* Added: Rails console shows current experiments status and also allows you to
|
5
|
+
choose which alternative you want to see.
|
6
|
+
* Changed: letters instead of numbers for options (option 1 => option A).
|
7
|
+
* Changed: experiment.alternatives is now an immutable snapshot.
|
8
|
+
* Changed: experiment.score returns populated alternative objects instead of
|
9
|
+
structs.
|
10
|
+
* Changed: experiment.chooses uses Redis to store state, better for (when we
|
11
|
+
get to) browser integration.
|
12
|
+
* Changed: experiment.chooses skips recording participant or conversion.
|
13
|
+
* Changed: to MIT license.
|
14
|
+
|
1
15
|
0.2.2 (2009-11-12)
|
2
16
|
* Added: vanity binary, with single command for generating a report.
|
3
17
|
* Added: return alternative by value from experiment.alternative(val) method.
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2009 Assaf Arkin
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README.rdoc
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
Vanity is an
|
1
|
+
Vanity is an Experiment Driven Development framework for Rails.
|
2
2
|
|
3
|
-
|
3
|
+
http://farm3.static.flickr.com/2540/4099665871_497f274f68_o.jpg
|
4
|
+
|
5
|
+
Requires Ruby 1.9.1 or later, Redis 1.0 or later.
|
4
6
|
|
5
7
|
|
6
8
|
== A/B Testing with Rails (in 5 easy steps)
|
@@ -38,116 +40,13 @@ Measure conversion:
|
|
38
40
|
|
39
41
|
Check the report:
|
40
42
|
|
41
|
-
vanity
|
42
|
-
|
43
|
-
|
44
|
-
== A/B Tests
|
45
|
-
|
46
|
-
Each A/B experiment represents several (two or more) alternatives. Use the
|
47
|
-
ab_test method to choose an alternative. Call ab_test without a block to return
|
48
|
-
the value of the chosen alternative. Call ab_test with a block to yield with
|
49
|
-
the value.
|
50
|
-
|
51
|
-
Here are some examples:
|
52
|
-
|
53
|
-
def index
|
54
|
-
if ab_test(:new_page) # classic false/true test
|
55
|
-
render action: "new_page"
|
56
|
-
else
|
57
|
-
render action: "index"
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def index
|
62
|
-
# alternatives are names of templates
|
63
|
-
render template: ab_test(:new_page)
|
64
|
-
end
|
65
|
-
|
66
|
-
<%= if ab_test(:banner) %>100% less complexity!<% end %>
|
67
|
-
|
68
|
-
<%= ab_test(:greeting) %> <%= current_user.name %>
|
69
|
-
|
70
|
-
<% ab_test :features do |count| %>
|
71
|
-
<%= count %> features to choose from!
|
72
|
-
<% end %>
|
73
|
-
|
74
|
-
To measure conversion, call ab_goal! with the experiment's name. Typically,
|
75
|
-
you would do that from a controller action, for example:
|
76
|
-
|
77
|
-
def create
|
78
|
-
ab_goal! :new_page
|
79
|
-
...
|
80
|
-
end
|
81
|
-
|
82
|
-
To measure conversion, simply call ab_goal! with the experiment name. From the
|
83
|
-
Vanity identity set by the filter we know which alternative was presented by
|
84
|
-
ab_test, and can correlate conversions to alternative. It's that simple!
|
85
|
-
|
86
|
-
|
87
|
-
== Managing Identity
|
88
|
-
|
89
|
-
For effective A/B tests, you want to:
|
90
|
-
- Randomly show different alternatives to different people
|
91
|
-
- Consistently show the same alternatives to the same person
|
92
|
-
- Know which alternative caused a conversion
|
93
|
-
- When running multiple tests at once, keep them independent
|
94
|
-
|
95
|
-
If you don't use any other mechanism, Vanity will assign a random value to a
|
96
|
-
persistent cookie and use it to track the same visitor on subsequent visits.
|
97
|
-
Cookie tracking is enabled by use_vanity.
|
98
|
-
|
99
|
-
If you keep track of users, you would want to use the user's identity instead.
|
100
|
-
Using user identity is more reliable than a cookie tied to a single Web
|
101
|
-
browser.
|
43
|
+
vanity --output vanity.html
|
102
44
|
|
103
|
-
|
104
|
-
with the desired id attribute. Alternatively, you can use a proc. These two
|
105
|
-
examples are equivalent:
|
106
|
-
|
107
|
-
use_vanity :current_user
|
108
|
-
use_vanity { |controller| controller.current_user.id }
|
109
|
-
|
110
|
-
There are times when you would want to use a different identity to distinguish
|
111
|
-
test alternatives. For example, your application may have groups and you may
|
112
|
-
want to A/B test an option that will be available (or not) to all people in the
|
113
|
-
same group.
|
114
|
-
|
115
|
-
You can tell Vanity to use a different identity on a particular controller
|
116
|
-
using use_vanity. Alternatively, you can configure the experiment to extract
|
117
|
-
the identity. The following example will apply to all controllers that have a
|
118
|
-
project attribute (without affecting other experiments):
|
119
|
-
|
120
|
-
example "New feature" do
|
121
|
-
description "New feature only available to some groups"
|
122
|
-
identify { |controller| controller.project.id }
|
123
|
-
end
|
124
|
-
|
125
|
-
|
126
|
-
== Configuring Vanity
|
127
|
-
|
128
|
-
Vanity will work out of the box on a default configuration. Assuming you're
|
129
|
-
using Redis on localhost, post 6379, there's nothing special to do.
|
130
|
-
|
131
|
-
If you run a different setup, use the playground object to configure Vanity.
|
132
|
-
For example:
|
133
|
-
|
134
|
-
Vanity.playground.host = "redis.local"
|
135
|
-
Vanity.playground.password = "supersecret"
|
45
|
+
Learn more about Vanity: http://assaf.github.com/vanity
|
136
46
|
|
137
47
|
|
138
48
|
== Credits
|
139
49
|
|
140
|
-
|
141
|
-
Apartly, there was coffee involved and out came the idea for Vanity.
|
142
|
-
|
143
|
-
First experiment, A/B tests, heavily influenced by Patrick McKenzie's awesome
|
144
|
-
A/Bingo (http://www.bingocardcreator.com/abingo)
|
145
|
-
|
146
|
-
Pain points courtesy of Google Analytics's stylish graphs and too-many-clicks
|
147
|
-
goal tracking process.
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
== License
|
50
|
+
Experiment Driven Development: Nathaniel Talbott (http://blog.talbott.ws).
|
152
51
|
|
153
|
-
|
52
|
+
Copyright (C) 2009 Assaf Arkin, released under the MIT license.
|
data/bin/vanity
CHANGED
@@ -7,8 +7,10 @@ require "optparse"
|
|
7
7
|
|
8
8
|
playground = Vanity.playground
|
9
9
|
options = Struct.new(:output).new
|
10
|
-
OptionParser.new("", 24, " ") do |opts|
|
11
|
-
opts.banner = "Usage: #{File.basename($0)} [options]\n"
|
10
|
+
opts = OptionParser.new("", 24, " ") do |opts|
|
11
|
+
opts.banner = "Usage: #{File.basename($0)} [options] command\n"
|
12
|
+
opts.banner << "Commands:\n"
|
13
|
+
opts.banner << " report Report on all running experiments"
|
12
14
|
|
13
15
|
opts.separator ""
|
14
16
|
opts.separator "General options:"
|
@@ -33,10 +35,15 @@ OptionParser.new("", 24, " ") do |opts|
|
|
33
35
|
puts "Vanity #{Vanity::Version::STRING}"
|
34
36
|
exit
|
35
37
|
end
|
36
|
-
end
|
38
|
+
end
|
39
|
+
|
40
|
+
opts.parse!(ARGV)
|
41
|
+
if ARGV.empty?
|
42
|
+
puts opts.banner
|
43
|
+
exit
|
44
|
+
end
|
37
45
|
|
38
|
-
|
39
|
-
cmds.each do |cmd|
|
46
|
+
ARGV.each do |cmd|
|
40
47
|
case cmd
|
41
48
|
when "report"
|
42
49
|
Vanity::Commands.report options.output
|
data/lib/vanity.rb
CHANGED
@@ -17,8 +17,8 @@ module Vanity
|
|
17
17
|
end
|
18
18
|
|
19
19
|
|
20
|
-
require
|
21
|
-
require
|
22
|
-
require
|
23
|
-
require
|
24
|
-
Vanity.autoload :Commands,
|
20
|
+
require "vanity/playground"
|
21
|
+
require "vanity/experiment/base"
|
22
|
+
require "vanity/experiment/ab_test"
|
23
|
+
require "vanity/rails" if defined?(Rails)
|
24
|
+
Vanity.autoload :Commands, "vanity/commands"
|
data/lib/vanity/commands.rb
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require "vanity/commands/report"
|
@@ -2,16 +2,32 @@ require "erb"
|
|
2
2
|
require "cgi"
|
3
3
|
|
4
4
|
module Vanity
|
5
|
+
|
6
|
+
# Render method available in templates (when running outside Rails).
|
7
|
+
module Render
|
8
|
+
|
9
|
+
# Render the named template. Used for reporting and the console.
|
10
|
+
def render(path, locals = {})
|
11
|
+
locals[:playground] = self
|
12
|
+
keys = locals.keys
|
13
|
+
struct = Struct.new(*keys)
|
14
|
+
struct.send :include, Render
|
15
|
+
locals = struct.new(*locals.values_at(*keys))
|
16
|
+
dir, base = File.split(path)
|
17
|
+
path = File.read(File.join(dir, "_#{base}"))
|
18
|
+
ERB.new(path, nil, '<').result(locals.instance_eval { binding })
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
5
23
|
module Commands
|
6
24
|
class << self
|
25
|
+
include Render
|
7
26
|
|
8
27
|
# Generate a report with all available tests. Outputs to the named file,
|
9
28
|
# or stdout with no arguments.
|
10
29
|
def report(output = nil)
|
11
|
-
|
12
|
-
erb = ERB.new(File.read("lib/vanity/report.erb"), nil, '<')
|
13
|
-
experiments = Vanity.playground.experiments
|
14
|
-
html = erb.result(binding)
|
30
|
+
html = render(Vanity.template("report"))
|
15
31
|
if output
|
16
32
|
File.open output, 'w' do |file|
|
17
33
|
file.write html
|
@@ -1,14 +1,15 @@
|
|
1
1
|
module Vanity
|
2
2
|
module Experiment
|
3
3
|
|
4
|
-
# Experiment alternative. See AbTest#alternatives.
|
4
|
+
# Experiment alternative. See AbTest#alternatives and AbTest#score.
|
5
5
|
class Alternative
|
6
6
|
|
7
|
-
def initialize(experiment, id, value) #:nodoc:
|
7
|
+
def initialize(experiment, id, value, participants, converted, conversions) #:nodoc:
|
8
8
|
@experiment = experiment
|
9
9
|
@id = id
|
10
|
-
@name = "option #{(@id +
|
10
|
+
@name = "option #{(@id + 65).chr}"
|
11
11
|
@value = value
|
12
|
+
@participants, @converted, @conversions = participants, converted, conversions
|
12
13
|
end
|
13
14
|
|
14
15
|
# Alternative id, only unique for this experiment.
|
@@ -20,46 +21,38 @@ module Vanity
|
|
20
21
|
# Alternative value.
|
21
22
|
attr_reader :value
|
22
23
|
|
24
|
+
# Experiment this alternative belongs to.
|
25
|
+
attr_reader :experiment
|
26
|
+
|
23
27
|
# Number of participants who viewed this alternative.
|
24
|
-
|
25
|
-
redis.scard(key("participants")).to_i
|
26
|
-
end
|
28
|
+
attr_reader :participants
|
27
29
|
|
28
30
|
# Number of participants who converted on this alternative.
|
29
|
-
|
30
|
-
redis.scard(key("converted")).to_i
|
31
|
-
end
|
31
|
+
attr_reader :converted
|
32
32
|
|
33
33
|
# Number of conversions for this alternative (same participant may be counted more than once).
|
34
|
-
|
35
|
-
redis[key("conversions")].to_i
|
36
|
-
end
|
34
|
+
attr_reader :conversions
|
37
35
|
|
38
|
-
#
|
39
|
-
|
40
|
-
c, p = converted.to_f, participants.to_f
|
41
|
-
p > 0 ? c/p : 0.0
|
42
|
-
end
|
36
|
+
# Z-score for this alternative. Populated by AbTest#score.
|
37
|
+
attr_accessor :z_score
|
43
38
|
|
44
|
-
|
45
|
-
|
46
|
-
|
39
|
+
# Confidence derived from z-score. Populated by AbTest#score.
|
40
|
+
attr_accessor :confidence
|
41
|
+
|
42
|
+
# Difference from least performing alternative. Populated by AbTest#score.
|
43
|
+
attr_accessor :difference
|
47
44
|
|
48
|
-
|
49
|
-
|
45
|
+
# Conversion rate calculated as converted/participants, rounded to 3 places.
|
46
|
+
def conversion_rate
|
47
|
+
@rate ||= (participants > 0 ? (converted.to_f/participants.to_f).round(3) : 0.0)
|
50
48
|
end
|
51
49
|
|
52
|
-
def
|
53
|
-
|
54
|
-
redis.sadd key("converted"), identity
|
55
|
-
redis.incr key("conversions")
|
56
|
-
end
|
50
|
+
def <=>(other) # sort by conversion rate
|
51
|
+
conversion_rate <=> other.conversion_rate
|
57
52
|
end
|
58
53
|
|
59
|
-
def
|
60
|
-
|
61
|
-
redis.del key("converted")
|
62
|
-
redis.del key("conversions")
|
54
|
+
def ==(other)
|
55
|
+
other && id == other.id && experiment == other.experiment
|
63
56
|
end
|
64
57
|
|
65
58
|
def to_s #:nodoc:
|
@@ -70,20 +63,6 @@ module Vanity
|
|
70
63
|
"#{name}: #{value} #{converted}/#{participants}"
|
71
64
|
end
|
72
65
|
|
73
|
-
protected
|
74
|
-
|
75
|
-
def key(name)
|
76
|
-
@experiment.key("alts:#{id}:#{name}")
|
77
|
-
end
|
78
|
-
|
79
|
-
def redis
|
80
|
-
@experiment.redis
|
81
|
-
end
|
82
|
-
|
83
|
-
def base
|
84
|
-
@base ||= @experiment.alternatives.first
|
85
|
-
end
|
86
|
-
|
87
66
|
end
|
88
67
|
|
89
68
|
|
@@ -96,39 +75,60 @@ module Vanity
|
|
96
75
|
confidence = AbTest::Z_TO_CONFIDENCE.find { |z,p| score >= z }
|
97
76
|
confidence ? confidence.last : 0
|
98
77
|
end
|
78
|
+
|
79
|
+
def friendly_name
|
80
|
+
"A/B Test"
|
81
|
+
end
|
82
|
+
|
99
83
|
end
|
100
84
|
|
101
85
|
def initialize(*args) #:nodoc:
|
102
86
|
super
|
87
|
+
@alternatives = [false, true]
|
103
88
|
end
|
104
89
|
|
105
90
|
# -- Alternatives --
|
106
91
|
|
107
|
-
# Call this method once to
|
108
|
-
#
|
109
|
-
#
|
110
|
-
# Call without argument to previously defined alternatives (see Alternative).
|
111
|
-
#
|
112
|
-
# For example:
|
92
|
+
# Call this method once to set alternative values for this experiment.
|
93
|
+
# Require at least two values. For example:
|
113
94
|
# experiment "Background color" do
|
114
95
|
# alternatives "red", "blue", "orange"
|
115
96
|
# end
|
116
|
-
#
|
97
|
+
#
|
98
|
+
# Call without arguments to obtain current list of alternatives. For example:
|
117
99
|
# alts = experiment(:background_color).alternatives
|
118
100
|
# puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
|
101
|
+
#
|
102
|
+
# If you want to know how well each alternative is faring, use #score.
|
119
103
|
def alternatives(*args)
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
104
|
+
unless args.empty?
|
105
|
+
@alternatives = args.clone
|
106
|
+
end
|
107
|
+
class << self
|
108
|
+
alias :alternatives :_alternatives
|
124
109
|
end
|
125
|
-
class << self ; self ; end.send(:define_method, :alternatives) { @alternatives }
|
126
110
|
alternatives
|
127
111
|
end
|
128
112
|
|
113
|
+
def _alternatives #:nodoc:
|
114
|
+
alts = []
|
115
|
+
@alternatives.each_with_index do |value, i|
|
116
|
+
participants = redis.scard(key("alts:#{i}:participants")).to_i
|
117
|
+
converted = redis.scard(key("alts:#{i}:converted")).to_i
|
118
|
+
conversions = redis[key("alts:#{i}:conversions")].to_i
|
119
|
+
alts << Alternative.new(self, i, value, participants, converted, conversions)
|
120
|
+
end
|
121
|
+
alts
|
122
|
+
end
|
123
|
+
|
129
124
|
# Returns an Alternative with the specified value.
|
130
125
|
def alternative(value)
|
131
|
-
|
126
|
+
if index = @alternatives.index(value)
|
127
|
+
participants = redis.scard(key("alts:#{index}:participants")).to_i
|
128
|
+
converted = redis.scard(key("alts:#{index}:converted")).to_i
|
129
|
+
conversions = redis[key("alts:#{index}:conversions")].to_i
|
130
|
+
Alternative.new(self, index, value, participants, converted, conversions)
|
131
|
+
end
|
132
132
|
end
|
133
133
|
|
134
134
|
# Sets this test to two alternatives: false and true.
|
@@ -148,15 +148,16 @@ module Vanity
|
|
148
148
|
def choose
|
149
149
|
if active?
|
150
150
|
identity = identify
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
151
|
+
index = redis[key("participant:#{identity}:show")]
|
152
|
+
unless index
|
153
|
+
index = alternative_for(identity)
|
154
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
155
|
+
check_completion!
|
156
|
+
end
|
157
157
|
else
|
158
|
-
|
158
|
+
index = redis[key("outcome")] || alternative_for(identify)
|
159
159
|
end
|
160
|
+
@alternatives[index.to_i]
|
160
161
|
end
|
161
162
|
|
162
163
|
# Records a conversion.
|
@@ -164,12 +165,15 @@ module Vanity
|
|
164
165
|
# For example:
|
165
166
|
# experiment(:which_blue).conversion!
|
166
167
|
def conversion!
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
168
|
+
return unless active?
|
169
|
+
identity = identify
|
170
|
+
return if redis[key("participants:#{identity}:show")]
|
171
|
+
index = alternative_for(identity)
|
172
|
+
if redis.sismember(key("alts:#{index}:participants"), identity)
|
173
|
+
redis.sadd key("alts:#{index}:converted"), identity
|
174
|
+
redis.incr key("alts:#{index}:conversions")
|
172
175
|
end
|
176
|
+
check_completion!
|
173
177
|
end
|
174
178
|
|
175
179
|
|
@@ -191,99 +195,113 @@ module Vanity
|
|
191
195
|
# experiment(:green_button).select(nil)
|
192
196
|
# end
|
193
197
|
def chooses(value)
|
194
|
-
|
195
|
-
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless
|
196
|
-
|
197
|
-
|
198
|
+
index = @alternatives.index(value)
|
199
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
200
|
+
identity = identify
|
201
|
+
redis[key("participant:#{identity}:show")] = index
|
202
|
+
self
|
203
|
+
end
|
204
|
+
|
205
|
+
def chosen?(alternative) #:nodoc:
|
206
|
+
identity = identify
|
207
|
+
index = redis[key("participant:#{identity}:show")]
|
208
|
+
index && index.to_i == alternative.id
|
209
|
+
end
|
210
|
+
|
211
|
+
# Used for testing.
|
212
|
+
def count(identity, value, *what) #:nodoc:
|
213
|
+
index = @alternatives.index(value)
|
214
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
215
|
+
if what.empty? || what.include?(:participant)
|
216
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
217
|
+
end
|
218
|
+
if what.empty? || what.include?(:conversion)
|
219
|
+
redis.sadd key("alts:#{index}:converted"), identity
|
220
|
+
redis.incr key("alts:#{index}:conversions")
|
221
|
+
end
|
222
|
+
self
|
198
223
|
end
|
199
224
|
|
200
225
|
|
201
226
|
# -- Reporting --
|
202
227
|
|
203
|
-
# Returns an object with the following
|
204
|
-
# [:alts]
|
205
|
-
# [:best]
|
206
|
-
# [:base]
|
207
|
-
# [:
|
228
|
+
# Returns an object with the following methods:
|
229
|
+
# [:alts] List of Alternative populated with interesting statistics.
|
230
|
+
# [:best] Best performing alternative.
|
231
|
+
# [:base] Second best performing alternative.
|
232
|
+
# [:least] Least performing alternative (but more than zero conversion).
|
233
|
+
# [:choice] Choice alterntive, either the outcome or best alternative (if confidence >= 90%).
|
208
234
|
#
|
209
|
-
#
|
210
|
-
# [:
|
211
|
-
# [:
|
212
|
-
# [:
|
213
|
-
# [:diff] Difference from least performant altenative (percentage).
|
214
|
-
# [:z] Z-score compared to base (above).
|
215
|
-
# [:conf] Confidence based on z-score (0, 90, 95, 99, 99.9).
|
235
|
+
# Alternatives returned by this method are populated with the following attributes:
|
236
|
+
# [:z_score] Z-score (relative to the base alternative).
|
237
|
+
# [:confidence] Confidence (z-score mapped to 0, 90, 95, 99 or 99.9%).
|
238
|
+
# [:difference] Difference from the least performant altenative.
|
216
239
|
def score
|
217
|
-
|
218
|
-
alts = alternatives.map { |alt| struct.new(alt.id, alt.conversion_rate.round(3), alt.participants) }
|
240
|
+
alts = alternatives
|
219
241
|
# sort by conversion rate to find second best and 2nd best
|
220
|
-
sorted = alts.sort_by(&:
|
242
|
+
sorted = alts.sort_by(&:conversion_rate)
|
221
243
|
base = sorted[-2]
|
222
244
|
# calculate z-score
|
223
|
-
pc = base.
|
224
|
-
nc = base.
|
245
|
+
pc = base.conversion_rate
|
246
|
+
nc = base.participants
|
225
247
|
alts.each do |alt|
|
226
|
-
p = alt.
|
227
|
-
n = alt.
|
228
|
-
alt.
|
229
|
-
alt.
|
248
|
+
p = alt.conversion_rate
|
249
|
+
n = alt.participants
|
250
|
+
alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
|
251
|
+
alt.confidence = AbTest.confidence(alt.z_score)
|
230
252
|
end
|
231
253
|
# difference is measured from least performant
|
232
|
-
if least = sorted.find { |alt| alt.
|
254
|
+
if least = sorted.find { |alt| alt.conversion_rate > 0 }
|
233
255
|
alts.each do |alt|
|
234
|
-
|
256
|
+
if alt.conversion_rate > least.conversion_rate
|
257
|
+
alt.difference = (alt.conversion_rate - least.conversion_rate) / least.conversion_rate * 100
|
258
|
+
end
|
235
259
|
end
|
236
260
|
end
|
237
261
|
# best alternative is one with highest conversion rate (best shot).
|
238
262
|
# choice alternative can only pick best if we have high confidence (>90%).
|
239
|
-
best = sorted.last if sorted.last.
|
240
|
-
choice = outcome ? alts[outcome.id] : (best && best.
|
241
|
-
Struct.new(:alts, :best, :base, :choice).new(alts, best, base, choice)
|
263
|
+
best = sorted.last if sorted.last.conversion_rate > 0.0
|
264
|
+
choice = outcome ? alts[outcome.id] : (best && best.confidence >= 90 ? best : nil)
|
265
|
+
Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
|
242
266
|
end
|
243
267
|
|
244
268
|
# Use the score returned by #score to derive a conclusion. Returns an
|
245
269
|
# array of claims.
|
246
270
|
def conclusion(score = score)
|
247
271
|
claims = []
|
248
|
-
# find name form alt structure returned from score
|
249
|
-
name = ->(alt){ alternatives[alt.id].name }
|
250
272
|
# only interested in sorted alternatives with conversion
|
251
|
-
sorted = score.alts.select { |alt| alt.
|
273
|
+
sorted = score.alts.select { |alt| alt.conversion_rate > 0.0 }.sort_by(&:conversion_rate).reverse
|
252
274
|
if sorted.size > 1
|
253
275
|
# start with alternatives that have conversion, from best to worst,
|
254
276
|
# then alternatives with no conversion.
|
255
277
|
sorted |= score.alts
|
256
278
|
# we want a result that's clearly better than 2nd best.
|
257
279
|
best, second = sorted[0], sorted[1]
|
258
|
-
if best.
|
259
|
-
diff = ((best.
|
260
|
-
better = " (%d%% better than %s)" % [diff, name
|
261
|
-
claims << "The best choice is %s: it converted at %.1f%%%s." % [name
|
262
|
-
if best.
|
263
|
-
claims << "With %d%% probability this result is statistically significant." % score.best.
|
280
|
+
if best.conversion_rate > second.conversion_rate
|
281
|
+
diff = ((best.conversion_rate - second.conversion_rate) / second.conversion_rate * 100).round
|
282
|
+
better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
|
283
|
+
claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.conversion_rate * 100, better]
|
284
|
+
if best.confidence >= 90
|
285
|
+
claims << "With %d%% probability this result is statistically significant." % score.best.confidence
|
264
286
|
else
|
265
287
|
claims << "This result is not statistically significant, suggest you continue this experiment."
|
266
288
|
end
|
267
289
|
sorted.delete best
|
268
290
|
end
|
269
291
|
sorted.each do |alt|
|
270
|
-
if alt.
|
271
|
-
claims << "%s converted at %.1f%%." % [
|
292
|
+
if alt.conversion_rate > 0.0
|
293
|
+
claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.conversion_rate * 100]
|
272
294
|
else
|
273
|
-
claims << "%s did not convert." %
|
295
|
+
claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
|
274
296
|
end
|
275
297
|
end
|
276
298
|
else
|
277
299
|
claims << "This experiment did not run long enough to find a clear winner."
|
278
300
|
end
|
279
|
-
claims << "#{
|
301
|
+
claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
|
280
302
|
claims
|
281
303
|
end
|
282
304
|
|
283
|
-
def humanize
|
284
|
-
"A/B Test"
|
285
|
-
end
|
286
|
-
|
287
305
|
|
288
306
|
# -- Completion --
|
289
307
|
|
@@ -311,40 +329,44 @@ module Vanity
|
|
311
329
|
outcome && alternatives[outcome.to_i]
|
312
330
|
end
|
313
331
|
|
314
|
-
def complete!
|
332
|
+
def complete!
|
333
|
+
return unless active?
|
315
334
|
super
|
316
335
|
if @outcome_is
|
317
336
|
begin
|
318
|
-
|
337
|
+
result = @outcome_is.call
|
338
|
+
outcome = result.id if result && result.experiment == self
|
319
339
|
rescue
|
320
340
|
# TODO: logging
|
321
341
|
end
|
322
|
-
|
323
|
-
unless outcome
|
342
|
+
else
|
324
343
|
best = score.best
|
325
344
|
outcome = best.id if best
|
326
345
|
end
|
327
346
|
# TODO: logging
|
328
|
-
redis.setnx key("outcome"), outcome
|
347
|
+
redis.setnx key("outcome"), outcome || 0
|
329
348
|
end
|
330
349
|
|
331
350
|
|
332
351
|
# -- Store/validate --
|
333
352
|
|
334
|
-
def save
|
353
|
+
def save
|
335
354
|
fail "Experiment #{name} needs at least two alternatives" unless alternatives.count >= 2
|
336
355
|
super
|
337
356
|
end
|
338
357
|
|
339
|
-
def reset!
|
358
|
+
def reset!
|
359
|
+
@alternatives.count.times do |i|
|
360
|
+
redis.del key("alts:#{i}:participants")
|
361
|
+
redis.del key("alts:#{i}:converted")
|
362
|
+
redis.del key("alts:#{i}:conversions")
|
363
|
+
end
|
340
364
|
redis.del key(:outcome)
|
341
|
-
alternatives.each(&:destroy)
|
342
365
|
super
|
343
366
|
end
|
344
367
|
|
345
|
-
def destroy
|
346
|
-
|
347
|
-
alternatives.each(&:destroy)
|
368
|
+
def destroy
|
369
|
+
reset
|
348
370
|
super
|
349
371
|
end
|
350
372
|
|
@@ -355,10 +377,7 @@ module Vanity
|
|
355
377
|
# identity, and randomly distributed alternatives for each identity (in the
|
356
378
|
# same experiment).
|
357
379
|
def alternative_for(identity)
|
358
|
-
|
359
|
-
index = session && session[id]
|
360
|
-
index ||= Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % alternatives.count
|
361
|
-
alternatives[index]
|
380
|
+
Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.count
|
362
381
|
end
|
363
382
|
|
364
383
|
begin
|