vanity 1.8.4 → 1.9.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +3 -2
- data/CHANGELOG +12 -0
- data/Gemfile +6 -3
- data/Gemfile.lock +12 -10
- data/README.rdoc +45 -16
- data/Rakefile +14 -9
- data/doc/_layouts/page.html +4 -6
- data/doc/ab_testing.textile +1 -1
- data/doc/configuring.textile +2 -4
- data/doc/email.textile +1 -3
- data/doc/index.textile +3 -63
- data/doc/rails.textile +34 -8
- data/gemfiles/rails3.gemfile +12 -3
- data/gemfiles/rails3.gemfile.lock +37 -11
- data/gemfiles/rails31.gemfile +12 -3
- data/gemfiles/rails31.gemfile.lock +37 -11
- data/gemfiles/rails32.gemfile +12 -3
- data/gemfiles/rails32.gemfile.lock +37 -11
- data/gemfiles/rails4.gemfile +12 -3
- data/gemfiles/rails4.gemfile.lock +37 -11
- data/lib/vanity/adapters/abstract_adapter.rb +4 -0
- data/lib/vanity/adapters/active_record_adapter.rb +18 -10
- data/lib/vanity/adapters/mock_adapter.rb +8 -4
- data/lib/vanity/adapters/mongodb_adapter.rb +11 -7
- data/lib/vanity/adapters/redis_adapter.rb +88 -37
- data/lib/vanity/commands/report.rb +9 -9
- data/lib/vanity/experiment/ab_test.rb +120 -101
- data/lib/vanity/experiment/alternative.rb +21 -21
- data/lib/vanity/experiment/base.rb +5 -5
- data/lib/vanity/experiment/bayesian_bandit_score.rb +51 -51
- data/lib/vanity/experiment/definition.rb +10 -10
- data/lib/vanity/frameworks/rails.rb +39 -36
- data/lib/vanity/helpers.rb +6 -4
- data/lib/vanity/metric/active_record.rb +1 -1
- data/lib/vanity/metric/base.rb +23 -24
- data/lib/vanity/metric/google_analytics.rb +5 -5
- data/lib/vanity/playground.rb +118 -24
- data/lib/vanity/templates/_report.erb +20 -6
- data/lib/vanity/templates/vanity.css +2 -0
- data/lib/vanity/version.rb +1 -1
- data/test/adapters/redis_adapter_test.rb +106 -1
- data/test/dummy/config/database.yml +21 -4
- data/test/dummy/config/routes.rb +1 -1
- data/test/experiment/ab_test.rb +93 -13
- data/test/metric/active_record_test.rb +9 -4
- data/test/passenger_test.rb +43 -42
- data/test/playground_test.rb +50 -1
- data/test/rails_dashboard_test.rb +38 -1
- data/test/rails_helper_test.rb +5 -0
- data/test/rails_test.rb +66 -15
- data/test/test_helper.rb +24 -2
- data/vanity.gemspec +0 -2
- metadata +45 -57
@@ -5,10 +5,10 @@ module Vanity
|
|
5
5
|
class Alternative
|
6
6
|
|
7
7
|
def initialize(experiment, id, value) #, participants, converted, conversions)
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
8
|
+
@experiment = experiment
|
9
|
+
@id = id
|
10
|
+
@name = "option #{(@id + 65).chr}"
|
11
|
+
@value = value
|
12
12
|
end
|
13
13
|
|
14
14
|
# Alternative id, only unique for this experiment.
|
@@ -25,22 +25,22 @@ module Vanity
|
|
25
25
|
|
26
26
|
# Number of participants who viewed this alternative.
|
27
27
|
def participants
|
28
|
-
|
29
|
-
|
28
|
+
load_counts unless @participants
|
29
|
+
@participants
|
30
30
|
end
|
31
31
|
|
32
32
|
# Number of participants who converted on this alternative (a
|
33
33
|
# participant is counted only once).
|
34
34
|
def converted
|
35
|
-
|
36
|
-
|
35
|
+
load_counts unless @converted
|
36
|
+
@converted
|
37
37
|
end
|
38
38
|
|
39
39
|
# Number of conversions for this alternative (same participant may be
|
40
40
|
# counted more than once).
|
41
41
|
def conversions
|
42
|
-
|
43
|
-
|
42
|
+
load_counts unless @conversions
|
43
|
+
@conversions
|
44
44
|
end
|
45
45
|
|
46
46
|
# Z-score for this alternative, related to 2nd-best performing
|
@@ -56,37 +56,37 @@ module Vanity
|
|
56
56
|
|
57
57
|
# Conversion rate calculated as converted/participants
|
58
58
|
def conversion_rate
|
59
|
-
|
59
|
+
@conversion_rate ||= (participants > 0 ? converted.to_f/participants.to_f : 0.0)
|
60
60
|
end
|
61
61
|
|
62
62
|
# The measure we use to order (sort) alternatives and decide which one
|
63
63
|
# is better (by calculating z-score). Defaults to conversion rate.
|
64
64
|
def measure
|
65
|
-
|
65
|
+
conversion_rate
|
66
66
|
end
|
67
67
|
|
68
68
|
def <=>(other)
|
69
|
-
|
69
|
+
measure <=> other.measure
|
70
70
|
end
|
71
71
|
|
72
72
|
def ==(other)
|
73
|
-
|
73
|
+
other && id == other.id && experiment == other.experiment
|
74
74
|
end
|
75
75
|
|
76
76
|
def to_s
|
77
|
-
|
77
|
+
name
|
78
78
|
end
|
79
79
|
|
80
80
|
def inspect
|
81
|
-
|
81
|
+
"#{name}: #{value} #{converted}/#{participants}"
|
82
82
|
end
|
83
83
|
|
84
84
|
def load_counts
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
85
|
+
if @experiment.playground.collecting?
|
86
|
+
@participants, @converted, @conversions = @experiment.playground.connection.ab_counts(@experiment.id, id).values_at(:participants, :converted, :conversions)
|
87
|
+
else
|
88
|
+
@participants = @converted = @conversions = 0
|
89
|
+
end
|
90
90
|
end
|
91
91
|
end
|
92
92
|
end
|
@@ -67,7 +67,7 @@ module Vanity
|
|
67
67
|
self.class.type
|
68
68
|
end
|
69
69
|
|
70
|
-
# Defines how we obtain an identity for the current experiment.
|
70
|
+
# Defines how we obtain an identity for the current experiment. Usually
|
71
71
|
# Vanity gets the identity form a session object (see use_vanity), but
|
72
72
|
# there are cases where you want a particular experiment to use a
|
73
73
|
# different identity.
|
@@ -95,8 +95,8 @@ module Vanity
|
|
95
95
|
# end
|
96
96
|
# end
|
97
97
|
def on_assignment(&block)
|
98
|
-
|
99
|
-
|
98
|
+
fail "Missing block" unless block
|
99
|
+
@on_assignment_block = block
|
100
100
|
end
|
101
101
|
|
102
102
|
# -- Reporting --
|
@@ -115,7 +115,7 @@ module Vanity
|
|
115
115
|
|
116
116
|
# -- Experiment completion --
|
117
117
|
|
118
|
-
# Define experiment completion condition.
|
118
|
+
# Define experiment completion condition. For example:
|
119
119
|
# complete_if do
|
120
120
|
# !score(95).chosen.nil?
|
121
121
|
# end
|
@@ -184,7 +184,7 @@ module Vanity
|
|
184
184
|
end
|
185
185
|
|
186
186
|
# Returns key for this experiment, or with an argument, return a key
|
187
|
-
# using the experiment as the namespace.
|
187
|
+
# using the experiment as the namespace. Examples:
|
188
188
|
# key => "vanity:experiments:green_button"
|
189
189
|
# key("participants") => "vanity:experiments:green_button:participants"
|
190
190
|
def key(name = nil)
|
@@ -8,85 +8,85 @@ module Vanity
|
|
8
8
|
DEFAULT_PROBABILITY = 90
|
9
9
|
|
10
10
|
def initialize(alternatives, outcome)
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
@alternatives = alternatives
|
12
|
+
@outcome = outcome
|
13
|
+
@method = :bayes_bandit_score
|
14
14
|
end
|
15
15
|
|
16
16
|
def calculate!(probability=DEFAULT_PROBABILITY)
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
# sort by conversion rate to find second best
|
18
|
+
@alts = @alternatives.sort_by(&:measure)
|
19
|
+
@base = @alts[-2]
|
20
20
|
|
21
|
-
|
21
|
+
assign_alternatives_bayesian_probability(@alts)
|
22
22
|
|
23
|
-
|
23
|
+
@least = assign_alternatives_difference(@alts)
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
# best alternative is one with highest conversion rate (best shot).
|
26
|
+
@best = @alts.last if @alts.last.measure > 0.0
|
27
|
+
# choice alternative can only pick best if we have high probability (>90%).
|
28
|
+
@choice = outcome_or_best_probability(@alternatives, @outcome, @best, probability)
|
29
|
+
self
|
30
30
|
end
|
31
31
|
|
32
32
|
protected
|
33
33
|
|
34
34
|
def outcome_or_best_probability(alternatives, outcome, best, probability)
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
if outcome
|
36
|
+
alternatives[@outcome.id]
|
37
|
+
elsif best && best.probability >= probability
|
38
|
+
best
|
39
|
+
else
|
40
|
+
nil
|
41
|
+
end
|
42
42
|
end
|
43
43
|
|
44
44
|
# Assign each alternative's bayesian probability of being the best
|
45
45
|
# alternative to alternative#probability.
|
46
46
|
def assign_alternatives_bayesian_probability(alternatives)
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
alternative_posteriors = calculate_alternative_posteriors(alternatives)
|
48
|
+
alternatives.each_with_index do |alternative, i|
|
49
|
+
alternative.probability = 100 * probability_alternative_is_best(alternative_posteriors[i], alternative_posteriors)
|
50
|
+
end
|
51
51
|
end
|
52
52
|
|
53
53
|
def calculate_alternative_posteriors(alternatives)
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
54
|
+
alternatives.map do |alternative|
|
55
|
+
x = alternative.converted
|
56
|
+
n = alternative.participants
|
57
|
+
Rubystats::BetaDistribution.new(x+1, n-x+1)
|
58
|
+
end
|
59
59
|
end
|
60
60
|
|
61
61
|
def probability_alternative_is_best(alternative_being_examined, all_alternatives)
|
62
|
-
|
63
|
-
|
64
|
-
|
62
|
+
Integration.integrate(0, 1, :tolerance=>1e-4) do |z|
|
63
|
+
pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
|
64
|
+
end
|
65
65
|
end
|
66
66
|
|
67
67
|
def pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
68
|
+
# get the pdf for this alternative at z
|
69
|
+
pdf = alternative_being_examined.pdf(z)
|
70
|
+
# now multiply by the probability that all the other alternatives are lower
|
71
|
+
all_alternatives.each do |alternative|
|
72
|
+
if alternative != alternative_being_examined
|
73
|
+
pdf = pdf * alternative.cdf(z)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
pdf
|
77
77
|
end
|
78
78
|
|
79
79
|
def assign_alternatives_difference(alternatives)
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
80
|
+
# difference is measured from least performant
|
81
|
+
least = alternatives.find { |alternative| alternative.measure > 0 }
|
82
|
+
if least
|
83
|
+
alternatives.each do |alternative|
|
84
|
+
if alternative.measure > least.measure
|
85
|
+
alternative.difference = (alternative.measure - least.measure) / least.measure * 100
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
least
|
90
90
|
end
|
91
91
|
end
|
92
92
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
module Vanity
|
2
2
|
module Experiment
|
3
3
|
# These methods are available from experiment definitions (files located in
|
4
|
-
# the experiments directory, automatically loaded by Vanity).
|
5
|
-
# methods to define
|
4
|
+
# the experiments directory, automatically loaded by Vanity). Use these
|
5
|
+
# methods to define your experiments, for example:
|
6
6
|
# ab_test "New Banner" do
|
7
7
|
# alternatives :red, :green, :blue
|
8
8
|
# metrics :signup
|
@@ -14,17 +14,17 @@ module Vanity
|
|
14
14
|
# Defines a new experiment, given the experiment's name, type and
|
15
15
|
# definition block.
|
16
16
|
def define(name, type, options = nil, &block)
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
17
|
+
fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
|
18
|
+
klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
|
19
|
+
experiment = klass.new(playground, @experiment_id, name, options)
|
20
|
+
experiment.instance_eval &block
|
21
|
+
experiment.save
|
22
|
+
playground.experiments[@experiment_id] = experiment
|
23
23
|
end
|
24
24
|
|
25
25
|
def new_binding(playground, id)
|
26
|
-
|
27
|
-
|
26
|
+
@playground, @experiment_id = playground, id
|
27
|
+
binding
|
28
28
|
end
|
29
29
|
|
30
30
|
end
|
@@ -7,7 +7,7 @@ module Vanity
|
|
7
7
|
# Do this at the very end of initialization, allowing you to change
|
8
8
|
# connection adapter, turn collection on/off, etc.
|
9
9
|
::Rails.configuration.after_initialize do
|
10
|
-
Vanity.playground.load!
|
10
|
+
Vanity.playground.load! if Vanity.playground.connected?
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
@@ -17,7 +17,8 @@ module Vanity
|
|
17
17
|
# Defines the vanity_identity method and the set_identity_context filter.
|
18
18
|
#
|
19
19
|
# Call with the name of a method that returns an object whose identity
|
20
|
-
# will be used as the Vanity identity
|
20
|
+
# will be used as the Vanity identity if the user is not already
|
21
|
+
# cookied. Confusing? Let's try by example:
|
21
22
|
#
|
22
23
|
# class ApplicationController < ActionController::Base
|
23
24
|
# use_vanity :current_user
|
@@ -28,11 +29,12 @@ module Vanity
|
|
28
29
|
# end
|
29
30
|
#
|
30
31
|
# If that method (current_user in this example) returns nil, Vanity will
|
31
|
-
#
|
32
|
-
# requests).
|
33
|
-
# identity object, by calling
|
32
|
+
# look for a vanity cookie. If there is none, it will create an identity
|
33
|
+
# (using a cookie to remember it across requests). It also uses this
|
34
|
+
# mechanism if you don't provide an identity object, by calling
|
35
|
+
# use_vanity with no arguments.
|
34
36
|
#
|
35
|
-
#
|
37
|
+
# You can also use a block:
|
36
38
|
# class ProjectController < ApplicationController
|
37
39
|
# use_vanity { |controller| controller.params[:project_id] }
|
38
40
|
# end
|
@@ -42,12 +44,20 @@ module Vanity
|
|
42
44
|
else
|
43
45
|
define_method :vanity_identity do
|
44
46
|
return @vanity_identity if @vanity_identity
|
45
|
-
|
46
|
-
|
47
|
-
|
47
|
+
|
48
|
+
# With user sign in, it's possible to visit not-logged in, get
|
49
|
+
# cookied and shown alternative A, then sign in and based on
|
50
|
+
# user.id, get shown alternative B.
|
51
|
+
# This implementation prefers an initial vanity cookie id over a
|
52
|
+
# new user.id to avoid the flash of alternative B (FOAB).
|
53
|
+
if request.get? && params[:_identity]
|
48
54
|
@vanity_identity = params[:_identity]
|
49
55
|
cookies["vanity_id"] = { :value=>@vanity_identity, :expires=>1.month.from_now }
|
50
56
|
@vanity_identity
|
57
|
+
elsif cookies["vanity_id"]
|
58
|
+
@vanity_identity = cookies["vanity_id"]
|
59
|
+
elsif symbol && object = send(symbol)
|
60
|
+
@vanity_identity = object.id
|
51
61
|
elsif response # everyday use
|
52
62
|
#conditional for Rails2 support
|
53
63
|
secure_random = defined?(SecureRandom) ? SecureRandom : ActiveSupport::SecureRandom
|
@@ -95,7 +105,7 @@ module Vanity
|
|
95
105
|
end
|
96
106
|
|
97
107
|
|
98
|
-
# Vanity needs these filters.
|
108
|
+
# Vanity needs these filters. They are includes in ActionController and
|
99
109
|
# automatically added when you use #use_vanity in your controller.
|
100
110
|
module Filters
|
101
111
|
# Around filter that sets Vanity.context to controller.
|
@@ -110,9 +120,9 @@ module Vanity
|
|
110
120
|
# parameter.
|
111
121
|
#
|
112
122
|
# Each alternative has a unique fingerprint (run vanity list command to
|
113
|
-
# see them all).
|
123
|
+
# see them all). A request with the _vanity query parameter is
|
114
124
|
# intercepted, the alternative is chosen, and the user redirected to the
|
115
|
-
# same request URL sans _vanity parameter.
|
125
|
+
# same request URL sans _vanity parameter. This only works for GET
|
116
126
|
# requests.
|
117
127
|
#
|
118
128
|
# For example, if the user requests the page
|
@@ -137,14 +147,14 @@ module Vanity
|
|
137
147
|
end
|
138
148
|
end
|
139
149
|
|
140
|
-
# Before filter to reload Vanity experiments/metrics.
|
150
|
+
# Before filter to reload Vanity experiments/metrics. Enabled when
|
141
151
|
# cache_classes is false (typically, testing environment).
|
142
152
|
def vanity_reload_filter
|
143
153
|
Vanity.playground.reload!
|
144
154
|
end
|
145
155
|
|
146
|
-
# Filter to track metrics
|
147
|
-
#
|
156
|
+
# Filter to track metrics. Pass _track param along to call track! on that
|
157
|
+
# alternative.
|
148
158
|
def vanity_track_filter
|
149
159
|
if request.get? && params[:_track]
|
150
160
|
track! params[:_track]
|
@@ -155,7 +165,7 @@ module Vanity
|
|
155
165
|
end
|
156
166
|
|
157
167
|
|
158
|
-
# Introduces ab_test helper (controllers and views).
|
168
|
+
# Introduces ab_test helper (controllers and views). Similar to the generic
|
159
169
|
# ab_test method, with the ability to capture content (applicable to views,
|
160
170
|
# see examples).
|
161
171
|
module Helpers
|
@@ -263,7 +273,8 @@ module Vanity
|
|
263
273
|
|
264
274
|
def setup_experiment(name)
|
265
275
|
@_vanity_experiments ||= {}
|
266
|
-
|
276
|
+
request = respond_to?(:request) ? self.request : nil
|
277
|
+
@_vanity_experiments[name] ||= Vanity.playground.experiment(name.to_sym).choose(request)
|
267
278
|
@_vanity_experiments[name].value
|
268
279
|
end
|
269
280
|
|
@@ -281,11 +292,14 @@ module Vanity
|
|
281
292
|
# Step 3: Open your browser to http://localhost:3000/vanity
|
282
293
|
module Dashboard
|
283
294
|
def index
|
284
|
-
|
295
|
+
@experiments = Vanity.playground.experiments
|
296
|
+
@experiments_persisted = Vanity.playground.experiments_persisted?
|
297
|
+
@metrics = Vanity.playground.metrics
|
298
|
+
render :file=>Vanity.template("_report"), :content_type=>Mime::HTML
|
285
299
|
end
|
286
300
|
|
287
301
|
def participant
|
288
|
-
render :file=>Vanity.template("_participant"), :locals=>{:participant_id => params[:id], :participant_info => Vanity.playground.participant_info(params[:id])}, :content_type=>Mime::HTML
|
302
|
+
render :file=>Vanity.template("_participant"), :locals=>{:participant_id => params[:id], :participant_info => Vanity.playground.participant_info(params[:id])}, :content_type=>Mime::HTML
|
289
303
|
end
|
290
304
|
|
291
305
|
def complete
|
@@ -308,13 +322,14 @@ module Vanity
|
|
308
322
|
render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
|
309
323
|
end
|
310
324
|
|
325
|
+
# JS callback action used by vanity_js
|
311
326
|
def add_participant
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
327
|
+
if params[:e].nil? || params[:e].empty?
|
328
|
+
render :status => 404, :nothing => true
|
329
|
+
return
|
330
|
+
end
|
316
331
|
exp = Vanity.playground.experiment(params[:e].to_sym)
|
317
|
-
exp.chooses(exp.alternatives[params[:a].to_i].value)
|
332
|
+
exp.chooses(exp.alternatives[params[:a].to_i].value, request)
|
318
333
|
render :status => 200, :nothing => true
|
319
334
|
end
|
320
335
|
end
|
@@ -337,18 +352,6 @@ if defined?(ActionController)
|
|
337
352
|
include Vanity::Rails::Filters
|
338
353
|
helper Vanity::Rails::Helpers
|
339
354
|
end
|
340
|
-
|
341
|
-
module ActionController
|
342
|
-
class TestCase
|
343
|
-
alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
|
344
|
-
# Sets Vanity.context to the current controller, so you can do things like:
|
345
|
-
# experiment(:simple).chooses(:green)
|
346
|
-
def setup_controller_request_and_response
|
347
|
-
setup_controller_request_and_response_without_vanity
|
348
|
-
Vanity.context = @controller
|
349
|
-
end
|
350
|
-
end
|
351
|
-
end
|
352
355
|
end
|
353
356
|
|
354
357
|
if defined?(ActionMailer)
|