vanity 1.8.4 → 1.9.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/.travis.yml +3 -2
  2. data/CHANGELOG +12 -0
  3. data/Gemfile +6 -3
  4. data/Gemfile.lock +12 -10
  5. data/README.rdoc +45 -16
  6. data/Rakefile +14 -9
  7. data/doc/_layouts/page.html +4 -6
  8. data/doc/ab_testing.textile +1 -1
  9. data/doc/configuring.textile +2 -4
  10. data/doc/email.textile +1 -3
  11. data/doc/index.textile +3 -63
  12. data/doc/rails.textile +34 -8
  13. data/gemfiles/rails3.gemfile +12 -3
  14. data/gemfiles/rails3.gemfile.lock +37 -11
  15. data/gemfiles/rails31.gemfile +12 -3
  16. data/gemfiles/rails31.gemfile.lock +37 -11
  17. data/gemfiles/rails32.gemfile +12 -3
  18. data/gemfiles/rails32.gemfile.lock +37 -11
  19. data/gemfiles/rails4.gemfile +12 -3
  20. data/gemfiles/rails4.gemfile.lock +37 -11
  21. data/lib/vanity/adapters/abstract_adapter.rb +4 -0
  22. data/lib/vanity/adapters/active_record_adapter.rb +18 -10
  23. data/lib/vanity/adapters/mock_adapter.rb +8 -4
  24. data/lib/vanity/adapters/mongodb_adapter.rb +11 -7
  25. data/lib/vanity/adapters/redis_adapter.rb +88 -37
  26. data/lib/vanity/commands/report.rb +9 -9
  27. data/lib/vanity/experiment/ab_test.rb +120 -101
  28. data/lib/vanity/experiment/alternative.rb +21 -21
  29. data/lib/vanity/experiment/base.rb +5 -5
  30. data/lib/vanity/experiment/bayesian_bandit_score.rb +51 -51
  31. data/lib/vanity/experiment/definition.rb +10 -10
  32. data/lib/vanity/frameworks/rails.rb +39 -36
  33. data/lib/vanity/helpers.rb +6 -4
  34. data/lib/vanity/metric/active_record.rb +1 -1
  35. data/lib/vanity/metric/base.rb +23 -24
  36. data/lib/vanity/metric/google_analytics.rb +5 -5
  37. data/lib/vanity/playground.rb +118 -24
  38. data/lib/vanity/templates/_report.erb +20 -6
  39. data/lib/vanity/templates/vanity.css +2 -0
  40. data/lib/vanity/version.rb +1 -1
  41. data/test/adapters/redis_adapter_test.rb +106 -1
  42. data/test/dummy/config/database.yml +21 -4
  43. data/test/dummy/config/routes.rb +1 -1
  44. data/test/experiment/ab_test.rb +93 -13
  45. data/test/metric/active_record_test.rb +9 -4
  46. data/test/passenger_test.rb +43 -42
  47. data/test/playground_test.rb +50 -1
  48. data/test/rails_dashboard_test.rb +38 -1
  49. data/test/rails_helper_test.rb +5 -0
  50. data/test/rails_test.rb +66 -15
  51. data/test/test_helper.rb +24 -2
  52. data/vanity.gemspec +0 -2
  53. metadata +45 -57
@@ -5,10 +5,10 @@ module Vanity
5
5
  class Alternative
6
6
 
7
7
  def initialize(experiment, id, value) #, participants, converted, conversions)
8
- @experiment = experiment
9
- @id = id
10
- @name = "option #{(@id + 65).chr}"
11
- @value = value
8
+ @experiment = experiment
9
+ @id = id
10
+ @name = "option #{(@id + 65).chr}"
11
+ @value = value
12
12
  end
13
13
 
14
14
  # Alternative id, only unique for this experiment.
@@ -25,22 +25,22 @@ module Vanity
25
25
 
26
26
  # Number of participants who viewed this alternative.
27
27
  def participants
28
- load_counts unless @participants
29
- @participants
28
+ load_counts unless @participants
29
+ @participants
30
30
  end
31
31
 
32
32
  # Number of participants who converted on this alternative (a
33
33
  # participant is counted only once).
34
34
  def converted
35
- load_counts unless @converted
36
- @converted
35
+ load_counts unless @converted
36
+ @converted
37
37
  end
38
38
 
39
39
  # Number of conversions for this alternative (same participant may be
40
40
  # counted more than once).
41
41
  def conversions
42
- load_counts unless @conversions
43
- @conversions
42
+ load_counts unless @conversions
43
+ @conversions
44
44
  end
45
45
 
46
46
  # Z-score for this alternative, related to 2nd-best performing
@@ -56,37 +56,37 @@ module Vanity
56
56
 
57
57
  # Conversion rate calculated as converted/participants
58
58
  def conversion_rate
59
- @conversion_rate ||= (participants > 0 ? converted.to_f/participants.to_f : 0.0)
59
+ @conversion_rate ||= (participants > 0 ? converted.to_f/participants.to_f : 0.0)
60
60
  end
61
61
 
62
62
  # The measure we use to order (sort) alternatives and decide which one
63
63
  # is better (by calculating z-score). Defaults to conversion rate.
64
64
  def measure
65
- conversion_rate
65
+ conversion_rate
66
66
  end
67
67
 
68
68
  def <=>(other)
69
- measure <=> other.measure
69
+ measure <=> other.measure
70
70
  end
71
71
 
72
72
  def ==(other)
73
- other && id == other.id && experiment == other.experiment
73
+ other && id == other.id && experiment == other.experiment
74
74
  end
75
75
 
76
76
  def to_s
77
- name
77
+ name
78
78
  end
79
79
 
80
80
  def inspect
81
- "#{name}: #{value} #{converted}/#{participants}"
81
+ "#{name}: #{value} #{converted}/#{participants}"
82
82
  end
83
83
 
84
84
  def load_counts
85
- if @experiment.playground.collecting?
86
- @participants, @converted, @conversions = @experiment.playground.connection.ab_counts(@experiment.id, id).values_at(:participants, :converted, :conversions)
87
- else
88
- @participants = @converted = @conversions = 0
89
- end
85
+ if @experiment.playground.collecting?
86
+ @participants, @converted, @conversions = @experiment.playground.connection.ab_counts(@experiment.id, id).values_at(:participants, :converted, :conversions)
87
+ else
88
+ @participants = @converted = @conversions = 0
89
+ end
90
90
  end
91
91
  end
92
92
  end
@@ -67,7 +67,7 @@ module Vanity
67
67
  self.class.type
68
68
  end
69
69
 
70
- # Defines how we obtain an identity for the current experiment. Usually
70
+ # Defines how we obtain an identity for the current experiment. Usually
71
71
  # Vanity gets the identity form a session object (see use_vanity), but
72
72
  # there are cases where you want a particular experiment to use a
73
73
  # different identity.
@@ -95,8 +95,8 @@ module Vanity
95
95
  # end
96
96
  # end
97
97
  def on_assignment(&block)
98
- fail "Missing block" unless block
99
- @on_assignment_block = block
98
+ fail "Missing block" unless block
99
+ @on_assignment_block = block
100
100
  end
101
101
 
102
102
  # -- Reporting --
@@ -115,7 +115,7 @@ module Vanity
115
115
 
116
116
  # -- Experiment completion --
117
117
 
118
- # Define experiment completion condition. For example:
118
+ # Define experiment completion condition. For example:
119
119
  # complete_if do
120
120
  # !score(95).chosen.nil?
121
121
  # end
@@ -184,7 +184,7 @@ module Vanity
184
184
  end
185
185
 
186
186
  # Returns key for this experiment, or with an argument, return a key
187
- # using the experiment as the namespace. Examples:
187
+ # using the experiment as the namespace. Examples:
188
188
  # key => "vanity:experiments:green_button"
189
189
  # key("participants") => "vanity:experiments:green_button:participants"
190
190
  def key(name = nil)
@@ -8,85 +8,85 @@ module Vanity
8
8
  DEFAULT_PROBABILITY = 90
9
9
 
10
10
  def initialize(alternatives, outcome)
11
- @alternatives = alternatives
12
- @outcome = outcome
13
- @method = :bayes_bandit_score
11
+ @alternatives = alternatives
12
+ @outcome = outcome
13
+ @method = :bayes_bandit_score
14
14
  end
15
15
 
16
16
  def calculate!(probability=DEFAULT_PROBABILITY)
17
- # sort by conversion rate to find second best
18
- @alts = @alternatives.sort_by(&:measure)
19
- @base = @alts[-2]
17
+ # sort by conversion rate to find second best
18
+ @alts = @alternatives.sort_by(&:measure)
19
+ @base = @alts[-2]
20
20
 
21
- assign_alternatives_bayesian_probability(@alts)
21
+ assign_alternatives_bayesian_probability(@alts)
22
22
 
23
- @least = assign_alternatives_difference(@alts)
23
+ @least = assign_alternatives_difference(@alts)
24
24
 
25
- # best alternative is one with highest conversion rate (best shot).
26
- @best = @alts.last if @alts.last.measure > 0.0
27
- # choice alternative can only pick best if we have high probability (>90%).
28
- @choice = outcome_or_best_probability(@alternatives, @outcome, @best, probability)
29
- self
25
+ # best alternative is one with highest conversion rate (best shot).
26
+ @best = @alts.last if @alts.last.measure > 0.0
27
+ # choice alternative can only pick best if we have high probability (>90%).
28
+ @choice = outcome_or_best_probability(@alternatives, @outcome, @best, probability)
29
+ self
30
30
  end
31
31
 
32
32
  protected
33
33
 
34
34
  def outcome_or_best_probability(alternatives, outcome, best, probability)
35
- if outcome
36
- alternatives[@outcome.id]
37
- elsif best && best.probability >= probability
38
- best
39
- else
40
- nil
41
- end
35
+ if outcome
36
+ alternatives[@outcome.id]
37
+ elsif best && best.probability >= probability
38
+ best
39
+ else
40
+ nil
41
+ end
42
42
  end
43
43
 
44
44
  # Assign each alternative's bayesian probability of being the best
45
45
  # alternative to alternative#probability.
46
46
  def assign_alternatives_bayesian_probability(alternatives)
47
- alternative_posteriors = calculate_alternative_posteriors(alternatives)
48
- alternatives.each_with_index do |alternative, i|
49
- alternative.probability = 100 * probability_alternative_is_best(alternative_posteriors[i], alternative_posteriors)
50
- end
47
+ alternative_posteriors = calculate_alternative_posteriors(alternatives)
48
+ alternatives.each_with_index do |alternative, i|
49
+ alternative.probability = 100 * probability_alternative_is_best(alternative_posteriors[i], alternative_posteriors)
50
+ end
51
51
  end
52
52
 
53
53
  def calculate_alternative_posteriors(alternatives)
54
- alternatives.map do |alternative|
55
- x = alternative.converted
56
- n = alternative.participants
57
- Rubystats::BetaDistribution.new(x+1, n-x+1)
58
- end
54
+ alternatives.map do |alternative|
55
+ x = alternative.converted
56
+ n = alternative.participants
57
+ Rubystats::BetaDistribution.new(x+1, n-x+1)
58
+ end
59
59
  end
60
60
 
61
61
  def probability_alternative_is_best(alternative_being_examined, all_alternatives)
62
- Integration.integrate(0, 1, :tolerance=>1e-4) do |z|
63
- pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
64
- end
62
+ Integration.integrate(0, 1, :tolerance=>1e-4) do |z|
63
+ pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
64
+ end
65
65
  end
66
66
 
67
67
  def pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
68
- # get the pdf for this alternative at z
69
- pdf = alternative_being_examined.pdf(z)
70
- # now multiply by the probability that all the other alternatives are lower
71
- all_alternatives.each do |alternative|
72
- if alternative != alternative_being_examined
73
- pdf = pdf * alternative.cdf(z)
74
- end
75
- end
76
- pdf
68
+ # get the pdf for this alternative at z
69
+ pdf = alternative_being_examined.pdf(z)
70
+ # now multiply by the probability that all the other alternatives are lower
71
+ all_alternatives.each do |alternative|
72
+ if alternative != alternative_being_examined
73
+ pdf = pdf * alternative.cdf(z)
74
+ end
75
+ end
76
+ pdf
77
77
  end
78
78
 
79
79
  def assign_alternatives_difference(alternatives)
80
- # difference is measured from least performant
81
- least = alternatives.find { |alternative| alternative.measure > 0 }
82
- if least
83
- alternatives.each do |alternative|
84
- if alternative.measure > least.measure
85
- alternative.difference = (alternative.measure - least.measure) / least.measure * 100
86
- end
87
- end
88
- end
89
- least
80
+ # difference is measured from least performant
81
+ least = alternatives.find { |alternative| alternative.measure > 0 }
82
+ if least
83
+ alternatives.each do |alternative|
84
+ if alternative.measure > least.measure
85
+ alternative.difference = (alternative.measure - least.measure) / least.measure * 100
86
+ end
87
+ end
88
+ end
89
+ least
90
90
  end
91
91
  end
92
92
  end
@@ -1,8 +1,8 @@
1
1
  module Vanity
2
2
  module Experiment
3
3
  # These methods are available from experiment definitions (files located in
4
- # the experiments directory, automatically loaded by Vanity). Use these
5
- # methods to define you experiments, for example:
4
+ # the experiments directory, automatically loaded by Vanity). Use these
5
+ # methods to define your experiments, for example:
6
6
  # ab_test "New Banner" do
7
7
  # alternatives :red, :green, :blue
8
8
  # metrics :signup
@@ -14,17 +14,17 @@ module Vanity
14
14
  # Defines a new experiment, given the experiment's name, type and
15
15
  # definition block.
16
16
  def define(name, type, options = nil, &block)
17
- fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
18
- klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
19
- experiment = klass.new(playground, @experiment_id, name, options)
20
- experiment.instance_eval &block
21
- experiment.save
22
- playground.experiments[@experiment_id] = experiment
17
+ fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
18
+ klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
19
+ experiment = klass.new(playground, @experiment_id, name, options)
20
+ experiment.instance_eval &block
21
+ experiment.save
22
+ playground.experiments[@experiment_id] = experiment
23
23
  end
24
24
 
25
25
  def new_binding(playground, id)
26
- @playground, @experiment_id = playground, id
27
- binding
26
+ @playground, @experiment_id = playground, id
27
+ binding
28
28
  end
29
29
 
30
30
  end
@@ -7,7 +7,7 @@ module Vanity
7
7
  # Do this at the very end of initialization, allowing you to change
8
8
  # connection adapter, turn collection on/off, etc.
9
9
  ::Rails.configuration.after_initialize do
10
- Vanity.playground.load!
10
+ Vanity.playground.load! if Vanity.playground.connected?
11
11
  end
12
12
  end
13
13
 
@@ -17,7 +17,8 @@ module Vanity
17
17
  # Defines the vanity_identity method and the set_identity_context filter.
18
18
  #
19
19
  # Call with the name of a method that returns an object whose identity
20
- # will be used as the Vanity identity. Confusing? Let's try by example:
20
+ # will be used as the Vanity identity if the user is not already
21
+ # cookied. Confusing? Let's try by example:
21
22
  #
22
23
  # class ApplicationController < ActionController::Base
23
24
  # use_vanity :current_user
@@ -28,11 +29,12 @@ module Vanity
28
29
  # end
29
30
  #
30
31
  # If that method (current_user in this example) returns nil, Vanity will
31
- # set the identity for you (using a cookie to remember it across
32
- # requests). It also uses this mechanism if you don't provide an
33
- # identity object, by calling use_vanity with no arguments.
32
+ # look for a vanity cookie. If there is none, it will create an identity
33
+ # (using a cookie to remember it across requests). It also uses this
34
+ # mechanism if you don't provide an identity object, by calling
35
+ # use_vanity with no arguments.
34
36
  #
35
- # Of course you can also use a block:
37
+ # You can also use a block:
36
38
  # class ProjectController < ApplicationController
37
39
  # use_vanity { |controller| controller.params[:project_id] }
38
40
  # end
@@ -42,12 +44,20 @@ module Vanity
42
44
  else
43
45
  define_method :vanity_identity do
44
46
  return @vanity_identity if @vanity_identity
45
- if symbol && object = send(symbol)
46
- @vanity_identity = object.id
47
- elsif request.get? && params[:_identity]
47
+
48
+ # With user sign in, it's possible to visit not-logged in, get
49
+ # cookied and shown alternative A, then sign in and based on
50
+ # user.id, get shown alternative B.
51
+ # This implementation prefers an initial vanity cookie id over a
52
+ # new user.id to avoid the flash of alternative B (FOAB).
53
+ if request.get? && params[:_identity]
48
54
  @vanity_identity = params[:_identity]
49
55
  cookies["vanity_id"] = { :value=>@vanity_identity, :expires=>1.month.from_now }
50
56
  @vanity_identity
57
+ elsif cookies["vanity_id"]
58
+ @vanity_identity = cookies["vanity_id"]
59
+ elsif symbol && object = send(symbol)
60
+ @vanity_identity = object.id
51
61
  elsif response # everyday use
52
62
  #conditional for Rails2 support
53
63
  secure_random = defined?(SecureRandom) ? SecureRandom : ActiveSupport::SecureRandom
@@ -95,7 +105,7 @@ module Vanity
95
105
  end
96
106
 
97
107
 
98
- # Vanity needs these filters. They are includes in ActionController and
108
+ # Vanity needs these filters. They are includes in ActionController and
99
109
  # automatically added when you use #use_vanity in your controller.
100
110
  module Filters
101
111
  # Around filter that sets Vanity.context to controller.
@@ -110,9 +120,9 @@ module Vanity
110
120
  # parameter.
111
121
  #
112
122
  # Each alternative has a unique fingerprint (run vanity list command to
113
- # see them all). A request with the _vanity query parameter is
123
+ # see them all). A request with the _vanity query parameter is
114
124
  # intercepted, the alternative is chosen, and the user redirected to the
115
- # same request URL sans _vanity parameter. This only works for GET
125
+ # same request URL sans _vanity parameter. This only works for GET
116
126
  # requests.
117
127
  #
118
128
  # For example, if the user requests the page
@@ -137,14 +147,14 @@ module Vanity
137
147
  end
138
148
  end
139
149
 
140
- # Before filter to reload Vanity experiments/metrics. Enabled when
150
+ # Before filter to reload Vanity experiments/metrics. Enabled when
141
151
  # cache_classes is false (typically, testing environment).
142
152
  def vanity_reload_filter
143
153
  Vanity.playground.reload!
144
154
  end
145
155
 
146
- # Filter to track metrics
147
- # pass _track param along to call track! on that alternative
156
+ # Filter to track metrics. Pass _track param along to call track! on that
157
+ # alternative.
148
158
  def vanity_track_filter
149
159
  if request.get? && params[:_track]
150
160
  track! params[:_track]
@@ -155,7 +165,7 @@ module Vanity
155
165
  end
156
166
 
157
167
 
158
- # Introduces ab_test helper (controllers and views). Similar to the generic
168
+ # Introduces ab_test helper (controllers and views). Similar to the generic
159
169
  # ab_test method, with the ability to capture content (applicable to views,
160
170
  # see examples).
161
171
  module Helpers
@@ -263,7 +273,8 @@ module Vanity
263
273
 
264
274
  def setup_experiment(name)
265
275
  @_vanity_experiments ||= {}
266
- @_vanity_experiments[name] ||= Vanity.playground.experiment(name.to_sym).choose
276
+ request = respond_to?(:request) ? self.request : nil
277
+ @_vanity_experiments[name] ||= Vanity.playground.experiment(name.to_sym).choose(request)
267
278
  @_vanity_experiments[name].value
268
279
  end
269
280
 
@@ -281,11 +292,14 @@ module Vanity
281
292
  # Step 3: Open your browser to http://localhost:3000/vanity
282
293
  module Dashboard
283
294
  def index
284
- render :file=>Vanity.template("_report"), :content_type=>Mime::HTML, :layout=>false
295
+ @experiments = Vanity.playground.experiments
296
+ @experiments_persisted = Vanity.playground.experiments_persisted?
297
+ @metrics = Vanity.playground.metrics
298
+ render :file=>Vanity.template("_report"), :content_type=>Mime::HTML
285
299
  end
286
300
 
287
301
  def participant
288
- render :file=>Vanity.template("_participant"), :locals=>{:participant_id => params[:id], :participant_info => Vanity.playground.participant_info(params[:id])}, :content_type=>Mime::HTML, :layout=>false
302
+ render :file=>Vanity.template("_participant"), :locals=>{:participant_id => params[:id], :participant_info => Vanity.playground.participant_info(params[:id])}, :content_type=>Mime::HTML
289
303
  end
290
304
 
291
305
  def complete
@@ -308,13 +322,14 @@ module Vanity
308
322
  render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
309
323
  end
310
324
 
325
+ # JS callback action used by vanity_js
311
326
  def add_participant
312
- if params[:e].nil? || params[:e].empty?
313
- render :status => 404, :nothing => true
314
- return
315
- end
327
+ if params[:e].nil? || params[:e].empty?
328
+ render :status => 404, :nothing => true
329
+ return
330
+ end
316
331
  exp = Vanity.playground.experiment(params[:e].to_sym)
317
- exp.chooses(exp.alternatives[params[:a].to_i].value)
332
+ exp.chooses(exp.alternatives[params[:a].to_i].value, request)
318
333
  render :status => 200, :nothing => true
319
334
  end
320
335
  end
@@ -337,18 +352,6 @@ if defined?(ActionController)
337
352
  include Vanity::Rails::Filters
338
353
  helper Vanity::Rails::Helpers
339
354
  end
340
-
341
- module ActionController
342
- class TestCase
343
- alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
344
- # Sets Vanity.context to the current controller, so you can do things like:
345
- # experiment(:simple).chooses(:green)
346
- def setup_controller_request_and_response
347
- setup_controller_request_and_response_without_vanity
348
- Vanity.context = @controller
349
- end
350
- end
351
- end
352
355
  end
353
356
 
354
357
  if defined?(ActionMailer)