tupalo-vanity 1.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +243 -0
- data/Gemfile +24 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +74 -0
- data/Rakefile +189 -0
- data/bin/vanity +69 -0
- data/lib/vanity.rb +36 -0
- data/lib/vanity/adapters/abstract_adapter.rb +135 -0
- data/lib/vanity/adapters/active_record_adapter.rb +304 -0
- data/lib/vanity/adapters/mock_adapter.rb +157 -0
- data/lib/vanity/adapters/mongodb_adapter.rb +162 -0
- data/lib/vanity/adapters/redis_adapter.rb +154 -0
- data/lib/vanity/backport.rb +26 -0
- data/lib/vanity/commands/list.rb +21 -0
- data/lib/vanity/commands/report.rb +64 -0
- data/lib/vanity/commands/upgrade.rb +34 -0
- data/lib/vanity/experiment/ab_test.rb +482 -0
- data/lib/vanity/experiment/base.rb +212 -0
- data/lib/vanity/frameworks/rails.rb +244 -0
- data/lib/vanity/helpers.rb +59 -0
- data/lib/vanity/metric/active_record.rb +83 -0
- data/lib/vanity/metric/base.rb +244 -0
- data/lib/vanity/metric/google_analytics.rb +83 -0
- data/lib/vanity/metric/remote.rb +53 -0
- data/lib/vanity/playground.rb +332 -0
- data/lib/vanity/templates/_ab_test.erb +28 -0
- data/lib/vanity/templates/_experiment.erb +5 -0
- data/lib/vanity/templates/_experiments.erb +7 -0
- data/lib/vanity/templates/_metric.erb +14 -0
- data/lib/vanity/templates/_metrics.erb +13 -0
- data/lib/vanity/templates/_report.erb +27 -0
- data/lib/vanity/templates/flot.min.js +1 -0
- data/lib/vanity/templates/jquery.min.js +19 -0
- data/lib/vanity/templates/vanity.css +26 -0
- data/lib/vanity/templates/vanity.js +82 -0
- data/lib/vanity/version.rb +11 -0
- data/test/experiment/ab_test.rb +700 -0
- data/test/experiment/base_test.rb +136 -0
- data/test/experiments/age_and_zipcode.rb +19 -0
- data/test/experiments/metrics/cheers.rb +3 -0
- data/test/experiments/metrics/signups.rb +2 -0
- data/test/experiments/metrics/yawns.rb +3 -0
- data/test/experiments/null_abc.rb +5 -0
- data/test/metric/active_record_test.rb +249 -0
- data/test/metric/base_test.rb +293 -0
- data/test/metric/google_analytics_test.rb +104 -0
- data/test/metric/remote_test.rb +108 -0
- data/test/myapp/app/controllers/application_controller.rb +2 -0
- data/test/myapp/app/controllers/main_controller.rb +7 -0
- data/test/myapp/config/boot.rb +110 -0
- data/test/myapp/config/environment.rb +10 -0
- data/test/myapp/config/environments/production.rb +0 -0
- data/test/myapp/config/routes.rb +3 -0
- data/test/passenger_test.rb +43 -0
- data/test/playground_test.rb +10 -0
- data/test/rails_test.rb +294 -0
- data/test/test_helper.rb +134 -0
- data/tupalo-vanity.gemspec +25 -0
- metadata +152 -0
@@ -0,0 +1,212 @@
|
|
1
|
+
module Vanity
|
2
|
+
module Experiment
|
3
|
+
|
4
|
+
# These methods are available from experiment definitions (files located in
|
5
|
+
# the experiments directory, automatically loaded by Vanity). Use these
|
6
|
+
# methods to define you experiments, for example:
|
7
|
+
# ab_test "New Banner" do
|
8
|
+
# alternatives :red, :green, :blue
|
9
|
+
# metrics :signup
|
10
|
+
# end
|
11
|
+
module Definition
|
12
|
+
|
13
|
+
attr_reader :playground
|
14
|
+
|
15
|
+
# Defines a new experiment, given the experiment's name, type and
|
16
|
+
# definition block.
|
17
|
+
def define(name, type, options = nil, &block)
|
18
|
+
fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
|
19
|
+
klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
|
20
|
+
experiment = klass.new(playground, @experiment_id, name, options)
|
21
|
+
experiment.instance_eval &block
|
22
|
+
experiment.save
|
23
|
+
playground.experiments[@experiment_id] = experiment
|
24
|
+
end
|
25
|
+
|
26
|
+
def new_binding(playground, id)
|
27
|
+
@playground, @experiment_id = playground, id
|
28
|
+
binding
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
# Base class that all experiment types are derived from.
|
34
|
+
class Base
|
35
|
+
|
36
|
+
class << self
|
37
|
+
|
38
|
+
# Returns the type of this class as a symbol (e.g. AbTest becomes
|
39
|
+
# ab_test).
|
40
|
+
def type
|
41
|
+
name.split("::").last.gsub(/([a-z])([A-Z])/) { "#{$1}_#{$2}" }.gsub(/([A-Z])([A-Z][a-z])/) { "#{$1}_#{$2}" }.downcase
|
42
|
+
end
|
43
|
+
|
44
|
+
# Playground uses this to load experiment definitions.
|
45
|
+
def load(playground, stack, file)
|
46
|
+
fail "Circular dependency detected: #{stack.join('=>')}=>#{file}" if stack.include?(file)
|
47
|
+
source = File.read(file)
|
48
|
+
stack.push file
|
49
|
+
id = File.basename(file, ".rb").downcase.gsub(/\W/, "_").to_sym
|
50
|
+
context = Object.new
|
51
|
+
context.instance_eval do
|
52
|
+
extend Definition
|
53
|
+
experiment = eval(source, context.new_binding(playground, id), file)
|
54
|
+
fail NameError.new("Expected #{file} to define experiment #{id}", id) unless playground.experiments[id]
|
55
|
+
experiment
|
56
|
+
end
|
57
|
+
rescue
|
58
|
+
error = NameError.exception($!.message, id)
|
59
|
+
error.set_backtrace $!.backtrace
|
60
|
+
raise error
|
61
|
+
ensure
|
62
|
+
stack.pop
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
def initialize(playground, id, name, options = nil)
|
68
|
+
@playground = playground
|
69
|
+
@id, @name = id.to_sym, name
|
70
|
+
@options = options || {}
|
71
|
+
@identify_block = method(:default_identify)
|
72
|
+
end
|
73
|
+
|
74
|
+
# Human readable experiment name (first argument you pass when creating a
|
75
|
+
# new experiment).
|
76
|
+
attr_reader :name
|
77
|
+
alias :to_s :name
|
78
|
+
|
79
|
+
# Unique identifier, derived from name experiment name, e.g. "Green
|
80
|
+
# Button" becomes :green_button.
|
81
|
+
attr_reader :id
|
82
|
+
|
83
|
+
# Time stamp when experiment was created.
|
84
|
+
def created_at
|
85
|
+
@created_at ||= connection.get_experiment_created_at(@id)
|
86
|
+
end
|
87
|
+
|
88
|
+
# Time stamp when experiment was completed.
|
89
|
+
attr_reader :completed_at
|
90
|
+
|
91
|
+
# Returns the type of this experiment as a symbol (e.g. :ab_test).
|
92
|
+
def type
|
93
|
+
self.class.type
|
94
|
+
end
|
95
|
+
|
96
|
+
# Defines how we obtain an identity for the current experiment. Usually
|
97
|
+
# Vanity gets the identity form a session object (see use_vanity), but
|
98
|
+
# there are cases where you want a particular experiment to use a
|
99
|
+
# different identity.
|
100
|
+
#
|
101
|
+
# For example, if all your experiments use current_user and you need one
|
102
|
+
# experiment to use the current project:
|
103
|
+
# ab_test "Project widget" do
|
104
|
+
# alternatives :small, :medium, :large
|
105
|
+
# identify do |controller|
|
106
|
+
# controller.project.id
|
107
|
+
# end
|
108
|
+
# end
|
109
|
+
def identify(&block)
|
110
|
+
fail "Missing block" unless block
|
111
|
+
@identify_block = block
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
# -- Reporting --
|
116
|
+
|
117
|
+
# Sets or returns description. For example
|
118
|
+
# ab_test "Simple" do
|
119
|
+
# description "A simple A/B experiment"
|
120
|
+
# end
|
121
|
+
#
|
122
|
+
# puts "Just defined: " + experiment(:simple).description
|
123
|
+
def description(text = nil)
|
124
|
+
@description = text if text
|
125
|
+
@description
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
# -- Experiment completion --
|
130
|
+
|
131
|
+
# Define experiment completion condition. For example:
|
132
|
+
# complete_if do
|
133
|
+
# !score(95).chosen.nil?
|
134
|
+
# end
|
135
|
+
def complete_if(&block)
|
136
|
+
raise ArgumentError, "Missing block" unless block
|
137
|
+
raise "complete_if already called on this experiment" if @complete_block
|
138
|
+
@complete_block = block
|
139
|
+
end
|
140
|
+
|
141
|
+
# Force experiment to complete.
|
142
|
+
def complete!
|
143
|
+
@playground.logger.info "vanity: completed experiment #{id}"
|
144
|
+
return unless @playground.collecting?
|
145
|
+
connection.set_experiment_completed_at @id, Time.now
|
146
|
+
@completed_at = connection.get_experiment_completed_at(@id)
|
147
|
+
end
|
148
|
+
|
149
|
+
# Time stamp when experiment was completed.
|
150
|
+
def completed_at
|
151
|
+
@completed_at ||= connection.get_experiment_completed_at(@id)
|
152
|
+
end
|
153
|
+
|
154
|
+
# Returns true if experiment active, false if completed.
|
155
|
+
def active?
|
156
|
+
!@playground.collecting? || !connection.is_experiment_completed?(@id)
|
157
|
+
end
|
158
|
+
|
159
|
+
# -- Store/validate --
|
160
|
+
|
161
|
+
# Get rid of all experiment data.
|
162
|
+
def destroy
|
163
|
+
connection.destroy_experiment @id
|
164
|
+
@created_at = @completed_at = nil
|
165
|
+
end
|
166
|
+
|
167
|
+
# Called by Playground to save the experiment definition.
|
168
|
+
def save
|
169
|
+
return unless @playground.collecting?
|
170
|
+
connection.set_experiment_created_at @id, Time.now
|
171
|
+
end
|
172
|
+
|
173
|
+
protected
|
174
|
+
|
175
|
+
def identity
|
176
|
+
@identify_block.call(Vanity.context)
|
177
|
+
end
|
178
|
+
|
179
|
+
def default_identify(context)
|
180
|
+
raise "No Vanity.context" unless context
|
181
|
+
raise "Vanity.context does not respond to vanity_identity" unless context.respond_to?(:vanity_identity)
|
182
|
+
context.vanity_identity or raise "Vanity.context.vanity_identity - no identity"
|
183
|
+
end
|
184
|
+
|
185
|
+
# Derived classes call this after state changes that may lead to
|
186
|
+
# experiment completing.
|
187
|
+
def check_completion!
|
188
|
+
if @complete_block
|
189
|
+
begin
|
190
|
+
complete! if @complete_block.call
|
191
|
+
rescue
|
192
|
+
# TODO: logging
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# Returns key for this experiment, or with an argument, return a key
|
198
|
+
# using the experiment as the namespace. Examples:
|
199
|
+
# key => "vanity:experiments:green_button"
|
200
|
+
# key("participants") => "vanity:experiments:green_button:participants"
|
201
|
+
def key(name = nil)
|
202
|
+
"#{@id}:#{name}"
|
203
|
+
end
|
204
|
+
|
205
|
+
# Shortcut for Vanity.playground.connection
|
206
|
+
def connection
|
207
|
+
@playground.connection
|
208
|
+
end
|
209
|
+
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
@@ -0,0 +1,244 @@
|
|
1
|
+
module Vanity
|
2
|
+
module Rails #:nodoc:
|
3
|
+
def self.load!
|
4
|
+
Vanity.playground.load_path = ::Rails.root + Vanity.playground.load_path
|
5
|
+
Vanity.playground.logger ||= ::Rails.logger
|
6
|
+
|
7
|
+
# Do this at the very end of initialization, allowing you to change
|
8
|
+
# connection adapter, turn collection on/off, etc.
|
9
|
+
::Rails.configuration.after_initialize do
|
10
|
+
Vanity.playground.load!
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# The use_vanity method will setup the controller to allow testing and
|
15
|
+
# tracking of the current user.
|
16
|
+
module UseVanity
|
17
|
+
# Defines the vanity_identity method and the set_identity_context filter.
|
18
|
+
#
|
19
|
+
# Call with the name of a method that returns an object whose identity
|
20
|
+
# will be used as the Vanity identity. Confusing? Let's try by example:
|
21
|
+
#
|
22
|
+
# class ApplicationController < ActionController::Base
|
23
|
+
# use_vanity :current_user
|
24
|
+
#
|
25
|
+
# def current_user
|
26
|
+
# User.find(session[:user_id])
|
27
|
+
# end
|
28
|
+
# end
|
29
|
+
#
|
30
|
+
# If that method (current_user in this example) returns nil, Vanity will
|
31
|
+
# set the identity for you (using a cookie to remember it across
|
32
|
+
# requests). It also uses this mechanism if you don't provide an
|
33
|
+
# identity object, by calling use_vanity with no arguments.
|
34
|
+
#
|
35
|
+
# Of course you can also use a block:
|
36
|
+
# class ProjectController < ApplicationController
|
37
|
+
# use_vanity { |controller| controller.params[:project_id] }
|
38
|
+
# end
|
39
|
+
def use_vanity(symbol = nil, &block)
|
40
|
+
if block
|
41
|
+
define_method(:vanity_identity) { block.call(self) }
|
42
|
+
else
|
43
|
+
define_method :vanity_identity do
|
44
|
+
return @vanity_identity if @vanity_identity
|
45
|
+
if symbol && object = send(symbol)
|
46
|
+
@vanity_identity = object.id
|
47
|
+
elsif response # everyday use
|
48
|
+
@vanity_identity = cookies["vanity_id"] || ActiveSupport::SecureRandom.hex(16)
|
49
|
+
cookies["vanity_id"] = { :value=>@vanity_identity, :expires=>1.month.from_now }
|
50
|
+
@vanity_identity
|
51
|
+
else # during functional testing
|
52
|
+
@vanity_identity = "test"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
around_filter :vanity_context_filter
|
57
|
+
before_filter :vanity_reload_filter unless ::Rails.configuration.cache_classes
|
58
|
+
before_filter :vanity_query_parameter_filter
|
59
|
+
end
|
60
|
+
protected :use_vanity
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
# Vanity needs these filters. They are includes in ActionController and
|
65
|
+
# automatically added when you use #use_vanity in your controller.
|
66
|
+
module Filters
|
67
|
+
# Around filter that sets Vanity.context to controller.
|
68
|
+
def vanity_context_filter
|
69
|
+
previous, Vanity.context = Vanity.context, self
|
70
|
+
yield
|
71
|
+
ensure
|
72
|
+
Vanity.context = previous
|
73
|
+
end
|
74
|
+
|
75
|
+
# This filter allows user to choose alternative in experiment using query
|
76
|
+
# parameter.
|
77
|
+
#
|
78
|
+
# Each alternative has a unique fingerprint (run vanity list command to
|
79
|
+
# see them all). A request with the _vanity query parameter is
|
80
|
+
# intercepted, the alternative is chosen, and the user redirected to the
|
81
|
+
# same request URL sans _vanity parameter. This only works for GET
|
82
|
+
# requests.
|
83
|
+
#
|
84
|
+
# For example, if the user requests the page
|
85
|
+
# http://example.com/?_vanity=2907dac4de, the first alternative of the
|
86
|
+
# :null_abc experiment is chosen and the user redirected to
|
87
|
+
# http://example.com/.
|
88
|
+
def vanity_query_parameter_filter
|
89
|
+
if request.get? && params[:_vanity]
|
90
|
+
hashes = Array(params.delete(:_vanity))
|
91
|
+
Vanity.playground.experiments.each do |id, experiment|
|
92
|
+
if experiment.respond_to?(:alternatives)
|
93
|
+
experiment.alternatives.each do |alt|
|
94
|
+
if hash = hashes.delete(experiment.fingerprint(alt))
|
95
|
+
experiment.chooses alt.value
|
96
|
+
break
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
break if hashes.empty?
|
101
|
+
end
|
102
|
+
redirect_to url_for(params)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# Before filter to reload Vanity experiments/metrics. Enabled when
|
107
|
+
# cache_classes is false (typically, testing environment).
|
108
|
+
def vanity_reload_filter
|
109
|
+
Vanity.playground.reload!
|
110
|
+
end
|
111
|
+
|
112
|
+
protected :vanity_context_filter, :vanity_query_parameter_filter, :vanity_reload_filter
|
113
|
+
end
|
114
|
+
|
115
|
+
|
116
|
+
# Introduces ab_test helper (controllers and views). Similar to the generic
|
117
|
+
# ab_test method, with the ability to capture content (applicable to views,
|
118
|
+
# see examples).
|
119
|
+
module Helpers
|
120
|
+
# This method returns one of the alternative values in the named A/B test.
|
121
|
+
#
|
122
|
+
# @example A/B two alternatives for a page
|
123
|
+
# def index
|
124
|
+
# if ab_test(:new_page) # true/false test
|
125
|
+
# render action: "new_page"
|
126
|
+
# else
|
127
|
+
# render action: "index"
|
128
|
+
# end
|
129
|
+
# end
|
130
|
+
# @example Similar, alternative value is page name
|
131
|
+
# def index
|
132
|
+
# render action: ab_test(:new_page)
|
133
|
+
# end
|
134
|
+
# @example A/B test inside ERB template (condition)
|
135
|
+
# <%= if ab_test(:banner) %>100% less complexity!<% end %>
|
136
|
+
# @example A/B test inside ERB template (value)
|
137
|
+
# <%= ab_test(:greeting) %> <%= current_user.name %>
|
138
|
+
# @example A/B test inside ERB template (capture)
|
139
|
+
# <% ab_test :features do |count| %>
|
140
|
+
# <%= count %> features to choose from!
|
141
|
+
# <% end %>
|
142
|
+
def ab_test(name, &block)
|
143
|
+
value = Vanity.playground.experiment(name).choose
|
144
|
+
if block
|
145
|
+
content = capture(value, &block)
|
146
|
+
block_called_from_erb?(block) ? concat(content) : content
|
147
|
+
else
|
148
|
+
value
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def vanity_h(text)
|
153
|
+
h(text)
|
154
|
+
end
|
155
|
+
|
156
|
+
def vanity_html_safe(text)
|
157
|
+
if text.respond_to?(:html_safe)
|
158
|
+
text.html_safe
|
159
|
+
else
|
160
|
+
text
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def vanity_simple_format(text, html_options={})
|
165
|
+
vanity_html_safe(simple_format(text, html_options))
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
|
170
|
+
# Step 1: Add a new resource in config/routes.rb:
|
171
|
+
# map.vanity "/vanity/:action/:id", :controller=>:vanity
|
172
|
+
#
|
173
|
+
# Step 2: Create a new experiments controller:
|
174
|
+
# class VanityController < ApplicationController
|
175
|
+
# include Vanity::Rails::Dashboard
|
176
|
+
# end
|
177
|
+
#
|
178
|
+
# Step 3: Open your browser to http://localhost:3000/vanity
|
179
|
+
module Dashboard
|
180
|
+
def index
|
181
|
+
render :file=>Vanity.template("_report"), :content_type=>Mime::HTML, :layout=>false
|
182
|
+
end
|
183
|
+
|
184
|
+
def chooses
|
185
|
+
exp = Vanity.playground.experiment(params[:e])
|
186
|
+
exp.chooses(exp.alternatives[params[:a].to_i].value)
|
187
|
+
render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
# Enhance ActionController with use_vanity, filters and helper methods.
|
195
|
+
if defined?(ActionController)
|
196
|
+
# Include in controller, add view helper methods.
|
197
|
+
ActionController::Base.class_eval do
|
198
|
+
extend Vanity::Rails::UseVanity
|
199
|
+
include Vanity::Rails::Filters
|
200
|
+
helper Vanity::Rails::Helpers
|
201
|
+
end
|
202
|
+
|
203
|
+
module ActionController
|
204
|
+
class TestCase
|
205
|
+
alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
|
206
|
+
# Sets Vanity.context to the current controller, so you can do things like:
|
207
|
+
# experiment(:simple).chooses(:green)
|
208
|
+
def setup_controller_request_and_response
|
209
|
+
setup_controller_request_and_response_without_vanity
|
210
|
+
Vanity.context = @controller
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
|
217
|
+
# Automatically configure Vanity.
|
218
|
+
if defined?(Rails)
|
219
|
+
if Rails.const_defined?(:Railtie) # Rails 3
|
220
|
+
class Plugin < Rails::Railtie # :nodoc:
|
221
|
+
initializer "vanity.require" do |app|
|
222
|
+
Vanity::Rails.load!
|
223
|
+
end
|
224
|
+
end
|
225
|
+
else
|
226
|
+
Rails.configuration.after_initialize do
|
227
|
+
Vanity::Rails.load!
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
|
233
|
+
# Reconnect whenever we fork under Passenger.
|
234
|
+
if defined?(PhusionPassenger)
|
235
|
+
PhusionPassenger.on_event(:starting_worker_process) do |forked|
|
236
|
+
if forked
|
237
|
+
begin
|
238
|
+
Vanity.playground.establish_connection if Vanity.playground.collecting?
|
239
|
+
rescue Exception=>ex
|
240
|
+
Rails.logger.error "Error reconnecting: #{ex.to_s}"
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|