vanity 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +34 -19
- data/README.rdoc +19 -15
- data/lib/vanity/commands/report.rb +11 -4
- data/lib/vanity/experiment/ab_test.rb +149 -108
- data/lib/vanity/experiment/base.rb +42 -53
- data/lib/vanity/playground.rb +48 -24
- data/lib/vanity/rails.rb +2 -1
- data/lib/vanity/rails/dashboard.rb +15 -0
- data/lib/vanity/rails/helpers.rb +19 -32
- data/lib/vanity/rails/testing.rb +3 -1
- data/lib/vanity/templates/_ab_test.erb +7 -5
- data/lib/vanity/templates/_experiment.erb +5 -0
- data/lib/vanity/templates/_experiments.erb +2 -7
- data/lib/vanity/templates/_report.erb +14 -14
- data/lib/vanity/templates/vanity.css +13 -0
- data/test/ab_test_test.rb +147 -110
- data/test/experiment_test.rb +15 -22
- data/test/experiments/age_and_zipcode.rb +17 -2
- data/test/experiments/null_abc.rb +1 -1
- data/test/playground_test.rb +53 -31
- data/test/rails_test.rb +1 -1
- data/test/test_helper.rb +2 -0
- data/vanity.gemspec +2 -2
- metadata +7 -6
- data/lib/vanity/rails/console.rb +0 -14
- data/lib/vanity/templates/_vanity.css +0 -13
@@ -14,65 +14,61 @@ module Vanity
|
|
14
14
|
|
15
15
|
end
|
16
16
|
|
17
|
-
def initialize(playground, id, name, &block)
|
17
|
+
def initialize(playground, id, name, options, &block)
|
18
18
|
@playground = playground
|
19
19
|
@id, @name = id.to_sym, name
|
20
|
+
@options = options || {}
|
20
21
|
@namespace = "#{@playground.namespace}:#{@id}"
|
21
22
|
@identify_block = ->(context){ context.vanity_identity }
|
22
23
|
end
|
23
24
|
|
24
|
-
# Human readable experiment name
|
25
|
+
# Human readable experiment name (first argument you pass when creating a
|
26
|
+
# new experiment).
|
25
27
|
attr_reader :name
|
26
28
|
|
27
|
-
# Unique identifier, derived from name, e.g. "Green
|
29
|
+
# Unique identifier, derived from name experiment name, e.g. "Green
|
30
|
+
# Button" becomes :green_button.
|
28
31
|
attr_reader :id
|
29
|
-
|
30
|
-
#
|
32
|
+
|
33
|
+
# Time stamp when experiment was created.
|
31
34
|
attr_reader :created_at
|
32
35
|
|
33
|
-
#
|
36
|
+
# Time stamp when experiment was completed.
|
34
37
|
attr_reader :completed_at
|
35
38
|
|
36
|
-
# Returns the type of this
|
39
|
+
# Returns the type of this experiment as a symbol (e.g. :ab_test).
|
37
40
|
def type
|
38
41
|
self.class.type
|
39
42
|
end
|
40
43
|
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
#
|
45
|
-
# For example, this experiment use the identity of the project associated
|
46
|
-
# with the controller:
|
47
|
-
#
|
48
|
-
# class ProjectController < ApplicationController
|
49
|
-
# before_filter :set_project
|
50
|
-
# attr_reader :project
|
44
|
+
# Defines how we obtain an identity for the current experiment. Usually
|
45
|
+
# Vanity gets the identity form a session object (see use_vanity), but
|
46
|
+
# there are cases where you want a particular experiment to use a
|
47
|
+
# different identity.
|
51
48
|
#
|
52
|
-
#
|
53
|
-
#
|
54
|
-
#
|
55
|
-
# experiment "Project widget" do
|
49
|
+
# For example, if all your experiments use current_user and you need one
|
50
|
+
# experiment to use the current project:
|
51
|
+
# ab_test "Project widget" do
|
56
52
|
# alternatives :small, :medium, :large
|
57
53
|
# identify do |controller|
|
58
54
|
# controller.project.id
|
59
55
|
# end
|
60
56
|
# end
|
61
57
|
def identify(&block)
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
end
|
58
|
+
@identify_block = block
|
59
|
+
end
|
60
|
+
|
61
|
+
def identity
|
62
|
+
@identify_block.call(Vanity.context) or fail "No identity found"
|
68
63
|
end
|
64
|
+
protected :identity
|
69
65
|
|
70
66
|
|
71
67
|
# -- Reporting --
|
72
68
|
|
73
69
|
# Sets or returns description. For example
|
74
|
-
#
|
75
|
-
# description "
|
70
|
+
# ab_test "Simple" do
|
71
|
+
# description "A simple A/B experiment"
|
76
72
|
# end
|
77
73
|
#
|
78
74
|
# puts "Just defined: " + experiment(:simple).description
|
@@ -90,8 +86,7 @@ module Vanity
|
|
90
86
|
|
91
87
|
# Define experiment completion condition. For example:
|
92
88
|
# complete_if do
|
93
|
-
#
|
94
|
-
# alternatives.any? { |alt| alt.confidence >= 0.95 }
|
89
|
+
# !score(95).chosen.nil?
|
95
90
|
# end
|
96
91
|
def complete_if(&block)
|
97
92
|
raise ArgumentError, "Missing block" unless block
|
@@ -129,41 +124,35 @@ module Vanity
|
|
129
124
|
redis[key(:completed_at)].nil?
|
130
125
|
end
|
131
126
|
|
132
|
-
|
133
127
|
# -- Store/validate --
|
134
128
|
|
135
|
-
#
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
def key(name = nil) #:nodoc:
|
140
|
-
name ? "#{@namespace}:#{name}" : @namespace
|
129
|
+
# Get rid of all experiment data.
|
130
|
+
def destroy
|
131
|
+
redis.del key(:created_at)
|
132
|
+
redis.del key(:completed_at)
|
141
133
|
end
|
142
134
|
|
143
|
-
# Shortcut for Vanity.playground.redis
|
144
|
-
def redis #:nodoc:
|
145
|
-
@playground.redis
|
146
|
-
end
|
147
|
-
|
148
135
|
# Called by Playground to save the experiment definition.
|
149
136
|
def save
|
150
137
|
redis.setnx key(:created_at), Time.now.to_i
|
151
138
|
@created_at = Time.at(redis[key(:created_at)].to_i)
|
152
139
|
end
|
153
140
|
|
154
|
-
|
155
|
-
def reset!
|
156
|
-
@created_at = Time.now
|
157
|
-
redis[key(:created_at)] = @created_at.to_i
|
158
|
-
redis.del key(:completed_at)
|
159
|
-
end
|
141
|
+
protected
|
160
142
|
|
161
|
-
#
|
162
|
-
|
163
|
-
|
164
|
-
|
143
|
+
# Returns key for this experiment, or with an argument, return a key
|
144
|
+
# using the experiment as the namespace. Examples:
|
145
|
+
# key => "vanity:experiments:green_button"
|
146
|
+
# key("participants") => "vanity:experiments:green_button:participants"
|
147
|
+
def key(name = nil)
|
148
|
+
name ? "#{@namespace}:#{name}" : @namespace
|
165
149
|
end
|
166
150
|
|
151
|
+
# Shortcut for Vanity.playground.redis
|
152
|
+
def redis
|
153
|
+
@playground.redis
|
154
|
+
end
|
155
|
+
|
167
156
|
end
|
168
157
|
end
|
169
158
|
end
|
data/lib/vanity/playground.rb
CHANGED
@@ -1,7 +1,21 @@
|
|
1
1
|
module Vanity
|
2
2
|
|
3
|
-
#
|
4
|
-
|
3
|
+
# These methods are available from experiment definitions (files located in
|
4
|
+
# the experiments directory, automatically loaded by Vanity). Use these
|
5
|
+
# methods to define you experiments, for example:
|
6
|
+
# ab_test "New Banner" do
|
7
|
+
# alternatives :red, :green, :blue
|
8
|
+
# end
|
9
|
+
module Definition
|
10
|
+
|
11
|
+
protected
|
12
|
+
# Defines a new experiment, given the experiment's name, type and
|
13
|
+
# definition block.
|
14
|
+
def define(name, type, options = nil, &block)
|
15
|
+
options ||= {}
|
16
|
+
Vanity.playground.define(name, type, options, &block)
|
17
|
+
end
|
18
|
+
|
5
19
|
end
|
6
20
|
|
7
21
|
# Playground catalogs all your experiments, holds the Vanity configuration.
|
@@ -40,14 +54,12 @@ module Vanity
|
|
40
54
|
attr_accessor :logger
|
41
55
|
|
42
56
|
# Defines a new experiment. Generally, do not call this directly,
|
43
|
-
# use
|
44
|
-
def define(name, options =
|
57
|
+
# use one of the definition methods (ab_test, measure, etc).
|
58
|
+
def define(name, type, options = {}, &block)
|
45
59
|
id = name.to_s.downcase.gsub(/\W/, "_")
|
46
60
|
raise "Experiment #{id} already defined once" if @experiments[id]
|
47
|
-
options ||= {}
|
48
|
-
type = options[:type] || :ab_test
|
49
61
|
klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
|
50
|
-
experiment = klass.new(self, id, name)
|
62
|
+
experiment = klass.new(self, id, name, options)
|
51
63
|
experiment.instance_eval &block
|
52
64
|
experiment.save
|
53
65
|
@experiments[id] = experiment
|
@@ -62,7 +74,23 @@ module Vanity
|
|
62
74
|
def experiment(name)
|
63
75
|
id = name.to_s.downcase.gsub(/\W/, "_")
|
64
76
|
unless @experiments.has_key?(id)
|
65
|
-
|
77
|
+
@loading ||= []
|
78
|
+
fail "Circular dependency detected: #{@loading.join('=>')}=>#{id}" if @loading.include?(id)
|
79
|
+
begin
|
80
|
+
@loading.push id
|
81
|
+
source = File.read(File.expand_path("#{id}.rb", load_path))
|
82
|
+
context = Object.new
|
83
|
+
context.instance_eval do
|
84
|
+
extend Definition
|
85
|
+
eval source
|
86
|
+
end
|
87
|
+
rescue
|
88
|
+
error = LoadError.exception($!.message)
|
89
|
+
error.set_backtrace $!.backtrace
|
90
|
+
raise error
|
91
|
+
ensure
|
92
|
+
@loading.pop
|
93
|
+
end
|
66
94
|
end
|
67
95
|
@experiments[id] or fail LoadError, "Expected experiments/#{id}.rb to define experiment #{name}"
|
68
96
|
end
|
@@ -70,11 +98,17 @@ module Vanity
|
|
70
98
|
# Returns list of all loaded experiments.
|
71
99
|
def experiments
|
72
100
|
Dir[File.join(load_path, "*.rb")].each do |file|
|
73
|
-
|
101
|
+
id = File.basename(file).gsub(/.rb$/, "")
|
102
|
+
experiment id
|
74
103
|
end
|
75
104
|
@experiments.values
|
76
105
|
end
|
77
106
|
|
107
|
+
# Reloads all experiments.
|
108
|
+
def reload!
|
109
|
+
@experiments.clear
|
110
|
+
end
|
111
|
+
|
78
112
|
# Use this instance to access the Redis database.
|
79
113
|
def redis
|
80
114
|
redis = Redis.new(host: self.host, port: self.port, db: self.db,
|
@@ -100,7 +134,7 @@ module Vanity
|
|
100
134
|
end
|
101
135
|
|
102
136
|
# Sets the Vanity context. For example, when using Rails this would be
|
103
|
-
# set by the set_vanity_context before filter (via use_vanity).
|
137
|
+
# set by the set_vanity_context before filter (via Vanity::Rails#use_vanity).
|
104
138
|
def context=(context)
|
105
139
|
Thread.current[:vanity_context] = context
|
106
140
|
end
|
@@ -115,22 +149,12 @@ module Vanity
|
|
115
149
|
end
|
116
150
|
end
|
117
151
|
|
152
|
+
|
118
153
|
class Object
|
119
154
|
|
120
|
-
# Use this method to
|
121
|
-
#
|
122
|
-
# To define an experiment, call with a name, options and a block. For
|
123
|
-
# example:
|
124
|
-
# experiment "Text size" do
|
125
|
-
# alternatives :small, :medium, :large
|
126
|
-
# end
|
127
|
-
#
|
155
|
+
# Use this method to access an experiment by name. For example:
|
128
156
|
# puts experiment(:text_size).alternatives
|
129
|
-
def experiment(name
|
130
|
-
|
131
|
-
Vanity.playground.define(name, options, &block)
|
132
|
-
else
|
133
|
-
Vanity.playground.experiment(name)
|
134
|
-
end
|
157
|
+
def experiment(name)
|
158
|
+
Vanity.playground.experiment(name)
|
135
159
|
end
|
136
160
|
end
|
data/lib/vanity/rails.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require "vanity"
|
2
2
|
require "vanity/rails/helpers"
|
3
3
|
require "vanity/rails/testing"
|
4
|
-
require "vanity/rails/
|
4
|
+
require "vanity/rails/dashboard"
|
5
5
|
|
6
6
|
# Use Rails logger by default.
|
7
7
|
Vanity.playground.logger ||= ActionController::Base.logger
|
@@ -9,6 +9,7 @@ Vanity.playground.load_path = "#{RAILS_ROOT}/experiments"
|
|
9
9
|
|
10
10
|
# Include in controller, add view helper methods.
|
11
11
|
ActionController::Base.class_eval do
|
12
|
+
extend Vanity::Rails::ClassMethods
|
12
13
|
include Vanity::Rails
|
13
14
|
helper Vanity::Rails
|
14
15
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Vanity
|
2
|
+
module Rails
|
3
|
+
module Dashboard
|
4
|
+
def index
|
5
|
+
render Vanity.template("_report"), content_type: Mime::HTML, layout: true
|
6
|
+
end
|
7
|
+
|
8
|
+
def chooses
|
9
|
+
exp = Vanity.playground.experiment(params[:e])
|
10
|
+
exp.chooses(exp.alternatives[params[:a].to_i].value)
|
11
|
+
render partial: Vanity.template("experiment"), locals: { experiment: exp }
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/vanity/rails/helpers.rb
CHANGED
@@ -14,43 +14,33 @@ module Vanity
|
|
14
14
|
# 3) Measure conversion:
|
15
15
|
#
|
16
16
|
# def signup
|
17
|
-
#
|
17
|
+
# track! :pricing
|
18
18
|
# . . .
|
19
19
|
# end
|
20
20
|
module Rails
|
21
21
|
module ClassMethods
|
22
22
|
|
23
|
-
# Defines the vanity_identity method
|
24
|
-
# filter.
|
23
|
+
# Defines the vanity_identity method and the set_identity_context filter.
|
25
24
|
#
|
26
|
-
#
|
27
|
-
# the
|
28
|
-
#
|
29
|
-
# identity, group, project. The object must provide its identity in
|
30
|
-
# response to the method +id+.
|
31
|
-
#
|
32
|
-
# For example, if +current_user+ returns a +User+ object, then to use the
|
33
|
-
# user's id:
|
25
|
+
# Call with the name of a method that returns an object whose identity
|
26
|
+
# will be used as the Vanity identity. Confusing? Let's try by example:
|
27
|
+
#
|
34
28
|
# class ApplicationController < ActionController::Base
|
35
29
|
# use_vanity :current_user
|
36
|
-
# end
|
37
|
-
#
|
38
|
-
# If that method returns nil (e.g. the user has not signed in), a random
|
39
|
-
# value will be used, instead. That random value is maintained using a
|
40
|
-
# cookie.
|
41
|
-
#
|
42
|
-
# Alternatively, if you call use_vanity with a block, it will yield to the
|
43
|
-
# block with controller.
|
44
|
-
#
|
45
|
-
# If there is no identity you can use, call use_vanity with the value +nil+.
|
46
30
|
#
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
31
|
+
# def current_user
|
32
|
+
# User.find(session[:user_id])
|
33
|
+
# end
|
50
34
|
# end
|
35
|
+
#
|
36
|
+
# If that method (current_user in this example) returns nil, Vanity will
|
37
|
+
# set the identity for you (using a cookie to remember it across
|
38
|
+
# requests). It also uses this mechanism if you don't provide an
|
39
|
+
# identity object, by calling use_vanity with no arguments.
|
51
40
|
#
|
41
|
+
# Of course you can also use a block:
|
52
42
|
# class ProjectController < ApplicationController
|
53
|
-
# use_vanity { |controller| controller.project_id }
|
43
|
+
# use_vanity { |controller| controller.params[:project_id] }
|
54
44
|
# end
|
55
45
|
def use_vanity(symbol = nil, &block)
|
56
46
|
define_method :vanity_identity do
|
@@ -71,13 +61,10 @@ module Vanity
|
|
71
61
|
Vanity.context = self
|
72
62
|
end
|
73
63
|
before_filter :set_vanity_context
|
64
|
+
before_filter { Vanity.playground.reload! } unless ::Rails.configuration.cache_classes
|
74
65
|
end
|
75
66
|
end
|
76
67
|
|
77
|
-
def self.included(base) #:nodoc:
|
78
|
-
base.extend ClassMethods
|
79
|
-
end
|
80
|
-
|
81
68
|
# This method returns one of the alternative values in the named A/B test.
|
82
69
|
#
|
83
70
|
# Examples using ab_test inside controller:
|
@@ -113,11 +100,11 @@ module Vanity
|
|
113
100
|
|
114
101
|
# This method records conversion on the named A/B test. For example:
|
115
102
|
# def create
|
116
|
-
#
|
103
|
+
# track! :call_to_action
|
117
104
|
# Acccount.create! params[:account]
|
118
105
|
# end
|
119
|
-
def
|
120
|
-
Vanity.playground.experiment(name).
|
106
|
+
def track!(name, *args)
|
107
|
+
Vanity.playground.experiment(name).track! *args
|
121
108
|
end
|
122
109
|
end
|
123
110
|
end
|
data/lib/vanity/rails/testing.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
module ActionController
|
1
|
+
module ActionController
|
2
2
|
class TestCase
|
3
3
|
alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
|
4
|
+
# Sets Vanity.context to the current controller, so you can do things like:
|
5
|
+
# experiment(:simple).chooses(:green)
|
4
6
|
def setup_controller_request_and_response
|
5
7
|
setup_controller_request_and_response_without_vanity
|
6
8
|
Vanity.context = @controller
|
@@ -1,21 +1,23 @@
|
|
1
1
|
<% score = experiment.score %>
|
2
2
|
<table>
|
3
|
-
<caption
|
3
|
+
<caption>
|
4
|
+
<%= experiment.conclusion(score).join(" ") %></caption>
|
4
5
|
<% score.alts.each do |alt| %>
|
5
6
|
<tr class="<%= "choice" if score.choice == alt %>">
|
6
7
|
<td class="option"><%= alt.name.gsub(/^o/, "O") %>:</td>
|
7
|
-
<td class="value"><code><%=
|
8
|
+
<td class="value"><code><%=h alt.value.to_s %></code></td>
|
8
9
|
<td>
|
9
10
|
<%= "%.1f%%" % [alt.conversion_rate * 100] %>
|
10
11
|
<%= "(%d%% better than %s)" % [alt.difference, score.least.name] if alt.difference && alt.difference >= 1 %>
|
11
12
|
</td>
|
12
13
|
<td class="action">
|
13
14
|
<% if experiment.active? && respond_to?(:chooses_experiments_url) %>
|
14
|
-
<% if experiment.
|
15
|
+
<% if experiment.showing?(alt) %>
|
15
16
|
showing
|
16
17
|
<% else %>
|
17
|
-
<%=
|
18
|
-
|
18
|
+
<%= link_to_remote "show", update: "experiment_#{experiment.id}",
|
19
|
+
url: chooses_experiments_url(e: experiment.id, a: alt.id), method: :post,
|
20
|
+
html: { class: "button", title: "Show me this alternative from now on" } %>
|
19
21
|
<% end %>
|
20
22
|
<% end %>
|
21
23
|
</td>
|
@@ -0,0 +1,5 @@
|
|
1
|
+
<h2><%=h experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
|
2
|
+
<%= experiment.description.to_s.split(/\n\s*\n/).map { |para| %{<p class="description">#{h para}</p>} }.join %>
|
3
|
+
<%= render Vanity.template(experiment.type), experiment: experiment %>
|
4
|
+
<p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d") %>
|
5
|
+
<%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d")}" unless experiment.active? %></p>
|