vanity 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +34 -0
- data/Gemfile +16 -0
- data/MIT-LICENSE +1 -1
- data/README.rdoc +10 -5
- data/Rakefile +119 -0
- data/bin/vanity +23 -18
- data/lib/vanity.rb +12 -4
- data/lib/vanity/commands.rb +1 -0
- data/lib/vanity/commands/list.rb +21 -0
- data/lib/vanity/experiment/ab_test.rb +8 -1
- data/lib/vanity/experiment/base.rb +40 -30
- data/lib/vanity/frameworks/rails.rb +222 -0
- data/lib/vanity/metric/active_record.rb +77 -0
- data/lib/vanity/{metric.rb → metric/base.rb} +6 -71
- data/lib/vanity/metric/google_analytics.rb +76 -0
- data/lib/vanity/playground.rb +93 -44
- data/lib/vanity/templates/_metric.erb +12 -7
- data/lib/vanity/templates/vanity.css +1 -0
- data/test/ab_test_test.rb +69 -48
- data/test/experiment_test.rb +29 -15
- data/test/metric_test.rb +104 -0
- data/test/myapp/app/controllers/application_controller.rb +2 -0
- data/test/myapp/app/controllers/main_controller.rb +7 -0
- data/test/myapp/config/boot.rb +110 -0
- data/test/myapp/config/environment.rb +10 -0
- data/test/myapp/config/environments/production.rb +0 -0
- data/test/myapp/config/routes.rb +3 -0
- data/test/myapp/log/production.log +80 -0
- data/test/passenger_test.rb +34 -0
- data/test/rails_test.rb +129 -1
- data/test/test_helper.rb +12 -4
- data/vanity.gemspec +2 -2
- data/vendor/cache/RedCloth-4.2.2.gem +0 -0
- data/vendor/cache/actionmailer-2.3.5.gem +0 -0
- data/vendor/cache/actionpack-2.3.5.gem +0 -0
- data/vendor/cache/activerecord-2.3.5.gem +0 -0
- data/vendor/cache/activeresource-2.3.5.gem +0 -0
- data/vendor/cache/activesupport-2.3.5.gem +0 -0
- data/vendor/cache/autotest-4.2.7.gem +0 -0
- data/vendor/cache/autotest-fsevent-0.2.1.gem +0 -0
- data/vendor/cache/autotest-growl-0.2.0.gem +0 -0
- data/vendor/cache/bundler-0.9.7.gem +0 -0
- data/vendor/cache/classifier-1.3.1.gem +0 -0
- data/vendor/cache/directory_watcher-1.3.1.gem +0 -0
- data/vendor/cache/fastthread-1.0.7.gem +0 -0
- data/vendor/cache/garb-0.7.0.gem +0 -0
- data/vendor/cache/happymapper-0.3.0.gem +0 -0
- data/vendor/cache/jekyll-0.5.7.gem +0 -0
- data/vendor/cache/libxml-ruby-1.1.3.gem +0 -0
- data/vendor/cache/liquid-2.0.0.gem +0 -0
- data/vendor/cache/maruku-0.6.0.gem +0 -0
- data/vendor/cache/mocha-0.9.8.gem +0 -0
- data/vendor/cache/open4-1.0.1.gem +0 -0
- data/vendor/cache/passenger-2.2.9.gem +0 -0
- data/vendor/cache/rack-1.0.1.gem +0 -0
- data/vendor/cache/rails-2.3.5.gem +0 -0
- data/vendor/cache/rake-0.8.7.gem +0 -0
- data/vendor/cache/rubygems-update-1.3.5.gem +0 -0
- data/vendor/cache/shoulda-2.10.3.gem +0 -0
- data/vendor/cache/sqlite3-ruby-1.2.5.gem +0 -0
- data/vendor/cache/stemmer-1.0.1.gem +0 -0
- data/vendor/cache/syntax-1.0.0.gem +0 -0
- data/vendor/cache/sys-uname-0.8.4.gem +0 -0
- data/vendor/cache/timecop-0.3.4.gem +0 -0
- metadata +60 -11
- data/lib/vanity/rails.rb +0 -22
- data/lib/vanity/rails/dashboard.rb +0 -24
- data/lib/vanity/rails/helpers.rb +0 -101
- data/lib/vanity/rails/testing.rb +0 -11
data/lib/vanity/playground.rb
CHANGED
@@ -7,35 +7,45 @@ module Vanity
|
|
7
7
|
# puts Vanity.playground.map(&:name)
|
8
8
|
class Playground
|
9
9
|
|
10
|
-
DEFAULTS = {
|
10
|
+
DEFAULTS = {
|
11
|
+
:connection=>"localhost:6379",
|
12
|
+
:load_path=>"experiments",
|
13
|
+
:namespace=>"vanity:#{Vanity::Version::MAJOR}"
|
14
|
+
}
|
15
|
+
|
16
|
+
# Created new Playground. Unless you need to, use the global
|
17
|
+
# Vanity.playground.
|
18
|
+
#
|
19
|
+
# First argument is connection specification (see #redis=), last argument is
|
20
|
+
# a set of options, both are optional. Supported options are:
|
21
|
+
# - connection -- Connection specification
|
22
|
+
# - namespace -- Namespace to use
|
23
|
+
# - load_path -- Path to load experiments/metrics from
|
24
|
+
# - logger -- Logger to use
|
25
|
+
def initialize(*args)
|
26
|
+
options = args.pop if Hash === args.last
|
27
|
+
@options = DEFAULTS.merge(options || {})
|
28
|
+
if @options.values_at(:host, :port, :db).any?
|
29
|
+
warn "Deprecated: please specify Redis connection as single argument (\"host:port\")"
|
30
|
+
@connection_spec = "#{@options[:host]}:#{@options[:port]}:#{@options[:db]}"
|
31
|
+
elsif @options[:redis]
|
32
|
+
@redis = @options[:redis]
|
33
|
+
else
|
34
|
+
@connection_spec = args.shift || @options[:connection]
|
35
|
+
end
|
11
36
|
|
12
|
-
|
13
|
-
|
14
|
-
@
|
15
|
-
@namespace = "vanity:#{Vanity::Version::MAJOR}"
|
16
|
-
@logger = options[:logger]
|
37
|
+
@namespace = @options[:namespace] || DEFAULTS[:namespace]
|
38
|
+
@load_path = @options[:load_path] || DEFAULTS[:load_path]
|
39
|
+
@logger = @options[:logger]
|
17
40
|
unless @logger
|
18
41
|
@logger = Logger.new(STDOUT)
|
19
42
|
@logger.level = Logger::ERROR
|
20
43
|
end
|
21
|
-
@redis = options[:redis]
|
22
44
|
@loading = []
|
23
45
|
end
|
24
|
-
|
25
|
-
#
|
26
|
-
attr_accessor :host
|
27
|
-
|
28
|
-
# Redis port number. Default is 6379.
|
29
|
-
attr_accessor :port
|
30
|
-
|
31
|
-
# Redis database number. Default is 0.
|
32
|
-
attr_accessor :db
|
33
|
-
|
34
|
-
# Redis database password.
|
35
|
-
attr_accessor :password
|
36
|
-
|
37
|
-
# Namespace for database keys. Default is vanity:n, where n is the major release number, e.g. vanity:1 for 1.0.3.
|
38
|
-
attr_accessor :namespace
|
46
|
+
|
47
|
+
# Deprecated. Use redis.server instead.
|
48
|
+
attr_accessor :host, :port, :db, :password, :namespace
|
39
49
|
|
40
50
|
# Path to load experiment files from.
|
41
51
|
attr_accessor :load_path
|
@@ -48,6 +58,7 @@ module Vanity
|
|
48
58
|
#
|
49
59
|
# @see Vanity::Experiment
|
50
60
|
def define(name, type, options = {}, &block)
|
61
|
+
warn "Deprecated: if you need this functionality let's make a better API"
|
51
62
|
id = name.to_s.downcase.gsub(/\W/, "_").to_sym
|
52
63
|
raise "Experiment #{id} already defined once" if experiments[id]
|
53
64
|
klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
|
@@ -64,7 +75,7 @@ module Vanity
|
|
64
75
|
def experiment(name)
|
65
76
|
id = name.to_s.downcase.gsub(/\W/, "_").to_sym
|
66
77
|
warn "Deprecated: pleae call experiment method with experiment identifier (a Ruby symbol)" unless id == name
|
67
|
-
experiments[id]
|
78
|
+
experiments[id.to_sym] or raise NameError, "No experiment #{id}"
|
68
79
|
end
|
69
80
|
|
70
81
|
# Returns hash of experiments (key is experiment id).
|
@@ -75,8 +86,7 @@ module Vanity
|
|
75
86
|
@experiments = {}
|
76
87
|
@logger.info "Vanity: loading experiments from #{load_path}"
|
77
88
|
Dir[File.join(load_path, "*.rb")].each do |file|
|
78
|
-
|
79
|
-
experiment id.to_sym
|
89
|
+
Experiment::Base.load self, @loading, file
|
80
90
|
end
|
81
91
|
end
|
82
92
|
@experiments
|
@@ -97,25 +107,6 @@ module Vanity
|
|
97
107
|
metrics
|
98
108
|
end
|
99
109
|
|
100
|
-
# Use this instance to access the Redis database.
|
101
|
-
def redis
|
102
|
-
@redis ||= Redis.new(:host=>self.host, :port=>self.port, :db=>self.db,
|
103
|
-
:password=>self.password, :logger=>self.logger)
|
104
|
-
class << self ; self ; end.send(:define_method, :redis) { @redis }
|
105
|
-
@redis
|
106
|
-
end
|
107
|
-
|
108
|
-
# Switches playground to use MockRedis instead of a live server.
|
109
|
-
# Particularly useful for testing, e.g. if you can't access Redis on your CI
|
110
|
-
# server. This method has no affect after playground accesses live Redis
|
111
|
-
# server.
|
112
|
-
#
|
113
|
-
# @example Put this in config/environments/test.rb
|
114
|
-
# config.after_initialize { Vanity.playground.mock! }
|
115
|
-
def mock!
|
116
|
-
@redis ||= MockRedis.new
|
117
|
-
end
|
118
|
-
|
119
110
|
# Returns a metric (raises NameError if no metric with that identifier).
|
120
111
|
#
|
121
112
|
# @see Vanity::Metric
|
@@ -148,6 +139,64 @@ module Vanity
|
|
148
139
|
def track!(id, count = 1)
|
149
140
|
metric(id).track! count
|
150
141
|
end
|
142
|
+
|
143
|
+
|
144
|
+
# -- Connection management --
|
145
|
+
|
146
|
+
# Tells the playground where to find Redis. Accepts one of the following:
|
147
|
+
# - "hostname:port"
|
148
|
+
# - ":port"
|
149
|
+
# - "hostname:port:db"
|
150
|
+
# - Instance of Redis connection.
|
151
|
+
def redis=(spec_or_connection)
|
152
|
+
case spec_or_connection
|
153
|
+
when String
|
154
|
+
@connection_spec = spec_or_connection
|
155
|
+
host, port, db = spec_or_connection.split(':').map { |x| x unless x.empty? }
|
156
|
+
@redis = Redis.new(:host=>host, :port=>port, :thread_safe=>true, :db=>db, :thread_safe=>true)
|
157
|
+
when Redis
|
158
|
+
@connection_spec = nil
|
159
|
+
@redis = spec_or_connection
|
160
|
+
when :mock
|
161
|
+
@connection_spec = nil
|
162
|
+
@redis = MockRedis.new
|
163
|
+
else
|
164
|
+
raise "I don't know what to do with #{spec_or_connection.inspect}"
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def redis
|
169
|
+
self.redis = @connection_spec unless @redis
|
170
|
+
@redis
|
171
|
+
end
|
172
|
+
|
173
|
+
def connected?
|
174
|
+
!@redis.nil?
|
175
|
+
end
|
176
|
+
|
177
|
+
def disconnect!
|
178
|
+
@redis.quit if connected? rescue nil
|
179
|
+
@redis = nil
|
180
|
+
end
|
181
|
+
|
182
|
+
def reconnect!
|
183
|
+
raise "Connect reconnect without connection specification" unless String === @connection_spec
|
184
|
+
disconnect! rescue nil
|
185
|
+
end
|
186
|
+
|
187
|
+
def mock!
|
188
|
+
warn "Deprecated: use Vanity.playground.test!"
|
189
|
+
test!
|
190
|
+
end
|
191
|
+
|
192
|
+
# Use this when testing to disable Redis (e.g. if your CI server doesn't
|
193
|
+
# have Redis).
|
194
|
+
#
|
195
|
+
# @example Put this in config/environments/test.rb
|
196
|
+
# config.after_initialize { Vanity.playground.test! }
|
197
|
+
def test!
|
198
|
+
self.redis = :mock
|
199
|
+
end
|
151
200
|
end
|
152
201
|
|
153
202
|
@playground = Playground.new
|
@@ -1,9 +1,14 @@
|
|
1
1
|
<h3><%=h metric.name %></h3>
|
2
2
|
<%= simple_format h(Vanity::Metric.description(metric).to_s), :class=>"description" %>
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
<%=
|
4
|
+
begin
|
5
|
+
data = Vanity::Metric.data(metric)
|
6
|
+
min, max = data.map(&:last).minmax
|
7
|
+
js = data.map { |date,value| "['#{date.to_time.httpdate}',#{value}]" }.join(",")
|
8
|
+
%{<div class="chart"></div>
|
9
|
+
<script type="text/javascript">
|
10
|
+
$(function(){Vanity.metric("#{h id.to_s}").plot([{label:"#{h metric.name}", data: [#{js}]}])})
|
11
|
+
</script>}
|
12
|
+
rescue Exception=>ex
|
13
|
+
%{<div class="error">#{h ex.message}</div>}
|
14
|
+
end %>
|
@@ -2,6 +2,7 @@
|
|
2
2
|
.vanity .meta { color: #666 }
|
3
3
|
.vanity .footer { margin-top: 2em; text-align: right; font-size: 90% }
|
4
4
|
.vanity h2, .vanity h3 { margin: 0 0 .3em 0 }
|
5
|
+
.vanity .error { color: #ff2020; border: 1px solid #ff2020; background: #fff; padding: .3em }
|
5
6
|
|
6
7
|
.vanity .experiments { list-style: none; margin: 0; padding: 0 }
|
7
8
|
.vanity .experiment { margin: 1em 0 2em 0; border-bottom: 1px solid #ddd }
|
data/test/ab_test_test.rb
CHANGED
@@ -35,23 +35,23 @@ class AbTestTest < ActionController::TestCase
|
|
35
35
|
|
36
36
|
def test_requires_at_least_two_alternatives_per_experiment
|
37
37
|
assert_raises RuntimeError do
|
38
|
-
|
38
|
+
new_ab_test :none do
|
39
39
|
alternatives []
|
40
40
|
end
|
41
41
|
end
|
42
42
|
assert_raises RuntimeError do
|
43
|
-
|
43
|
+
new_ab_test :one do
|
44
44
|
alternatives "foo"
|
45
45
|
end
|
46
46
|
end
|
47
|
-
|
47
|
+
new_ab_test :two do
|
48
48
|
alternatives "foo", "bar"
|
49
49
|
metrics :coolness
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
53
|
def test_returning_alternative_by_value
|
54
|
-
|
54
|
+
new_ab_test :abcd do
|
55
55
|
alternatives :a, :b, :c, :d
|
56
56
|
metrics :coolness
|
57
57
|
end
|
@@ -60,7 +60,7 @@ class AbTestTest < ActionController::TestCase
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def test_alternative_name
|
63
|
-
|
63
|
+
new_ab_test :abcd do
|
64
64
|
alternatives :a, :b
|
65
65
|
metrics :coolness
|
66
66
|
end
|
@@ -68,25 +68,50 @@ class AbTestTest < ActionController::TestCase
|
|
68
68
|
assert_equal "option B", experiment(:abcd).alternative(:b).name
|
69
69
|
end
|
70
70
|
|
71
|
+
def test_alternative_fingerprint_is_unique
|
72
|
+
new_ab_test :ab do
|
73
|
+
metrics :coolness
|
74
|
+
alternatives :a, :b
|
75
|
+
end
|
76
|
+
new_ab_test :cd do
|
77
|
+
metrics :coolness
|
78
|
+
alternatives :a, :b
|
79
|
+
end
|
80
|
+
fingerprints = Vanity.playground.experiments.map { |id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
|
81
|
+
assert_equal 4, fingerprints.uniq.size
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_alternative_fingerprint_is_consistent
|
85
|
+
new_ab_test :ab do
|
86
|
+
alternatives :a, :b
|
87
|
+
metrics :coolness
|
88
|
+
end
|
89
|
+
fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
|
90
|
+
fingerprints.each do |fingerprint|
|
91
|
+
assert_match /^[0-9a-f]{10}$/i, fingerprint
|
92
|
+
end
|
93
|
+
assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
|
94
|
+
end
|
95
|
+
|
71
96
|
|
72
97
|
# -- Experiment metric --
|
73
98
|
|
74
99
|
def test_explicit_metric
|
75
|
-
|
100
|
+
new_ab_test :abcd do
|
76
101
|
metrics :coolness
|
77
102
|
end
|
78
103
|
assert_equal [Vanity.playground.metric(:coolness)], experiment(:abcd).metrics
|
79
104
|
end
|
80
105
|
|
81
106
|
def test_implicit_metric
|
82
|
-
|
107
|
+
new_ab_test :abcd do
|
83
108
|
end
|
84
109
|
assert_equal [Vanity.playground.metric(:abcd)], experiment(:abcd).metrics
|
85
110
|
end
|
86
111
|
|
87
112
|
def test_metric_tracking_into_alternative
|
88
113
|
metric "Coolness"
|
89
|
-
|
114
|
+
new_ab_test :abcd do
|
90
115
|
metrics :coolness
|
91
116
|
end
|
92
117
|
Vanity.playground.track! :coolness
|
@@ -97,7 +122,7 @@ class AbTestTest < ActionController::TestCase
|
|
97
122
|
# -- Running experiment --
|
98
123
|
|
99
124
|
def test_returns_the_same_alternative_consistently
|
100
|
-
|
125
|
+
new_ab_test :foobar do
|
101
126
|
alternatives "foo", "bar"
|
102
127
|
identify { "6e98ec" }
|
103
128
|
metrics :coolness
|
@@ -110,7 +135,7 @@ class AbTestTest < ActionController::TestCase
|
|
110
135
|
end
|
111
136
|
|
112
137
|
def test_returns_different_alternatives_for_each_participant
|
113
|
-
|
138
|
+
new_ab_test :foobar do
|
114
139
|
alternatives "foo", "bar"
|
115
140
|
identify { rand }
|
116
141
|
metrics :coolness
|
@@ -122,7 +147,7 @@ class AbTestTest < ActionController::TestCase
|
|
122
147
|
|
123
148
|
def test_records_all_participants_in_each_alternative
|
124
149
|
ids = (Array.new(200) { |i| i } * 5).shuffle
|
125
|
-
|
150
|
+
new_ab_test :foobar do
|
126
151
|
alternatives "foo", "bar"
|
127
152
|
identify { ids.pop }
|
128
153
|
metrics :coolness
|
@@ -135,7 +160,7 @@ class AbTestTest < ActionController::TestCase
|
|
135
160
|
|
136
161
|
def test_records_each_converted_participant_only_once
|
137
162
|
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
138
|
-
|
163
|
+
new_ab_test :foobar do
|
139
164
|
alternatives "foo", "bar"
|
140
165
|
identify { ids.pop }
|
141
166
|
metrics :coolness
|
@@ -150,7 +175,7 @@ class AbTestTest < ActionController::TestCase
|
|
150
175
|
|
151
176
|
def test_records_conversion_only_for_participants
|
152
177
|
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
153
|
-
|
178
|
+
new_ab_test :foobar do
|
154
179
|
alternatives "foo", "bar"
|
155
180
|
identify { ids.pop }
|
156
181
|
metrics :coolness
|
@@ -166,7 +191,7 @@ class AbTestTest < ActionController::TestCase
|
|
166
191
|
|
167
192
|
|
168
193
|
def test_destroy_experiment
|
169
|
-
|
194
|
+
new_ab_test :simple do
|
170
195
|
identify { "me" }
|
171
196
|
metrics :coolness
|
172
197
|
complete_if { alternatives.map(&:converted).sum >= 1 }
|
@@ -196,7 +221,7 @@ class AbTestTest < ActionController::TestCase
|
|
196
221
|
end
|
197
222
|
|
198
223
|
def test_ab_test_chooses_in_render
|
199
|
-
|
224
|
+
new_ab_test :simple do
|
200
225
|
metrics :coolness
|
201
226
|
end
|
202
227
|
responses = Array.new(100) do
|
@@ -208,7 +233,7 @@ class AbTestTest < ActionController::TestCase
|
|
208
233
|
end
|
209
234
|
|
210
235
|
def test_ab_test_chooses_view_helper
|
211
|
-
|
236
|
+
new_ab_test :simple do
|
212
237
|
metrics :coolness
|
213
238
|
end
|
214
239
|
responses = Array.new(100) do
|
@@ -220,7 +245,7 @@ class AbTestTest < ActionController::TestCase
|
|
220
245
|
end
|
221
246
|
|
222
247
|
def test_ab_test_with_capture
|
223
|
-
|
248
|
+
new_ab_test :simple do
|
224
249
|
metrics :coolness
|
225
250
|
end
|
226
251
|
responses = Array.new(100) do
|
@@ -232,7 +257,7 @@ class AbTestTest < ActionController::TestCase
|
|
232
257
|
end
|
233
258
|
|
234
259
|
def test_ab_test_track
|
235
|
-
|
260
|
+
new_ab_test :simple do
|
236
261
|
metrics :coolness
|
237
262
|
end
|
238
263
|
responses = Array.new(100) do
|
@@ -246,7 +271,7 @@ class AbTestTest < ActionController::TestCase
|
|
246
271
|
# -- Testing with tests --
|
247
272
|
|
248
273
|
def test_with_given_choice
|
249
|
-
|
274
|
+
new_ab_test :simple do
|
250
275
|
alternatives :a, :b, :c
|
251
276
|
metrics :coolness
|
252
277
|
end
|
@@ -259,7 +284,7 @@ class AbTestTest < ActionController::TestCase
|
|
259
284
|
end
|
260
285
|
|
261
286
|
def test_which_chooses_non_existent_alternative
|
262
|
-
|
287
|
+
new_ab_test :simple do
|
263
288
|
metrics :coolness
|
264
289
|
end
|
265
290
|
assert_raises ArgumentError do
|
@@ -268,7 +293,7 @@ class AbTestTest < ActionController::TestCase
|
|
268
293
|
end
|
269
294
|
|
270
295
|
def test_chooses_cleared_with_nil
|
271
|
-
|
296
|
+
new_ab_test :simple do
|
272
297
|
identify { rand }
|
273
298
|
alternatives :a, :b, :c
|
274
299
|
metrics :coolness
|
@@ -287,7 +312,7 @@ class AbTestTest < ActionController::TestCase
|
|
287
312
|
# -- Scoring --
|
288
313
|
|
289
314
|
def test_scoring
|
290
|
-
|
315
|
+
new_ab_test :abcd do
|
291
316
|
alternatives :a, :b, :c, :d
|
292
317
|
metrics :coolness
|
293
318
|
end
|
@@ -313,7 +338,7 @@ class AbTestTest < ActionController::TestCase
|
|
313
338
|
end
|
314
339
|
|
315
340
|
def test_scoring_with_no_performers
|
316
|
-
|
341
|
+
new_ab_test :abcd do
|
317
342
|
alternatives :a, :b, :c, :d
|
318
343
|
metrics :coolness
|
319
344
|
end
|
@@ -326,7 +351,7 @@ class AbTestTest < ActionController::TestCase
|
|
326
351
|
end
|
327
352
|
|
328
353
|
def test_scoring_with_one_performer
|
329
|
-
|
354
|
+
new_ab_test :abcd do
|
330
355
|
alternatives :a, :b, :c, :d
|
331
356
|
metrics :coolness
|
332
357
|
end
|
@@ -341,7 +366,7 @@ class AbTestTest < ActionController::TestCase
|
|
341
366
|
end
|
342
367
|
|
343
368
|
def test_scoring_with_some_performers
|
344
|
-
|
369
|
+
new_ab_test :abcd do
|
345
370
|
alternatives :a, :b, :c, :d
|
346
371
|
metrics :coolness
|
347
372
|
end
|
@@ -360,7 +385,7 @@ class AbTestTest < ActionController::TestCase
|
|
360
385
|
end
|
361
386
|
|
362
387
|
def test_scoring_with_different_probability
|
363
|
-
|
388
|
+
new_ab_test :abcd do
|
364
389
|
alternatives :a, :b, :c, :d
|
365
390
|
metrics :coolness
|
366
391
|
end
|
@@ -375,7 +400,7 @@ class AbTestTest < ActionController::TestCase
|
|
375
400
|
# -- Conclusion --
|
376
401
|
|
377
402
|
def test_conclusion
|
378
|
-
|
403
|
+
new_ab_test :abcd do
|
379
404
|
alternatives :a, :b, :c, :d
|
380
405
|
metrics :coolness
|
381
406
|
end
|
@@ -398,7 +423,7 @@ Option D selected as the best alternative.
|
|
398
423
|
end
|
399
424
|
|
400
425
|
def test_conclusion_with_some_performers
|
401
|
-
|
426
|
+
new_ab_test :abcd do
|
402
427
|
alternatives :a, :b, :c, :d
|
403
428
|
metrics :coolness
|
404
429
|
end
|
@@ -416,7 +441,7 @@ Option D selected as the best alternative.
|
|
416
441
|
end
|
417
442
|
|
418
443
|
def test_conclusion_without_clear_winner
|
419
|
-
|
444
|
+
new_ab_test :abcd do
|
420
445
|
alternatives :a, :b, :c, :d
|
421
446
|
metrics :coolness
|
422
447
|
end
|
@@ -433,7 +458,7 @@ Option C did not convert.
|
|
433
458
|
end
|
434
459
|
|
435
460
|
def test_conclusion_without_close_performers
|
436
|
-
|
461
|
+
new_ab_test :abcd do
|
437
462
|
alternatives :a, :b, :c, :d
|
438
463
|
metrics :coolness
|
439
464
|
end
|
@@ -450,7 +475,7 @@ Option C did not convert.
|
|
450
475
|
end
|
451
476
|
|
452
477
|
def test_conclusion_without_equal_performers
|
453
|
-
|
478
|
+
new_ab_test :abcd do
|
454
479
|
alternatives :a, :b, :c, :d
|
455
480
|
metrics :coolness
|
456
481
|
end
|
@@ -466,7 +491,7 @@ Option C did not convert.
|
|
466
491
|
end
|
467
492
|
|
468
493
|
def test_conclusion_with_one_performers
|
469
|
-
|
494
|
+
new_ab_test :abcd do
|
470
495
|
alternatives :a, :b, :c, :d
|
471
496
|
metrics :coolness
|
472
497
|
end
|
@@ -479,7 +504,7 @@ This experiment did not run long enough to find a clear winner.
|
|
479
504
|
end
|
480
505
|
|
481
506
|
def test_conclusion_with_no_performers
|
482
|
-
|
507
|
+
new_ab_test :abcd do
|
483
508
|
alternatives :a, :b, :c, :d
|
484
509
|
metrics :coolness
|
485
510
|
end
|
@@ -493,7 +518,7 @@ This experiment did not run long enough to find a clear winner.
|
|
493
518
|
# -- Completion --
|
494
519
|
|
495
520
|
def test_completion_if
|
496
|
-
|
521
|
+
new_ab_test :simple do
|
497
522
|
identify { rand }
|
498
523
|
complete_if { true }
|
499
524
|
metrics :coolness
|
@@ -503,7 +528,7 @@ This experiment did not run long enough to find a clear winner.
|
|
503
528
|
end
|
504
529
|
|
505
530
|
def test_completion_if_fails
|
506
|
-
|
531
|
+
new_ab_test :simple do
|
507
532
|
identify { rand }
|
508
533
|
complete_if { fail }
|
509
534
|
metrics :coolness
|
@@ -514,7 +539,7 @@ This experiment did not run long enough to find a clear winner.
|
|
514
539
|
|
515
540
|
def test_completion
|
516
541
|
ids = Array.new(100) { |i| i.to_s }.shuffle
|
517
|
-
|
542
|
+
new_ab_test :simple do
|
518
543
|
identify { ids.pop }
|
519
544
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
520
545
|
metrics :coolness
|
@@ -530,7 +555,7 @@ This experiment did not run long enough to find a clear winner.
|
|
530
555
|
|
531
556
|
def test_ab_methods_after_completion
|
532
557
|
ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
|
533
|
-
|
558
|
+
new_ab_test :simple do
|
534
559
|
identify { ids.pop }
|
535
560
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
536
561
|
outcome_is { alternatives[1] }
|
@@ -559,7 +584,7 @@ This experiment did not run long enough to find a clear winner.
|
|
559
584
|
# -- Outcome --
|
560
585
|
|
561
586
|
def test_completion_outcome
|
562
|
-
|
587
|
+
new_ab_test :quick do
|
563
588
|
outcome_is { alternatives[1] }
|
564
589
|
metrics :coolness
|
565
590
|
end
|
@@ -568,7 +593,7 @@ This experiment did not run long enough to find a clear winner.
|
|
568
593
|
end
|
569
594
|
|
570
595
|
def test_outcome_is_returns_nil
|
571
|
-
|
596
|
+
new_ab_test :quick do
|
572
597
|
outcome_is { nil }
|
573
598
|
metrics :coolness
|
574
599
|
end
|
@@ -577,7 +602,7 @@ This experiment did not run long enough to find a clear winner.
|
|
577
602
|
end
|
578
603
|
|
579
604
|
def test_outcome_is_returns_something_else
|
580
|
-
|
605
|
+
new_ab_test :quick do
|
581
606
|
outcome_is { "error" }
|
582
607
|
metrics :coolness
|
583
608
|
end
|
@@ -586,7 +611,7 @@ This experiment did not run long enough to find a clear winner.
|
|
586
611
|
end
|
587
612
|
|
588
613
|
def test_outcome_is_fails
|
589
|
-
|
614
|
+
new_ab_test :quick do
|
590
615
|
outcome_is { fail }
|
591
616
|
metrics :coolness
|
592
617
|
end
|
@@ -595,7 +620,7 @@ This experiment did not run long enough to find a clear winner.
|
|
595
620
|
end
|
596
621
|
|
597
622
|
def test_outcome_choosing_best_alternative
|
598
|
-
|
623
|
+
new_ab_test :quick do
|
599
624
|
metrics :coolness
|
600
625
|
end
|
601
626
|
fake :quick, false=>[2,0], true=>10
|
@@ -604,7 +629,7 @@ This experiment did not run long enough to find a clear winner.
|
|
604
629
|
end
|
605
630
|
|
606
631
|
def test_outcome_only_performing_alternative
|
607
|
-
|
632
|
+
new_ab_test :quick do
|
608
633
|
metrics :coolness
|
609
634
|
end
|
610
635
|
fake :quick, true=>2
|
@@ -613,7 +638,7 @@ This experiment did not run long enough to find a clear winner.
|
|
613
638
|
end
|
614
639
|
|
615
640
|
def test_outcome_choosing_equal_alternatives
|
616
|
-
|
641
|
+
new_ab_test :quick do
|
617
642
|
metrics :coolness
|
618
643
|
end
|
619
644
|
fake :quick, false=>8, true=>8
|
@@ -624,10 +649,6 @@ This experiment did not run long enough to find a clear winner.
|
|
624
649
|
|
625
650
|
# -- Helper methods --
|
626
651
|
|
627
|
-
def ab_test(name, &block)
|
628
|
-
Vanity.playground.define name, :ab_test, &block
|
629
|
-
end
|
630
|
-
|
631
652
|
def fake(name, args)
|
632
653
|
experiment(name).instance_eval { fake args }
|
633
654
|
end
|