vanity 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +14 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +8 -109
- data/bin/vanity +12 -5
- data/lib/vanity.rb +5 -5
- data/lib/vanity/commands.rb +1 -1
- data/lib/vanity/commands/report.rb +20 -4
- data/lib/vanity/experiment/ab_test.rb +151 -132
- data/lib/vanity/experiment/base.rb +5 -5
- data/lib/vanity/playground.rb +24 -16
- data/lib/vanity/rails.rb +4 -3
- data/lib/vanity/rails/console.rb +14 -0
- data/lib/vanity/rails/helpers.rb +3 -1
- data/lib/vanity/rails/testing.rb +1 -1
- data/lib/vanity/templates/_ab_test.erb +25 -0
- data/lib/vanity/templates/_experiments.erb +12 -0
- data/lib/vanity/templates/_report.erb +16 -0
- data/lib/vanity/templates/_vanity.css +13 -0
- data/test/ab_test_test.rb +119 -134
- data/test/experiments/age_and_zipcode.rb +3 -0
- data/test/experiments/null_abc.rb +2 -2
- data/test/test_helper.rb +1 -3
- data/vanity.gemspec +5 -5
- metadata +13 -7
- data/lib/vanity/report.erb +0 -22
@@ -18,8 +18,6 @@ module Vanity
|
|
18
18
|
@playground = playground
|
19
19
|
@id, @name = id.to_sym, name
|
20
20
|
@namespace = "#{@playground.namespace}:#{@id}"
|
21
|
-
redis.setnx key(:created_at), Time.now.to_i
|
22
|
-
@created_at = Time.at(redis[key(:created_at)].to_i)
|
23
21
|
@identify_block = ->(context){ context.vanity_identity }
|
24
22
|
end
|
25
23
|
|
@@ -147,11 +145,13 @@ module Vanity
|
|
147
145
|
@playground.redis
|
148
146
|
end
|
149
147
|
|
150
|
-
# Called to save the experiment definition.
|
151
|
-
def save
|
148
|
+
# Called by Playground to save the experiment definition.
|
149
|
+
def save
|
150
|
+
redis.setnx key(:created_at), Time.now.to_i
|
151
|
+
@created_at = Time.at(redis[key(:created_at)].to_i)
|
152
152
|
end
|
153
153
|
|
154
|
-
# Reset experiment.
|
154
|
+
# Reset experiment to its initial state.
|
155
155
|
def reset!
|
156
156
|
@created_at = Time.now
|
157
157
|
redis[key(:created_at)] = @created_at.to_i
|
data/lib/vanity/playground.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require "active_support"
|
2
|
-
|
3
1
|
module Vanity
|
4
2
|
|
5
3
|
# Vanity.playground.configuration
|
@@ -88,23 +86,33 @@ module Vanity
|
|
88
86
|
end
|
89
87
|
|
90
88
|
@playground = Playground.new
|
91
|
-
|
92
|
-
def self.playground
|
93
|
-
@playground
|
94
|
-
end
|
89
|
+
class << self
|
95
90
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
end
|
91
|
+
# Returns the playground instance.
|
92
|
+
def playground
|
93
|
+
@playground
|
94
|
+
end
|
101
95
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
96
|
+
# Returns the Vanity context. For example, when using Rails this would be
|
97
|
+
# the current controller, which can be used to get/set the vanity identity.
|
98
|
+
def context
|
99
|
+
Thread.current[:vanity_context]
|
100
|
+
end
|
107
101
|
|
102
|
+
# Sets the Vanity context. For example, when using Rails this would be
|
103
|
+
# set by the set_vanity_context before filter (via use_vanity).
|
104
|
+
def context=(context)
|
105
|
+
Thread.current[:vanity_context] = context
|
106
|
+
end
|
107
|
+
|
108
|
+
# Path to template.
|
109
|
+
def template(name)
|
110
|
+
path = File.join(File.dirname(__FILE__), "templates/#{name}")
|
111
|
+
path << ".erb" unless name["."]
|
112
|
+
path
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
108
116
|
end
|
109
117
|
|
110
118
|
class Object
|
data/lib/vanity/rails.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "vanity"
|
2
|
+
require "vanity/rails/helpers"
|
3
|
+
require "vanity/rails/testing"
|
4
|
+
require "vanity/rails/console"
|
4
5
|
|
5
6
|
# Use Rails logger by default.
|
6
7
|
Vanity.playground.logger ||= ActionController::Base.logger
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Vanity
|
2
|
+
module Rails
|
3
|
+
module ConsoleActions
|
4
|
+
def index
|
5
|
+
render Vanity.template("_report"), content_type: Mime::HTML, layout: true
|
6
|
+
end
|
7
|
+
|
8
|
+
def chooses
|
9
|
+
experiment(params[:e]).chooses(experiment(params[:e]).alternatives[params[:a].to_i].value)
|
10
|
+
redirect_to :back
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/vanity/rails/helpers.rb
CHANGED
@@ -59,10 +59,12 @@ module Vanity
|
|
59
59
|
@vanity_identity = block.call(self)
|
60
60
|
elsif symbol && object = send(symbol)
|
61
61
|
@vanity_identity = object.id
|
62
|
-
|
62
|
+
elsif response # everyday use
|
63
63
|
@vanity_identity = cookies["vanity_id"] || OpenSSL::Random.random_bytes(16).unpack("H*")[0]
|
64
64
|
cookies["vanity_id"] = { value: @vanity_identity, expires: 1.month.from_now }
|
65
65
|
@vanity_identity
|
66
|
+
else # during functional testing
|
67
|
+
@vanity_identity = "test"
|
66
68
|
end
|
67
69
|
end
|
68
70
|
define_method :set_vanity_context do
|
data/lib/vanity/rails/testing.rb
CHANGED
@@ -3,7 +3,7 @@ module ActionController #:nodoc:
|
|
3
3
|
alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
|
4
4
|
def setup_controller_request_and_response
|
5
5
|
setup_controller_request_and_response_without_vanity
|
6
|
-
Vanity.context = @
|
6
|
+
Vanity.context = @controller
|
7
7
|
end
|
8
8
|
end
|
9
9
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
<% score = experiment.score %>
|
2
|
+
<table>
|
3
|
+
<caption><%= experiment.conclusion(score).join(" ") %></caption>
|
4
|
+
<% score.alts.each do |alt| %>
|
5
|
+
<tr class="<%= "choice" if score.choice == alt %>">
|
6
|
+
<td class="option"><%= alt.name.gsub(/^o/, "O") %>:</td>
|
7
|
+
<td class="value"><code><%= CGI.escape_html alt.value.to_s %></code></td>
|
8
|
+
<td>
|
9
|
+
<%= "%.1f%%" % [alt.conversion_rate * 100] %>
|
10
|
+
<%= "(%d%% better than %s)" % [alt.difference, score.least.name] if alt.difference && alt.difference >= 1 %>
|
11
|
+
</td>
|
12
|
+
<td class="action">
|
13
|
+
<% if experiment.active? && respond_to?(:chooses_experiments_url) %>
|
14
|
+
<% if experiment.chosen?(alt) %>
|
15
|
+
showing
|
16
|
+
<% else %>
|
17
|
+
<%= link_to "show", chooses_experiments_url(e: experiment.id, a: alt.id), method: :post,
|
18
|
+
class: "button", title: "Show me this alternative from now on" %>
|
19
|
+
<% end %>
|
20
|
+
<% end %>
|
21
|
+
</td>
|
22
|
+
</tr>
|
23
|
+
<% end %>
|
24
|
+
</table>
|
25
|
+
<%= %>
|
@@ -0,0 +1,12 @@
|
|
1
|
+
<ul class="experiments">
|
2
|
+
<% experiments.sort_by(&:created_at).each do |experiment| %>
|
3
|
+
<li class="experiment" id="experiment_<%= CGI.escape experiment.id.to_s %>">
|
4
|
+
<h2 class="<%= experiment.type %>"><%= CGI.escape_html experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
|
5
|
+
<p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
|
6
|
+
<%= render Vanity.template("ab_test"), experiment: experiment %>
|
7
|
+
<p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
|
8
|
+
<%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
|
9
|
+
</p>
|
10
|
+
</li>
|
11
|
+
<% end %>
|
12
|
+
</ul>
|
@@ -0,0 +1,16 @@
|
|
1
|
+
<html>
|
2
|
+
<head>
|
3
|
+
<title>Experiments (<%= Time.now %>)</title>
|
4
|
+
<style>
|
5
|
+
.vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
|
6
|
+
.vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
|
7
|
+
</style>
|
8
|
+
<%= render Vanity.template("vanity.css") %>
|
9
|
+
</head>
|
10
|
+
<body>
|
11
|
+
<div class="vanity">
|
12
|
+
<h1>Experiments</h1>
|
13
|
+
<%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
|
14
|
+
</div>
|
15
|
+
</body>
|
16
|
+
</html>
|
@@ -0,0 +1,13 @@
|
|
1
|
+
<style>
|
2
|
+
.vanity .experiments { list-style: none; margin: 0; padding: 0 }
|
3
|
+
.vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
|
4
|
+
.vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
|
5
|
+
.vanity .experiment table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
|
6
|
+
.vanity .experiment td { padding: .5em; border-top: 1px solid #ccc }
|
7
|
+
.vanity .experiment .choice td { font-weight: bold; background: #f0f0f8 }
|
8
|
+
.vanity .experiment td.option { width: 5em; white-space: nowrap; overflow: hidden }
|
9
|
+
.vanity .experiment td.value { width: 8em; white-space: nowrap; overflow: hidden }
|
10
|
+
.vanity .experiment td.action { width: 6em; overflow: hidden; text-align: center }
|
11
|
+
.vanity .experiment caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
|
12
|
+
.vanity .experiment .meta { color: #444; font-style: italic }
|
13
|
+
</style>
|
data/test/ab_test_test.rb
CHANGED
@@ -5,19 +5,19 @@ class AbTestController < ActionController::Base
|
|
5
5
|
attr_accessor :current_user
|
6
6
|
|
7
7
|
def test_render
|
8
|
-
render text: ab_test(:
|
8
|
+
render text: ab_test(:simple)
|
9
9
|
end
|
10
10
|
|
11
11
|
def test_view
|
12
|
-
render inline: "<%= ab_test(:
|
12
|
+
render inline: "<%= ab_test(:simple) %>"
|
13
13
|
end
|
14
14
|
|
15
15
|
def test_capture
|
16
|
-
render inline: "<% ab_test :
|
16
|
+
render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
|
17
17
|
end
|
18
18
|
|
19
19
|
def goal
|
20
|
-
ab_goal! :
|
20
|
+
ab_goal! :simple
|
21
21
|
render text: ""
|
22
22
|
end
|
23
23
|
end
|
@@ -25,10 +25,6 @@ end
|
|
25
25
|
|
26
26
|
class AbTestTest < ActionController::TestCase
|
27
27
|
tests AbTestController
|
28
|
-
def setup
|
29
|
-
experiment(:simple_ab) { }
|
30
|
-
end
|
31
|
-
|
32
28
|
|
33
29
|
# -- Experiment definition --
|
34
30
|
|
@@ -65,8 +61,8 @@ class AbTestTest < ActionController::TestCase
|
|
65
61
|
experiment :abcd do
|
66
62
|
alternatives :a, :b
|
67
63
|
end
|
68
|
-
assert_equal "option
|
69
|
-
assert_equal "option
|
64
|
+
assert_equal "option A", experiment(:abcd).alternative(:a).name
|
65
|
+
assert_equal "option B", experiment(:abcd).alternative(:b).name
|
70
66
|
end
|
71
67
|
|
72
68
|
|
@@ -87,7 +83,7 @@ class AbTestTest < ActionController::TestCase
|
|
87
83
|
def test_returns_different_alternatives_for_each_participant
|
88
84
|
experiment :foobar do
|
89
85
|
alternatives "foo", "bar"
|
90
|
-
identify { rand
|
86
|
+
identify { rand }
|
91
87
|
end
|
92
88
|
alts = Array.new(1000) { experiment(:foobar).choose }
|
93
89
|
assert_equal %w{bar foo}, alts.uniq.sort
|
@@ -95,48 +91,44 @@ class AbTestTest < ActionController::TestCase
|
|
95
91
|
end
|
96
92
|
|
97
93
|
def test_records_all_participants_in_each_alternative
|
98
|
-
ids = (Array.new(200) { |i| i
|
94
|
+
ids = (Array.new(200) { |i| i } * 5).shuffle
|
99
95
|
experiment :foobar do
|
100
96
|
alternatives "foo", "bar"
|
101
97
|
identify { ids.pop }
|
102
98
|
end
|
103
99
|
1000.times { experiment(:foobar).choose }
|
104
100
|
alts = experiment(:foobar).alternatives
|
105
|
-
assert_equal 200, alts.
|
101
|
+
assert_equal 200, alts.map(&:participants).sum
|
106
102
|
assert_in_delta alts.first.participants, 100, 20
|
107
103
|
end
|
108
104
|
|
109
105
|
def test_records_each_converted_participant_only_once
|
110
|
-
ids = (
|
111
|
-
test = self
|
106
|
+
ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
|
112
107
|
experiment :foobar do
|
113
108
|
alternatives "foo", "bar"
|
114
|
-
identify {
|
109
|
+
identify { ids.pop }
|
115
110
|
end
|
116
111
|
500.times do
|
117
|
-
test.identity = nil
|
118
112
|
experiment(:foobar).choose
|
119
113
|
experiment(:foobar).conversion!
|
120
114
|
end
|
121
115
|
alts = experiment(:foobar).alternatives
|
122
|
-
assert_equal 100, alts.
|
116
|
+
assert_equal 100, alts.map(&:converted).sum
|
123
117
|
end
|
124
118
|
|
125
119
|
def test_records_conversion_only_for_participants
|
126
|
-
|
120
|
+
ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
|
127
121
|
experiment :foobar do
|
128
122
|
alternatives "foo", "bar"
|
129
|
-
identify {
|
123
|
+
identify { ids.pop }
|
130
124
|
end
|
131
|
-
|
132
|
-
test.identity = nil
|
125
|
+
500.times do
|
133
126
|
experiment(:foobar).choose
|
134
127
|
experiment(:foobar).conversion!
|
135
|
-
test.identity << "!"
|
136
128
|
experiment(:foobar).conversion!
|
137
129
|
end
|
138
130
|
alts = experiment(:foobar).alternatives
|
139
|
-
assert_equal 100, alts.
|
131
|
+
assert_equal 100, alts.map(&:converted).sum
|
140
132
|
end
|
141
133
|
|
142
134
|
def test_reset_experiment
|
@@ -163,13 +155,13 @@ class AbTestTest < ActionController::TestCase
|
|
163
155
|
# -- A/B helper methods --
|
164
156
|
|
165
157
|
def test_fail_if_no_experiment
|
166
|
-
new_playground
|
167
158
|
assert_raise MissingSourceFile do
|
168
159
|
get :test_render
|
169
160
|
end
|
170
161
|
end
|
171
162
|
|
172
163
|
def test_ab_test_chooses_in_render
|
164
|
+
experiment(:simple) { }
|
173
165
|
responses = Array.new(100) do
|
174
166
|
@controller = nil ; setup_controller_request_and_response
|
175
167
|
get :test_render
|
@@ -179,6 +171,7 @@ class AbTestTest < ActionController::TestCase
|
|
179
171
|
end
|
180
172
|
|
181
173
|
def test_ab_test_chooses_view_helper
|
174
|
+
experiment(:simple) { }
|
182
175
|
responses = Array.new(100) do
|
183
176
|
@controller = nil ; setup_controller_request_and_response
|
184
177
|
get :test_view
|
@@ -188,6 +181,7 @@ class AbTestTest < ActionController::TestCase
|
|
188
181
|
end
|
189
182
|
|
190
183
|
def test_ab_test_with_capture
|
184
|
+
experiment(:simple) { }
|
191
185
|
responses = Array.new(100) do
|
192
186
|
@controller = nil ; setup_controller_request_and_response
|
193
187
|
get :test_capture
|
@@ -197,6 +191,7 @@ class AbTestTest < ActionController::TestCase
|
|
197
191
|
end
|
198
192
|
|
199
193
|
def test_ab_test_goal
|
194
|
+
experiment(:simple) { }
|
200
195
|
responses = Array.new(100) do
|
201
196
|
@controller.send(:cookies).clear
|
202
197
|
get :goal
|
@@ -208,88 +203,95 @@ class AbTestTest < ActionController::TestCase
|
|
208
203
|
# -- Testing with tests --
|
209
204
|
|
210
205
|
def test_with_given_choice
|
211
|
-
|
206
|
+
experiment(:simple) { alternatives :a, :b, :c }
|
207
|
+
100.times do |i|
|
212
208
|
@controller = nil ; setup_controller_request_and_response
|
213
|
-
experiment(:
|
209
|
+
experiment(:simple).chooses(:b)
|
214
210
|
get :test_render
|
215
|
-
|
211
|
+
assert "b", @response.body
|
216
212
|
end
|
217
|
-
alts = experiment(:simple_ab).alternatives
|
218
|
-
assert_equal [0,100], alts.map { |alt| alt.participants }
|
219
|
-
assert_equal [0,100], alts.map { |alt| alt.conversions }
|
220
213
|
end
|
221
214
|
|
222
215
|
def test_which_chooses_non_existent_alternative
|
216
|
+
experiment(:simple) { }
|
223
217
|
assert_raises ArgumentError do
|
224
|
-
experiment(:
|
218
|
+
experiment(:simple).chooses(404)
|
225
219
|
end
|
226
220
|
end
|
227
221
|
|
228
222
|
|
229
223
|
# -- Scoring --
|
230
|
-
|
224
|
+
|
231
225
|
def test_scoring
|
232
226
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
233
227
|
# participating, conversions, rate, z-score
|
234
228
|
# Control: 182 35 19.23% N/A
|
235
|
-
182.times { |i| experiment(:abcd).
|
236
|
-
35.times { |i| experiment(:abcd).
|
229
|
+
182.times { |i| experiment(:abcd).count i, :a, :participant }
|
230
|
+
35.times { |i| experiment(:abcd).count i, :a, :conversion }
|
237
231
|
# Treatment A: 180 45 25.00% 1.33
|
238
|
-
180.times { |i| experiment(:abcd).
|
239
|
-
45.times { |i| experiment(:abcd).
|
232
|
+
180.times { |i| experiment(:abcd).count i, :b, :participant }
|
233
|
+
45.times { |i| experiment(:abcd).count i, :b, :conversion }
|
240
234
|
# treatment B: 189 28 14.81% -1.13
|
241
|
-
189.times { |i| experiment(:abcd).
|
242
|
-
28.times { |i| experiment(:abcd).
|
235
|
+
189.times { |i| experiment(:abcd).count i, :c, :participant }
|
236
|
+
28.times { |i| experiment(:abcd).count i, :c, :conversion }
|
243
237
|
# treatment C: 188 61 32.45% 2.94
|
244
|
-
188.times { |i| experiment(:abcd).
|
245
|
-
61.times { |i| experiment(:abcd).
|
238
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
239
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
246
240
|
|
247
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.
|
241
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
248
242
|
assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
|
249
|
-
confidences = experiment(:abcd).score.alts.map(&:
|
243
|
+
confidences = experiment(:abcd).score.alts.map(&:confidence)
|
250
244
|
assert_equal [90, 0, 99, 90], confidences
|
251
245
|
|
252
|
-
diff = experiment(:abcd).score.alts.map { |alt| alt.
|
246
|
+
diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
|
253
247
|
assert_equal [30, 69, nil, 119], diff
|
254
248
|
assert_equal 3, experiment(:abcd).score.best.id
|
255
249
|
assert_equal 3, experiment(:abcd).score.choice.id
|
250
|
+
|
251
|
+
assert_equal 1, experiment(:abcd).score.base.id
|
252
|
+
assert_equal 2, experiment(:abcd).score.least.id
|
256
253
|
end
|
257
254
|
|
258
255
|
def test_scoring_with_no_performers
|
259
256
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
260
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
261
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
262
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
257
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
258
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
|
259
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
263
260
|
assert_nil experiment(:abcd).score.best
|
264
261
|
assert_nil experiment(:abcd).score.choice
|
262
|
+
assert_nil experiment(:abcd).score.least
|
265
263
|
end
|
266
264
|
|
267
265
|
def test_scoring_with_one_performer
|
268
266
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
269
|
-
10.times { |i| experiment(:abcd).
|
270
|
-
8.times { |i| experiment(:abcd).
|
271
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
272
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
273
|
-
assert experiment(:abcd).score.alts.all? { |alt| alt.
|
267
|
+
10.times { |i| experiment(:abcd).count i, :b, :participant }
|
268
|
+
8.times { |i| experiment(:abcd).count i, :b, :conversion }
|
269
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
|
270
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
|
271
|
+
assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
|
274
272
|
assert 1, experiment(:abcd).score.best.id
|
275
273
|
assert_nil experiment(:abcd).score.choice
|
274
|
+
assert 1, experiment(:abcd).score.base.id
|
275
|
+
assert 1, experiment(:abcd).score.least.id
|
276
276
|
end
|
277
277
|
|
278
278
|
def test_scoring_with_some_performers
|
279
279
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
280
|
-
10.times { |i| experiment(:abcd).
|
281
|
-
8.times { |i| experiment(:abcd).
|
282
|
-
12.times { |i| experiment(:abcd).
|
283
|
-
5.times { |i| experiment(:abcd).
|
280
|
+
10.times { |i| experiment(:abcd).count i, :b, :participant }
|
281
|
+
8.times { |i| experiment(:abcd).count i, :b, :conversion }
|
282
|
+
12.times { |i| experiment(:abcd).count i, :d, :participant }
|
283
|
+
5.times { |i| experiment(:abcd).count i, :d, :conversion }
|
284
284
|
|
285
|
-
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.
|
285
|
+
z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
|
286
286
|
assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
|
287
|
-
confidences = experiment(:abcd).score.alts.map(&:
|
287
|
+
confidences = experiment(:abcd).score.alts.map(&:confidence)
|
288
288
|
assert_equal [0, 95, 0, 0], confidences
|
289
|
-
diff = experiment(:abcd).score.alts.map { |alt| alt.
|
289
|
+
diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
|
290
290
|
assert_equal [nil, 92, nil, nil], diff
|
291
291
|
assert_equal 1, experiment(:abcd).score.best.id
|
292
292
|
assert_equal 1, experiment(:abcd).score.choice.id
|
293
|
+
assert_equal 3, experiment(:abcd).score.base.id
|
294
|
+
assert_equal 3, experiment(:abcd).score.least.id
|
293
295
|
end
|
294
296
|
|
295
297
|
|
@@ -299,105 +301,105 @@ class AbTestTest < ActionController::TestCase
|
|
299
301
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
300
302
|
# participating, conversions, rate, z-score
|
301
303
|
# Control: 182 35 19.23% N/A
|
302
|
-
182.times { |i| experiment(:abcd).
|
303
|
-
35.times { |i| experiment(:abcd).
|
304
|
+
182.times { |i| experiment(:abcd).count i, :a, :participant }
|
305
|
+
35.times { |i| experiment(:abcd).count i, :a, :conversion }
|
304
306
|
# Treatment A: 180 45 25.00% 1.33
|
305
|
-
180.times { |i| experiment(:abcd).
|
306
|
-
45.times { |i| experiment(:abcd).
|
307
|
+
180.times { |i| experiment(:abcd).count i, :b, :participant }
|
308
|
+
45.times { |i| experiment(:abcd).count i, :b, :conversion }
|
307
309
|
# treatment B: 189 28 14.81% -1.13
|
308
|
-
189.times { |i| experiment(:abcd).
|
309
|
-
28.times { |i| experiment(:abcd).
|
310
|
+
189.times { |i| experiment(:abcd).count i, :c, :participant }
|
311
|
+
28.times { |i| experiment(:abcd).count i, :c, :conversion }
|
310
312
|
# treatment C: 188 61 32.45% 2.94
|
311
|
-
188.times { |i| experiment(:abcd).
|
312
|
-
61.times { |i| experiment(:abcd).
|
313
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
314
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
313
315
|
|
314
316
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
315
|
-
The best choice is option
|
317
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
316
318
|
With 90% probability this result is statistically significant.
|
317
|
-
Option
|
318
|
-
Option
|
319
|
-
Option
|
320
|
-
Option
|
319
|
+
Option B converted at 25.0%.
|
320
|
+
Option A converted at 19.2%.
|
321
|
+
Option C converted at 14.8%.
|
322
|
+
Option D selected as the best alternative.
|
321
323
|
TEXT
|
322
324
|
end
|
323
325
|
|
324
326
|
def test_conclusion_with_some_performers
|
325
327
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
326
328
|
# Treatment A: 180 45 25.00% 1.33
|
327
|
-
180.times { |i| experiment(:abcd).
|
328
|
-
45.times { |i| experiment(:abcd).
|
329
|
+
180.times { |i| experiment(:abcd).count i, :b, :participant }
|
330
|
+
45.times { |i| experiment(:abcd).count i, :b, :conversion }
|
329
331
|
# treatment C: 188 61 32.45% 2.94
|
330
|
-
188.times { |i| experiment(:abcd).
|
331
|
-
61.times { |i| experiment(:abcd).
|
332
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
333
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
332
334
|
|
333
335
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
334
|
-
The best choice is option
|
336
|
+
The best choice is option D: it converted at 32.4% (30% better than option B).
|
335
337
|
With 90% probability this result is statistically significant.
|
336
|
-
Option
|
337
|
-
Option
|
338
|
-
Option
|
339
|
-
Option
|
338
|
+
Option B converted at 25.0%.
|
339
|
+
Option A did not convert.
|
340
|
+
Option C did not convert.
|
341
|
+
Option D selected as the best alternative.
|
340
342
|
TEXT
|
341
343
|
end
|
342
344
|
|
343
345
|
def test_conclusion_without_clear_winner
|
344
346
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
345
347
|
# Treatment A: 180 45 25.00% 1.33
|
346
|
-
180.times { |i| experiment(:abcd).
|
347
|
-
58.times { |i| experiment(:abcd).
|
348
|
+
180.times { |i| experiment(:abcd).count i, :b, :participant }
|
349
|
+
58.times { |i| experiment(:abcd).count i, :b, :conversion }
|
348
350
|
# treatment C: 188 61 32.45% 2.94
|
349
|
-
188.times { |i| experiment(:abcd).
|
350
|
-
61.times { |i| experiment(:abcd).
|
351
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
352
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
351
353
|
|
352
354
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
353
|
-
The best choice is option
|
355
|
+
The best choice is option D: it converted at 32.4% (1% better than option B).
|
354
356
|
This result is not statistically significant, suggest you continue this experiment.
|
355
|
-
Option
|
356
|
-
Option
|
357
|
-
Option
|
357
|
+
Option B converted at 32.2%.
|
358
|
+
Option A did not convert.
|
359
|
+
Option C did not convert.
|
358
360
|
TEXT
|
359
361
|
end
|
360
362
|
|
361
363
|
def test_conclusion_without_close_performers
|
362
364
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
363
365
|
# Treatment A: 180 45 25.00% 1.33
|
364
|
-
186.times { |i| experiment(:abcd).
|
365
|
-
60.times { |i| experiment(:abcd).
|
366
|
+
186.times { |i| experiment(:abcd).count i, :b, :participant }
|
367
|
+
60.times { |i| experiment(:abcd).count i, :b, :conversion }
|
366
368
|
# treatment C: 188 61 32.45% 2.94
|
367
|
-
188.times { |i| experiment(:abcd).
|
368
|
-
61.times { |i| experiment(:abcd).
|
369
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
370
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
369
371
|
|
370
372
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
371
|
-
The best choice is option
|
373
|
+
The best choice is option D: it converted at 32.4%.
|
372
374
|
This result is not statistically significant, suggest you continue this experiment.
|
373
|
-
Option
|
374
|
-
Option
|
375
|
-
Option
|
375
|
+
Option B converted at 32.3%.
|
376
|
+
Option A did not convert.
|
377
|
+
Option C did not convert.
|
376
378
|
TEXT
|
377
379
|
end
|
378
380
|
|
379
381
|
def test_conclusion_without_equal_performers
|
380
382
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
381
383
|
# Treatment A: 180 45 25.00% 1.33
|
382
|
-
188.times { |i| experiment(:abcd).
|
383
|
-
61.times { |i| experiment(:abcd).
|
384
|
+
188.times { |i| experiment(:abcd).count i, :b, :participant }
|
385
|
+
61.times { |i| experiment(:abcd).count i, :b, :conversion }
|
384
386
|
# treatment C: 188 61 32.45% 2.94
|
385
|
-
188.times { |i| experiment(:abcd).
|
386
|
-
61.times { |i| experiment(:abcd).
|
387
|
+
188.times { |i| experiment(:abcd).count i, :d, :participant }
|
388
|
+
61.times { |i| experiment(:abcd).count i, :d, :conversion }
|
387
389
|
|
388
390
|
assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
|
389
|
-
Option
|
390
|
-
Option
|
391
|
-
Option
|
392
|
-
Option
|
391
|
+
Option D converted at 32.4%.
|
392
|
+
Option B converted at 32.4%.
|
393
|
+
Option A did not convert.
|
394
|
+
Option C did not convert.
|
393
395
|
TEXT
|
394
396
|
end
|
395
397
|
|
396
398
|
def test_conclusion_with_one_performers
|
397
399
|
experiment(:abcd) { alternatives :a, :b, :c, :d }
|
398
400
|
# Treatment A: 180 45 25.00% 1.33
|
399
|
-
180.times { |i| experiment(:abcd).
|
400
|
-
45.times { |i| experiment(:abcd).
|
401
|
+
180.times { |i| experiment(:abcd).count i, :b, :participant }
|
402
|
+
45.times { |i| experiment(:abcd).count i, :b, :conversion }
|
401
403
|
|
402
404
|
assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
|
403
405
|
end
|
@@ -444,17 +446,15 @@ Option 3 did not convert.
|
|
444
446
|
end
|
445
447
|
|
446
448
|
def test_ab_methods_after_completion
|
447
|
-
ids = Array.new(200) { |i| i
|
448
|
-
test = self
|
449
|
+
ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
|
449
450
|
experiment :simple do
|
450
|
-
identify {
|
451
|
+
identify { ids.pop }
|
451
452
|
complete_if { alternatives.map(&:participants).sum >= 100 }
|
452
453
|
outcome_is { alternatives[1] }
|
453
454
|
end
|
454
455
|
# Run experiment to completion (100 participants)
|
455
456
|
results = Set.new
|
456
457
|
100.times do
|
457
|
-
test.identity = nil
|
458
458
|
results << experiment(:simple).choose
|
459
459
|
experiment(:simple).conversion!
|
460
460
|
end
|
@@ -463,7 +463,6 @@ Option 3 did not convert.
|
|
463
463
|
|
464
464
|
# Test that we always get the same choice (true)
|
465
465
|
100.times do
|
466
|
-
test.identity = nil
|
467
466
|
assert_equal true, experiment(:simple).choose
|
468
467
|
experiment(:simple).conversion!
|
469
468
|
end
|
@@ -510,41 +509,27 @@ Option 3 did not convert.
|
|
510
509
|
def test_outcome_choosing_best_alternative
|
511
510
|
experiment :quick do
|
512
511
|
end
|
513
|
-
2.times
|
514
|
-
|
515
|
-
end
|
516
|
-
10.times do |i|
|
517
|
-
experiment(:quick).alternatives[1].participating!(i)
|
518
|
-
experiment(:quick).alternatives[1].conversion!(i)
|
519
|
-
end
|
512
|
+
2.times { |i| experiment(:quick).count i, false, :participant }
|
513
|
+
10.times { |i| experiment(:quick).count i, true }
|
520
514
|
experiment(:quick).complete!
|
521
|
-
assert_equal experiment(:quick).
|
515
|
+
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
522
516
|
end
|
523
517
|
|
524
518
|
def test_outcome_only_performing_alternative
|
525
519
|
experiment :quick do
|
526
520
|
end
|
527
|
-
2.times
|
528
|
-
experiment(:quick).alternatives[1].participating!(i)
|
529
|
-
experiment(:quick).alternatives[1].conversion!(i)
|
530
|
-
end
|
521
|
+
2.times { |i| experiment(:quick).count i, true }
|
531
522
|
experiment(:quick).complete!
|
532
|
-
assert_equal experiment(:quick).
|
523
|
+
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
533
524
|
end
|
534
525
|
|
535
526
|
def test_outcome_choosing_equal_alternatives
|
536
527
|
experiment :quick do
|
537
528
|
end
|
538
|
-
8.times
|
539
|
-
|
540
|
-
experiment(:quick).alternatives[0].conversion!(i)
|
541
|
-
end
|
542
|
-
8.times do |i|
|
543
|
-
experiment(:quick).alternatives[1].participating!(i)
|
544
|
-
experiment(:quick).alternatives[1].conversion!(i)
|
545
|
-
end
|
529
|
+
8.times { |i| experiment(:quick).count i, false }
|
530
|
+
8.times { |i| experiment(:quick).count i, true }
|
546
531
|
experiment(:quick).complete!
|
547
|
-
assert_equal experiment(:quick).
|
532
|
+
assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
|
548
533
|
end
|
549
534
|
|
550
535
|
end
|