vanity 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,8 +18,6 @@ module Vanity
18
18
  @playground = playground
19
19
  @id, @name = id.to_sym, name
20
20
  @namespace = "#{@playground.namespace}:#{@id}"
21
- redis.setnx key(:created_at), Time.now.to_i
22
- @created_at = Time.at(redis[key(:created_at)].to_i)
23
21
  @identify_block = ->(context){ context.vanity_identity }
24
22
  end
25
23
 
@@ -147,11 +145,13 @@ module Vanity
147
145
  @playground.redis
148
146
  end
149
147
 
150
- # Called to save the experiment definition.
151
- def save #:nodoc:
148
+ # Called by Playground to save the experiment definition.
149
+ def save
150
+ redis.setnx key(:created_at), Time.now.to_i
151
+ @created_at = Time.at(redis[key(:created_at)].to_i)
152
152
  end
153
153
 
154
- # Reset experiment.
154
+ # Reset experiment to its initial state.
155
155
  def reset!
156
156
  @created_at = Time.now
157
157
  redis[key(:created_at)] = @created_at.to_i
@@ -1,5 +1,3 @@
1
- require "active_support"
2
-
3
1
  module Vanity
4
2
 
5
3
  # Vanity.playground.configuration
@@ -88,23 +86,33 @@ module Vanity
88
86
  end
89
87
 
90
88
  @playground = Playground.new
91
- # Returns the playground instance.
92
- def self.playground
93
- @playground
94
- end
89
+ class << self
95
90
 
96
- # Returns the Vanity context. For example, when using Rails this would be
97
- # the current controller, which can be used to get/set the vanity identity.
98
- def self.context
99
- Thread.current[:vanity_context]
100
- end
91
+ # Returns the playground instance.
92
+ def playground
93
+ @playground
94
+ end
101
95
 
102
- # Sets the Vanity context. For example, when using Rails this would be
103
- # set by the set_vanity_context before filter (via use_vanity).
104
- def self.context=(context)
105
- Thread.current[:vanity_context] = context
106
- end
96
+ # Returns the Vanity context. For example, when using Rails this would be
97
+ # the current controller, which can be used to get/set the vanity identity.
98
+ def context
99
+ Thread.current[:vanity_context]
100
+ end
107
101
 
102
+ # Sets the Vanity context. For example, when using Rails this would be
103
+ # set by the set_vanity_context before filter (via use_vanity).
104
+ def context=(context)
105
+ Thread.current[:vanity_context] = context
106
+ end
107
+
108
+ # Path to template.
109
+ def template(name)
110
+ path = File.join(File.dirname(__FILE__), "templates/#{name}")
111
+ path << ".erb" unless name["."]
112
+ path
113
+ end
114
+
115
+ end
108
116
  end
109
117
 
110
118
  class Object
data/lib/vanity/rails.rb CHANGED
@@ -1,6 +1,7 @@
1
- require File.join(File.dirname(__FILE__), "../vanity")
2
- require File.join(File.dirname(__FILE__), "rails/helpers")
3
- require File.join(File.dirname(__FILE__), "rails/testing")
1
+ require "vanity"
2
+ require "vanity/rails/helpers"
3
+ require "vanity/rails/testing"
4
+ require "vanity/rails/console"
4
5
 
5
6
  # Use Rails logger by default.
6
7
  Vanity.playground.logger ||= ActionController::Base.logger
@@ -0,0 +1,14 @@
1
+ module Vanity
2
+ module Rails
3
+ module ConsoleActions
4
+ def index
5
+ render Vanity.template("_report"), content_type: Mime::HTML, layout: true
6
+ end
7
+
8
+ def chooses
9
+ experiment(params[:e]).chooses(experiment(params[:e]).alternatives[params[:a].to_i].value)
10
+ redirect_to :back
11
+ end
12
+ end
13
+ end
14
+ end
@@ -59,10 +59,12 @@ module Vanity
59
59
  @vanity_identity = block.call(self)
60
60
  elsif symbol && object = send(symbol)
61
61
  @vanity_identity = object.id
62
- else
62
+ elsif response # everyday use
63
63
  @vanity_identity = cookies["vanity_id"] || OpenSSL::Random.random_bytes(16).unpack("H*")[0]
64
64
  cookies["vanity_id"] = { value: @vanity_identity, expires: 1.month.from_now }
65
65
  @vanity_identity
66
+ else # during functional testing
67
+ @vanity_identity = "test"
66
68
  end
67
69
  end
68
70
  define_method :set_vanity_context do
@@ -3,7 +3,7 @@ module ActionController #:nodoc:
3
3
  alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
4
4
  def setup_controller_request_and_response
5
5
  setup_controller_request_and_response_without_vanity
6
- Vanity.context = @request
6
+ Vanity.context = @controller
7
7
  end
8
8
  end
9
9
  end
@@ -0,0 +1,25 @@
1
+ <% score = experiment.score %>
2
+ <table>
3
+ <caption><%= experiment.conclusion(score).join(" ") %></caption>
4
+ <% score.alts.each do |alt| %>
5
+ <tr class="<%= "choice" if score.choice == alt %>">
6
+ <td class="option"><%= alt.name.gsub(/^o/, "O") %>:</td>
7
+ <td class="value"><code><%= CGI.escape_html alt.value.to_s %></code></td>
8
+ <td>
9
+ <%= "%.1f%%" % [alt.conversion_rate * 100] %>
10
+ <%= "(%d%% better than %s)" % [alt.difference, score.least.name] if alt.difference && alt.difference >= 1 %>
11
+ </td>
12
+ <td class="action">
13
+ <% if experiment.active? && respond_to?(:chooses_experiments_url) %>
14
+ <% if experiment.chosen?(alt) %>
15
+ showing
16
+ <% else %>
17
+ <%= link_to "show", chooses_experiments_url(e: experiment.id, a: alt.id), method: :post,
18
+ class: "button", title: "Show me this alternative from now on" %>
19
+ <% end %>
20
+ <% end %>
21
+ </td>
22
+ </tr>
23
+ <% end %>
24
+ </table>
25
+ <%= %>
@@ -0,0 +1,12 @@
1
+ <ul class="experiments">
2
+ <% experiments.sort_by(&:created_at).each do |experiment| %>
3
+ <li class="experiment" id="experiment_<%= CGI.escape experiment.id.to_s %>">
4
+ <h2 class="<%= experiment.type %>"><%= CGI.escape_html experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
5
+ <p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
6
+ <%= render Vanity.template("ab_test"), experiment: experiment %>
7
+ <p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
8
+ <%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
9
+ </p>
10
+ </li>
11
+ <% end %>
12
+ </ul>
@@ -0,0 +1,16 @@
1
+ <html>
2
+ <head>
3
+ <title>Experiments (<%= Time.now %>)</title>
4
+ <style>
5
+ .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
6
+ .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
7
+ </style>
8
+ <%= render Vanity.template("vanity.css") %>
9
+ </head>
10
+ <body>
11
+ <div class="vanity">
12
+ <h1>Experiments</h1>
13
+ <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
14
+ </div>
15
+ </body>
16
+ </html>
@@ -0,0 +1,13 @@
1
+ <style>
2
+ .vanity .experiments { list-style: none; margin: 0; padding: 0 }
3
+ .vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
4
+ .vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
5
+ .vanity .experiment table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
6
+ .vanity .experiment td { padding: .5em; border-top: 1px solid #ccc }
7
+ .vanity .experiment .choice td { font-weight: bold; background: #f0f0f8 }
8
+ .vanity .experiment td.option { width: 5em; white-space: nowrap; overflow: hidden }
9
+ .vanity .experiment td.value { width: 8em; white-space: nowrap; overflow: hidden }
10
+ .vanity .experiment td.action { width: 6em; overflow: hidden; text-align: center }
11
+ .vanity .experiment caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
12
+ .vanity .experiment .meta { color: #444; font-style: italic }
13
+ </style>
data/test/ab_test_test.rb CHANGED
@@ -5,19 +5,19 @@ class AbTestController < ActionController::Base
5
5
  attr_accessor :current_user
6
6
 
7
7
  def test_render
8
- render text: ab_test(:simple_ab)
8
+ render text: ab_test(:simple)
9
9
  end
10
10
 
11
11
  def test_view
12
- render inline: "<%= ab_test(:simple_ab) %>"
12
+ render inline: "<%= ab_test(:simple) %>"
13
13
  end
14
14
 
15
15
  def test_capture
16
- render inline: "<% ab_test :simple_ab do |value| %><%= value %><% end %>"
16
+ render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
17
17
  end
18
18
 
19
19
  def goal
20
- ab_goal! :simple_ab
20
+ ab_goal! :simple
21
21
  render text: ""
22
22
  end
23
23
  end
@@ -25,10 +25,6 @@ end
25
25
 
26
26
  class AbTestTest < ActionController::TestCase
27
27
  tests AbTestController
28
- def setup
29
- experiment(:simple_ab) { }
30
- end
31
-
32
28
 
33
29
  # -- Experiment definition --
34
30
 
@@ -65,8 +61,8 @@ class AbTestTest < ActionController::TestCase
65
61
  experiment :abcd do
66
62
  alternatives :a, :b
67
63
  end
68
- assert_equal "option 1", experiment(:abcd).alternative(:a).name
69
- assert_equal "option 2", experiment(:abcd).alternative(:b).name
64
+ assert_equal "option A", experiment(:abcd).alternative(:a).name
65
+ assert_equal "option B", experiment(:abcd).alternative(:b).name
70
66
  end
71
67
 
72
68
 
@@ -87,7 +83,7 @@ class AbTestTest < ActionController::TestCase
87
83
  def test_returns_different_alternatives_for_each_participant
88
84
  experiment :foobar do
89
85
  alternatives "foo", "bar"
90
- identify { rand(1000).to_s }
86
+ identify { rand }
91
87
  end
92
88
  alts = Array.new(1000) { experiment(:foobar).choose }
93
89
  assert_equal %w{bar foo}, alts.uniq.sort
@@ -95,48 +91,44 @@ class AbTestTest < ActionController::TestCase
95
91
  end
96
92
 
97
93
  def test_records_all_participants_in_each_alternative
98
- ids = (Array.new(200) { |i| i.to_s } * 5).shuffle
94
+ ids = (Array.new(200) { |i| i } * 5).shuffle
99
95
  experiment :foobar do
100
96
  alternatives "foo", "bar"
101
97
  identify { ids.pop }
102
98
  end
103
99
  1000.times { experiment(:foobar).choose }
104
100
  alts = experiment(:foobar).alternatives
105
- assert_equal 200, alts.inject(0) { |total,alt| total + alt.participants }
101
+ assert_equal 200, alts.map(&:participants).sum
106
102
  assert_in_delta alts.first.participants, 100, 20
107
103
  end
108
104
 
109
105
  def test_records_each_converted_participant_only_once
110
- ids = (Array.new(100) { |i| i.to_s } * 5).shuffle
111
- test = self
106
+ ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
112
107
  experiment :foobar do
113
108
  alternatives "foo", "bar"
114
- identify { test.identity ||= ids.pop }
109
+ identify { ids.pop }
115
110
  end
116
111
  500.times do
117
- test.identity = nil
118
112
  experiment(:foobar).choose
119
113
  experiment(:foobar).conversion!
120
114
  end
121
115
  alts = experiment(:foobar).alternatives
122
- assert_equal 100, alts.inject(0) { |total,alt| total + alt.converted }
116
+ assert_equal 100, alts.map(&:converted).sum
123
117
  end
124
118
 
125
119
  def test_records_conversion_only_for_participants
126
- test = self
120
+ ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
127
121
  experiment :foobar do
128
122
  alternatives "foo", "bar"
129
- identify { test.identity ||= rand(100).to_s }
123
+ identify { ids.pop }
130
124
  end
131
- 1000.times do
132
- test.identity = nil
125
+ 500.times do
133
126
  experiment(:foobar).choose
134
127
  experiment(:foobar).conversion!
135
- test.identity << "!"
136
128
  experiment(:foobar).conversion!
137
129
  end
138
130
  alts = experiment(:foobar).alternatives
139
- assert_equal 100, alts.inject(0) { |t,a| t + a.converted }
131
+ assert_equal 100, alts.map(&:converted).sum
140
132
  end
141
133
 
142
134
  def test_reset_experiment
@@ -163,13 +155,13 @@ class AbTestTest < ActionController::TestCase
163
155
  # -- A/B helper methods --
164
156
 
165
157
  def test_fail_if_no_experiment
166
- new_playground
167
158
  assert_raise MissingSourceFile do
168
159
  get :test_render
169
160
  end
170
161
  end
171
162
 
172
163
  def test_ab_test_chooses_in_render
164
+ experiment(:simple) { }
173
165
  responses = Array.new(100) do
174
166
  @controller = nil ; setup_controller_request_and_response
175
167
  get :test_render
@@ -179,6 +171,7 @@ class AbTestTest < ActionController::TestCase
179
171
  end
180
172
 
181
173
  def test_ab_test_chooses_view_helper
174
+ experiment(:simple) { }
182
175
  responses = Array.new(100) do
183
176
  @controller = nil ; setup_controller_request_and_response
184
177
  get :test_view
@@ -188,6 +181,7 @@ class AbTestTest < ActionController::TestCase
188
181
  end
189
182
 
190
183
  def test_ab_test_with_capture
184
+ experiment(:simple) { }
191
185
  responses = Array.new(100) do
192
186
  @controller = nil ; setup_controller_request_and_response
193
187
  get :test_capture
@@ -197,6 +191,7 @@ class AbTestTest < ActionController::TestCase
197
191
  end
198
192
 
199
193
  def test_ab_test_goal
194
+ experiment(:simple) { }
200
195
  responses = Array.new(100) do
201
196
  @controller.send(:cookies).clear
202
197
  get :goal
@@ -208,88 +203,95 @@ class AbTestTest < ActionController::TestCase
208
203
  # -- Testing with tests --
209
204
 
210
205
  def test_with_given_choice
211
- 100.times do
206
+ experiment(:simple) { alternatives :a, :b, :c }
207
+ 100.times do |i|
212
208
  @controller = nil ; setup_controller_request_and_response
213
- experiment(:simple_ab).chooses(true)
209
+ experiment(:simple).chooses(:b)
214
210
  get :test_render
215
- post :goal
211
+ assert "b", @response.body
216
212
  end
217
- alts = experiment(:simple_ab).alternatives
218
- assert_equal [0,100], alts.map { |alt| alt.participants }
219
- assert_equal [0,100], alts.map { |alt| alt.conversions }
220
213
  end
221
214
 
222
215
  def test_which_chooses_non_existent_alternative
216
+ experiment(:simple) { }
223
217
  assert_raises ArgumentError do
224
- experiment(:simple_ab).chooses(404)
218
+ experiment(:simple).chooses(404)
225
219
  end
226
220
  end
227
221
 
228
222
 
229
223
  # -- Scoring --
230
-
224
+
231
225
  def test_scoring
232
226
  experiment(:abcd) { alternatives :a, :b, :c, :d }
233
227
  # participating, conversions, rate, z-score
234
228
  # Control: 182 35 19.23% N/A
235
- 182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
236
- 35.times { |i| experiment(:abcd).alternative(:a).conversion!(i) }
229
+ 182.times { |i| experiment(:abcd).count i, :a, :participant }
230
+ 35.times { |i| experiment(:abcd).count i, :a, :conversion }
237
231
  # Treatment A: 180 45 25.00% 1.33
238
- 180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
239
- 45.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
232
+ 180.times { |i| experiment(:abcd).count i, :b, :participant }
233
+ 45.times { |i| experiment(:abcd).count i, :b, :conversion }
240
234
  # treatment B: 189 28 14.81% -1.13
241
- 189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
242
- 28.times { |i| experiment(:abcd).alternative(:c).conversion!(i) }
235
+ 189.times { |i| experiment(:abcd).count i, :c, :participant }
236
+ 28.times { |i| experiment(:abcd).count i, :c, :conversion }
243
237
  # treatment C: 188 61 32.45% 2.94
244
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
245
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
238
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
239
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
246
240
 
247
- z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
241
+ z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
248
242
  assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
249
- confidences = experiment(:abcd).score.alts.map(&:conf)
243
+ confidences = experiment(:abcd).score.alts.map(&:confidence)
250
244
  assert_equal [90, 0, 99, 90], confidences
251
245
 
252
- diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
246
+ diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
253
247
  assert_equal [30, 69, nil, 119], diff
254
248
  assert_equal 3, experiment(:abcd).score.best.id
255
249
  assert_equal 3, experiment(:abcd).score.choice.id
250
+
251
+ assert_equal 1, experiment(:abcd).score.base.id
252
+ assert_equal 2, experiment(:abcd).score.least.id
256
253
  end
257
254
 
258
255
  def test_scoring_with_no_performers
259
256
  experiment(:abcd) { alternatives :a, :b, :c, :d }
260
- assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
261
- assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
262
- assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
257
+ assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
258
+ assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
259
+ assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
263
260
  assert_nil experiment(:abcd).score.best
264
261
  assert_nil experiment(:abcd).score.choice
262
+ assert_nil experiment(:abcd).score.least
265
263
  end
266
264
 
267
265
  def test_scoring_with_one_performer
268
266
  experiment(:abcd) { alternatives :a, :b, :c, :d }
269
- 10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
270
- 8.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
271
- assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
272
- assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
273
- assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
267
+ 10.times { |i| experiment(:abcd).count i, :b, :participant }
268
+ 8.times { |i| experiment(:abcd).count i, :b, :conversion }
269
+ assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
270
+ assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
271
+ assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
274
272
  assert 1, experiment(:abcd).score.best.id
275
273
  assert_nil experiment(:abcd).score.choice
274
+ assert 1, experiment(:abcd).score.base.id
275
+ assert 1, experiment(:abcd).score.least.id
276
276
  end
277
277
 
278
278
  def test_scoring_with_some_performers
279
279
  experiment(:abcd) { alternatives :a, :b, :c, :d }
280
- 10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
281
- 8.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
282
- 12.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
283
- 5.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
280
+ 10.times { |i| experiment(:abcd).count i, :b, :participant }
281
+ 8.times { |i| experiment(:abcd).count i, :b, :conversion }
282
+ 12.times { |i| experiment(:abcd).count i, :d, :participant }
283
+ 5.times { |i| experiment(:abcd).count i, :d, :conversion }
284
284
 
285
- z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
285
+ z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
286
286
  assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
287
- confidences = experiment(:abcd).score.alts.map(&:conf)
287
+ confidences = experiment(:abcd).score.alts.map(&:confidence)
288
288
  assert_equal [0, 95, 0, 0], confidences
289
- diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
289
+ diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
290
290
  assert_equal [nil, 92, nil, nil], diff
291
291
  assert_equal 1, experiment(:abcd).score.best.id
292
292
  assert_equal 1, experiment(:abcd).score.choice.id
293
+ assert_equal 3, experiment(:abcd).score.base.id
294
+ assert_equal 3, experiment(:abcd).score.least.id
293
295
  end
294
296
 
295
297
 
@@ -299,105 +301,105 @@ class AbTestTest < ActionController::TestCase
299
301
  experiment(:abcd) { alternatives :a, :b, :c, :d }
300
302
  # participating, conversions, rate, z-score
301
303
  # Control: 182 35 19.23% N/A
302
- 182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
303
- 35.times { |i| experiment(:abcd).alternative(:a).conversion!(i) }
304
+ 182.times { |i| experiment(:abcd).count i, :a, :participant }
305
+ 35.times { |i| experiment(:abcd).count i, :a, :conversion }
304
306
  # Treatment A: 180 45 25.00% 1.33
305
- 180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
306
- 45.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
307
+ 180.times { |i| experiment(:abcd).count i, :b, :participant }
308
+ 45.times { |i| experiment(:abcd).count i, :b, :conversion }
307
309
  # treatment B: 189 28 14.81% -1.13
308
- 189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
309
- 28.times { |i| experiment(:abcd).alternative(:c).conversion!(i) }
310
+ 189.times { |i| experiment(:abcd).count i, :c, :participant }
311
+ 28.times { |i| experiment(:abcd).count i, :c, :conversion }
310
312
  # treatment C: 188 61 32.45% 2.94
311
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
312
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
313
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
314
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
313
315
 
314
316
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
315
- The best choice is option 4: it converted at 32.4% (30% better than option 2).
317
+ The best choice is option D: it converted at 32.4% (30% better than option B).
316
318
  With 90% probability this result is statistically significant.
317
- Option 2 converted at 25.0%.
318
- Option 1 converted at 19.2%.
319
- Option 3 converted at 14.8%.
320
- Option 4 selected as the best alternative.
319
+ Option B converted at 25.0%.
320
+ Option A converted at 19.2%.
321
+ Option C converted at 14.8%.
322
+ Option D selected as the best alternative.
321
323
  TEXT
322
324
  end
323
325
 
324
326
  def test_conclusion_with_some_performers
325
327
  experiment(:abcd) { alternatives :a, :b, :c, :d }
326
328
  # Treatment A: 180 45 25.00% 1.33
327
- 180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
328
- 45.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
329
+ 180.times { |i| experiment(:abcd).count i, :b, :participant }
330
+ 45.times { |i| experiment(:abcd).count i, :b, :conversion }
329
331
  # treatment C: 188 61 32.45% 2.94
330
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
331
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
332
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
333
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
332
334
 
333
335
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
334
- The best choice is option 4: it converted at 32.4% (30% better than option 2).
336
+ The best choice is option D: it converted at 32.4% (30% better than option B).
335
337
  With 90% probability this result is statistically significant.
336
- Option 2 converted at 25.0%.
337
- Option 1 did not convert.
338
- Option 3 did not convert.
339
- Option 4 selected as the best alternative.
338
+ Option B converted at 25.0%.
339
+ Option A did not convert.
340
+ Option C did not convert.
341
+ Option D selected as the best alternative.
340
342
  TEXT
341
343
  end
342
344
 
343
345
  def test_conclusion_without_clear_winner
344
346
  experiment(:abcd) { alternatives :a, :b, :c, :d }
345
347
  # Treatment A: 180 45 25.00% 1.33
346
- 180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
347
- 58.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
348
+ 180.times { |i| experiment(:abcd).count i, :b, :participant }
349
+ 58.times { |i| experiment(:abcd).count i, :b, :conversion }
348
350
  # treatment C: 188 61 32.45% 2.94
349
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
350
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
351
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
352
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
351
353
 
352
354
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
353
- The best choice is option 4: it converted at 32.4% (1% better than option 2).
355
+ The best choice is option D: it converted at 32.4% (1% better than option B).
354
356
  This result is not statistically significant, suggest you continue this experiment.
355
- Option 2 converted at 32.2%.
356
- Option 1 did not convert.
357
- Option 3 did not convert.
357
+ Option B converted at 32.2%.
358
+ Option A did not convert.
359
+ Option C did not convert.
358
360
  TEXT
359
361
  end
360
362
 
361
363
  def test_conclusion_without_close_performers
362
364
  experiment(:abcd) { alternatives :a, :b, :c, :d }
363
365
  # Treatment A: 180 45 25.00% 1.33
364
- 186.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
365
- 60.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
366
+ 186.times { |i| experiment(:abcd).count i, :b, :participant }
367
+ 60.times { |i| experiment(:abcd).count i, :b, :conversion }
366
368
  # treatment C: 188 61 32.45% 2.94
367
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
368
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
369
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
370
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
369
371
 
370
372
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
371
- The best choice is option 4: it converted at 32.4%.
373
+ The best choice is option D: it converted at 32.4%.
372
374
  This result is not statistically significant, suggest you continue this experiment.
373
- Option 2 converted at 32.3%.
374
- Option 1 did not convert.
375
- Option 3 did not convert.
375
+ Option B converted at 32.3%.
376
+ Option A did not convert.
377
+ Option C did not convert.
376
378
  TEXT
377
379
  end
378
380
 
379
381
  def test_conclusion_without_equal_performers
380
382
  experiment(:abcd) { alternatives :a, :b, :c, :d }
381
383
  # Treatment A: 180 45 25.00% 1.33
382
- 188.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
383
- 61.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
384
+ 188.times { |i| experiment(:abcd).count i, :b, :participant }
385
+ 61.times { |i| experiment(:abcd).count i, :b, :conversion }
384
386
  # treatment C: 188 61 32.45% 2.94
385
- 188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
386
- 61.times { |i| experiment(:abcd).alternative(:d).conversion!(i) }
387
+ 188.times { |i| experiment(:abcd).count i, :d, :participant }
388
+ 61.times { |i| experiment(:abcd).count i, :d, :conversion }
387
389
 
388
390
  assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
389
- Option 4 converted at 32.4%.
390
- Option 2 converted at 32.4%.
391
- Option 1 did not convert.
392
- Option 3 did not convert.
391
+ Option D converted at 32.4%.
392
+ Option B converted at 32.4%.
393
+ Option A did not convert.
394
+ Option C did not convert.
393
395
  TEXT
394
396
  end
395
397
 
396
398
  def test_conclusion_with_one_performers
397
399
  experiment(:abcd) { alternatives :a, :b, :c, :d }
398
400
  # Treatment A: 180 45 25.00% 1.33
399
- 180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
400
- 45.times { |i| experiment(:abcd).alternative(:b).conversion!(i) }
401
+ 180.times { |i| experiment(:abcd).count i, :b, :participant }
402
+ 45.times { |i| experiment(:abcd).count i, :b, :conversion }
401
403
 
402
404
  assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
403
405
  end
@@ -444,17 +446,15 @@ Option 3 did not convert.
444
446
  end
445
447
 
446
448
  def test_ab_methods_after_completion
447
- ids = Array.new(200) { |i| i.to_s }.shuffle
448
- test = self
449
+ ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
449
450
  experiment :simple do
450
- identify { test.identity ||= ids.pop }
451
+ identify { ids.pop }
451
452
  complete_if { alternatives.map(&:participants).sum >= 100 }
452
453
  outcome_is { alternatives[1] }
453
454
  end
454
455
  # Run experiment to completion (100 participants)
455
456
  results = Set.new
456
457
  100.times do
457
- test.identity = nil
458
458
  results << experiment(:simple).choose
459
459
  experiment(:simple).conversion!
460
460
  end
@@ -463,7 +463,6 @@ Option 3 did not convert.
463
463
 
464
464
  # Test that we always get the same choice (true)
465
465
  100.times do
466
- test.identity = nil
467
466
  assert_equal true, experiment(:simple).choose
468
467
  experiment(:simple).conversion!
469
468
  end
@@ -510,41 +509,27 @@ Option 3 did not convert.
510
509
  def test_outcome_choosing_best_alternative
511
510
  experiment :quick do
512
511
  end
513
- 2.times do |i|
514
- experiment(:quick).alternatives[0].participating!(i)
515
- end
516
- 10.times do |i|
517
- experiment(:quick).alternatives[1].participating!(i)
518
- experiment(:quick).alternatives[1].conversion!(i)
519
- end
512
+ 2.times { |i| experiment(:quick).count i, false, :participant }
513
+ 10.times { |i| experiment(:quick).count i, true }
520
514
  experiment(:quick).complete!
521
- assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
515
+ assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
522
516
  end
523
517
 
524
518
  def test_outcome_only_performing_alternative
525
519
  experiment :quick do
526
520
  end
527
- 2.times do |i|
528
- experiment(:quick).alternatives[1].participating!(i)
529
- experiment(:quick).alternatives[1].conversion!(i)
530
- end
521
+ 2.times { |i| experiment(:quick).count i, true }
531
522
  experiment(:quick).complete!
532
- assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
523
+ assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
533
524
  end
534
525
 
535
526
  def test_outcome_choosing_equal_alternatives
536
527
  experiment :quick do
537
528
  end
538
- 8.times do |i|
539
- experiment(:quick).alternatives[0].participating!(i)
540
- experiment(:quick).alternatives[0].conversion!(i)
541
- end
542
- 8.times do |i|
543
- experiment(:quick).alternatives[1].participating!(i)
544
- experiment(:quick).alternatives[1].conversion!(i)
545
- end
529
+ 8.times { |i| experiment(:quick).count i, false }
530
+ 8.times { |i| experiment(:quick).count i, true }
546
531
  experiment(:quick).complete!
547
- assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
532
+ assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
548
533
  end
549
534
 
550
535
  end