mikeg-vanity 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +153 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +83 -0
- data/bin/vanity +53 -0
- data/lib/vanity.rb +38 -0
- data/lib/vanity/backport.rb +43 -0
- data/lib/vanity/commands.rb +2 -0
- data/lib/vanity/commands/list.rb +21 -0
- data/lib/vanity/commands/report.rb +60 -0
- data/lib/vanity/experiment/ab_test.rb +477 -0
- data/lib/vanity/experiment/base.rb +212 -0
- data/lib/vanity/helpers.rb +59 -0
- data/lib/vanity/metric/active_record.rb +77 -0
- data/lib/vanity/metric/base.rb +221 -0
- data/lib/vanity/metric/google_analytics.rb +70 -0
- data/lib/vanity/mock_redis.rb +76 -0
- data/lib/vanity/playground.rb +197 -0
- data/lib/vanity/rails.rb +22 -0
- data/lib/vanity/rails/dashboard.rb +24 -0
- data/lib/vanity/rails/helpers.rb +158 -0
- data/lib/vanity/rails/testing.rb +11 -0
- data/lib/vanity/templates/_ab_test.erb +26 -0
- data/lib/vanity/templates/_experiment.erb +5 -0
- data/lib/vanity/templates/_experiments.erb +7 -0
- data/lib/vanity/templates/_metric.erb +14 -0
- data/lib/vanity/templates/_metrics.erb +13 -0
- data/lib/vanity/templates/_report.erb +27 -0
- data/lib/vanity/templates/flot.min.js +1 -0
- data/lib/vanity/templates/jquery.min.js +19 -0
- data/lib/vanity/templates/vanity.css +26 -0
- data/lib/vanity/templates/vanity.js +82 -0
- data/test/ab_test_test.rb +656 -0
- data/test/experiment_test.rb +136 -0
- data/test/experiments/age_and_zipcode.rb +19 -0
- data/test/experiments/metrics/cheers.rb +3 -0
- data/test/experiments/metrics/signups.rb +2 -0
- data/test/experiments/metrics/yawns.rb +3 -0
- data/test/experiments/null_abc.rb +5 -0
- data/test/metric_test.rb +518 -0
- data/test/playground_test.rb +10 -0
- data/test/rails_test.rb +104 -0
- data/test/test_helper.rb +135 -0
- data/vanity.gemspec +18 -0
- data/vendor/redis-rb/LICENSE +20 -0
- data/vendor/redis-rb/README.markdown +36 -0
- data/vendor/redis-rb/Rakefile +62 -0
- data/vendor/redis-rb/bench.rb +44 -0
- data/vendor/redis-rb/benchmarking/suite.rb +24 -0
- data/vendor/redis-rb/benchmarking/worker.rb +71 -0
- data/vendor/redis-rb/bin/distredis +33 -0
- data/vendor/redis-rb/examples/basic.rb +16 -0
- data/vendor/redis-rb/examples/incr-decr.rb +18 -0
- data/vendor/redis-rb/examples/list.rb +26 -0
- data/vendor/redis-rb/examples/sets.rb +36 -0
- data/vendor/redis-rb/lib/dist_redis.rb +124 -0
- data/vendor/redis-rb/lib/hash_ring.rb +128 -0
- data/vendor/redis-rb/lib/pipeline.rb +21 -0
- data/vendor/redis-rb/lib/redis.rb +370 -0
- data/vendor/redis-rb/lib/redis/raketasks.rb +1 -0
- data/vendor/redis-rb/profile.rb +22 -0
- data/vendor/redis-rb/redis-rb.gemspec +30 -0
- data/vendor/redis-rb/spec/redis_spec.rb +637 -0
- data/vendor/redis-rb/spec/spec_helper.rb +4 -0
- data/vendor/redis-rb/speed.rb +16 -0
- data/vendor/redis-rb/tasks/redis.tasks.rb +140 -0
- metadata +125 -0
@@ -0,0 +1,477 @@
|
|
1
|
+
require "digest/md5"
|
2
|
+
|
3
|
+
module Vanity
|
4
|
+
module Experiment
|
5
|
+
|
6
|
+
# One of several alternatives in an A/B test (see AbTest#alternatives).
|
7
|
+
class Alternative
|
8
|
+
|
9
|
+
def initialize(experiment, id, value, participants, converted, conversions)
|
10
|
+
@experiment = experiment
|
11
|
+
@id = id
|
12
|
+
@name = "option #{(@id + 65).chr}"
|
13
|
+
@value = value
|
14
|
+
@participants, @converted, @conversions = participants, converted, conversions
|
15
|
+
end
|
16
|
+
|
17
|
+
# Alternative id, only unique for this experiment.
|
18
|
+
attr_reader :id
|
19
|
+
|
20
|
+
# Alternative name (option A, option B, etc).
|
21
|
+
attr_reader :name
|
22
|
+
|
23
|
+
# Alternative value.
|
24
|
+
attr_reader :value
|
25
|
+
|
26
|
+
# Experiment this alternative belongs to.
|
27
|
+
attr_reader :experiment
|
28
|
+
|
29
|
+
# Number of participants who viewed this alternative.
|
30
|
+
attr_reader :participants
|
31
|
+
|
32
|
+
# Number of participants who converted on this alternative (a participant is counted only once).
|
33
|
+
attr_reader :converted
|
34
|
+
|
35
|
+
# Number of conversions for this alternative (same participant may be counted more than once).
|
36
|
+
attr_reader :conversions
|
37
|
+
|
38
|
+
# Z-score for this alternative, related to 2nd-best performing alternative. Populated by AbTest#score.
|
39
|
+
attr_accessor :z_score
|
40
|
+
|
41
|
+
# Probability derived from z-score. Populated by AbTest#score.
|
42
|
+
attr_accessor :probability
|
43
|
+
|
44
|
+
# Difference from least performing alternative. Populated by AbTest#score.
|
45
|
+
attr_accessor :difference
|
46
|
+
|
47
|
+
# Conversion rate calculated as converted/participants, rounded to 3 places.
|
48
|
+
def conversion_rate
|
49
|
+
@conversion_rate ||= (participants > 0 ? (converted.to_f/participants.to_f * 1000).round / 1000.0 : 0.0)
|
50
|
+
end
|
51
|
+
|
52
|
+
# The measure we use to order (sort) alternatives and decide which one is better (by calculating z-score).
|
53
|
+
# Defaults to conversion rate.
|
54
|
+
def measure
|
55
|
+
conversion_rate
|
56
|
+
end
|
57
|
+
|
58
|
+
def <=>(other)
|
59
|
+
measure <=> other.measure
|
60
|
+
end
|
61
|
+
|
62
|
+
def ==(other)
|
63
|
+
other && id == other.id && experiment == other.experiment
|
64
|
+
end
|
65
|
+
|
66
|
+
def to_s
|
67
|
+
name
|
68
|
+
end
|
69
|
+
|
70
|
+
def inspect
|
71
|
+
"#{name}: #{value} #{converted}/#{participants}"
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
# The meat.
|
78
|
+
class AbTest < Base
|
79
|
+
class << self
|
80
|
+
|
81
|
+
# Convert z-score to probability.
|
82
|
+
def probability(score)
|
83
|
+
score = score.abs
|
84
|
+
probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
|
85
|
+
probability ? probability.last : 0
|
86
|
+
end
|
87
|
+
|
88
|
+
def friendly_name
|
89
|
+
"A/B Test"
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
def initialize(*args)
|
95
|
+
super
|
96
|
+
@alternatives = [false, true]
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
# -- Metric --
|
101
|
+
|
102
|
+
# Tells A/B test which metric we're measuring, or returns metric in use.
|
103
|
+
#
|
104
|
+
# @example Define A/B test against coolness metric
|
105
|
+
# ab_test "Background color" do
|
106
|
+
# metrics :coolness
|
107
|
+
# alternatives "red", "blue", "orange"
|
108
|
+
# end
|
109
|
+
# @example Find metric for A/B test
|
110
|
+
# puts "Measures: " + experiment(:background_color).metrics.map(&:name)
|
111
|
+
def metrics(*args)
|
112
|
+
@metrics = args.map { |id| @playground.metric(id) } unless args.empty?
|
113
|
+
@metrics
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
# -- Alternatives --
|
118
|
+
|
119
|
+
# Call this method once to set alternative values for this experiment
|
120
|
+
# (requires at least two values). Call without arguments to obtain
|
121
|
+
# current list of alternatives.
|
122
|
+
#
|
123
|
+
# @example Define A/B test with three alternatives
|
124
|
+
# ab_test "Background color" do
|
125
|
+
# metrics :coolness
|
126
|
+
# alternatives "red", "blue", "orange"
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
# @example Find out which alternatives this test uses
|
130
|
+
# alts = experiment(:background_color).alternatives
|
131
|
+
# puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
|
132
|
+
def alternatives(*args)
|
133
|
+
unless args.empty?
|
134
|
+
@alternatives = args.clone
|
135
|
+
end
|
136
|
+
class << self
|
137
|
+
define_method :alternatives, instance_method(:_alternatives)
|
138
|
+
end
|
139
|
+
alternatives
|
140
|
+
end
|
141
|
+
|
142
|
+
def _alternatives
|
143
|
+
alts = []
|
144
|
+
@alternatives.each_with_index do |value, i|
|
145
|
+
participants = redis.scard(key("alts:#{i}:participants")).to_i
|
146
|
+
converted = redis.scard(key("alts:#{i}:converted")).to_i
|
147
|
+
conversions = redis[key("alts:#{i}:conversions")].to_i
|
148
|
+
alts << Alternative.new(self, i, value, participants, converted, conversions)
|
149
|
+
end
|
150
|
+
alts
|
151
|
+
end
|
152
|
+
private :_alternatives
|
153
|
+
|
154
|
+
# Returns an Alternative with the specified value.
|
155
|
+
#
|
156
|
+
# @example
|
157
|
+
# alternative(:red) == alternatives[0]
|
158
|
+
# alternative(:blue) == alternatives[2]
|
159
|
+
def alternative(value)
|
160
|
+
alternatives.find { |alt| alt.value == value }
|
161
|
+
end
|
162
|
+
|
163
|
+
# Defines an A/B test with two alternatives: false and true. This is the
|
164
|
+
# default pair of alternatives, so just syntactic sugar for those who love
|
165
|
+
# being explicit.
|
166
|
+
#
|
167
|
+
# @example
|
168
|
+
# ab_test "More bacon" do
|
169
|
+
# metrics :yummyness
|
170
|
+
# false_true
|
171
|
+
# end
|
172
|
+
#
|
173
|
+
def false_true
|
174
|
+
alternatives false, true
|
175
|
+
end
|
176
|
+
alias true_false false_true
|
177
|
+
|
178
|
+
# Chooses a value for this experiment. You probably want to use the
|
179
|
+
# Rails helper method ab_test instead.
|
180
|
+
#
|
181
|
+
# This method picks an alternative for the current identity and returns
|
182
|
+
# the alternative's value. It will consistently choose the same
|
183
|
+
# alternative for the same identity, and randomly split alternatives
|
184
|
+
# between different identities.
|
185
|
+
#
|
186
|
+
# @example
|
187
|
+
# color = experiment(:which_blue).choose
|
188
|
+
def choose
|
189
|
+
if active?
|
190
|
+
identity = identity()
|
191
|
+
index = redis[key("participant:#{identity}:show")]
|
192
|
+
unless index
|
193
|
+
index = alternative_for(identity)
|
194
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
195
|
+
check_completion!
|
196
|
+
end
|
197
|
+
else
|
198
|
+
index = redis[key("outcome")] || alternative_for(identify)
|
199
|
+
end
|
200
|
+
@alternatives[index.to_i]
|
201
|
+
end
|
202
|
+
|
203
|
+
# Returns fingerprint (hash) for given alternative. Can be used to lookup
|
204
|
+
# alternative for experiment without revealing what values are available
|
205
|
+
# (e.g. choosing alternative from HTTP query parameter).
|
206
|
+
def fingerprint(alternative)
|
207
|
+
Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
|
208
|
+
end
|
209
|
+
|
210
|
+
|
211
|
+
# -- Testing --
|
212
|
+
|
213
|
+
# Forces this experiment to use a particular alternative. You'll want to
|
214
|
+
# use this from your test cases to test for the different alternatives.
|
215
|
+
#
|
216
|
+
# @example Setup test to red button
|
217
|
+
# setup do
|
218
|
+
# experiment(:button_color).select(:red)
|
219
|
+
# end
|
220
|
+
#
|
221
|
+
# def test_shows_red_button
|
222
|
+
# . . .
|
223
|
+
# end
|
224
|
+
#
|
225
|
+
# @example Use nil to clear selection
|
226
|
+
# teardown do
|
227
|
+
# experiment(:green_button).select(nil)
|
228
|
+
# end
|
229
|
+
def chooses(value)
|
230
|
+
if value.nil?
|
231
|
+
redis.del key("participant:#{identity}:show")
|
232
|
+
else
|
233
|
+
index = @alternatives.index(value)
|
234
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
235
|
+
redis[key("participant:#{identity}:show")] = index
|
236
|
+
end
|
237
|
+
self
|
238
|
+
end
|
239
|
+
|
240
|
+
# True if this alternative is currently showing (see #chooses).
|
241
|
+
def showing?(alternative)
|
242
|
+
identity = identity()
|
243
|
+
index = redis[key("participant:#{identity}:show")]
|
244
|
+
index && index.to_i == alternative.id
|
245
|
+
end
|
246
|
+
|
247
|
+
|
248
|
+
# -- Reporting --
|
249
|
+
|
250
|
+
# Scores alternatives based on the current tracking data. This method
|
251
|
+
# returns a structure with the following attributes:
|
252
|
+
# [:alts] Ordered list of alternatives, populated with scoring info.
|
253
|
+
# [:base] Second best performing alternative.
|
254
|
+
# [:least] Least performing alternative (but more than zero conversion).
|
255
|
+
# [:choice] Choice alterntive, either the outcome or best alternative.
|
256
|
+
#
|
257
|
+
# Alternatives returned by this method are populated with the following
|
258
|
+
# attributes:
|
259
|
+
# [:z_score] Z-score (relative to the base alternative).
|
260
|
+
# [:probability] Probability (z-score mapped to 0, 90, 95, 99 or 99.9%).
|
261
|
+
# [:difference] Difference from the least performant altenative.
|
262
|
+
#
|
263
|
+
# The choice alternative is set only if its probability is higher or
|
264
|
+
# equal to the specified probability (default is 90%).
|
265
|
+
def score(probability = 90)
|
266
|
+
alts = alternatives
|
267
|
+
# sort by conversion rate to find second best and 2nd best
|
268
|
+
sorted = alts.sort_by(&:measure)
|
269
|
+
base = sorted[-2]
|
270
|
+
# calculate z-score
|
271
|
+
pc = base.measure
|
272
|
+
nc = base.participants
|
273
|
+
alts.each do |alt|
|
274
|
+
p = alt.measure
|
275
|
+
n = alt.participants
|
276
|
+
alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
|
277
|
+
alt.probability = AbTest.probability(alt.z_score)
|
278
|
+
end
|
279
|
+
# difference is measured from least performant
|
280
|
+
if least = sorted.find { |alt| alt.measure > 0 }
|
281
|
+
alts.each do |alt|
|
282
|
+
if alt.measure > least.measure
|
283
|
+
alt.difference = (alt.measure - least.measure) / least.measure * 100
|
284
|
+
end
|
285
|
+
end
|
286
|
+
end
|
287
|
+
# best alternative is one with highest conversion rate (best shot).
|
288
|
+
# choice alternative can only pick best if we have high probability (>90%).
|
289
|
+
best = sorted.last if sorted.last.measure > 0.0
|
290
|
+
choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
|
291
|
+
Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
|
292
|
+
end
|
293
|
+
|
294
|
+
# Use the result of #score to derive a conclusion. Returns an
|
295
|
+
# array of claims.
|
296
|
+
def conclusion(score = score)
|
297
|
+
claims = []
|
298
|
+
participants = score.alts.inject(0) { |t,alt| t + alt.participants }
|
299
|
+
claims << case participants
|
300
|
+
when 0 ; "There are no participants in this experiment yet."
|
301
|
+
when 1 ; "There is one participant in this experiment."
|
302
|
+
else ; "There are #{participants} participants in this experiment."
|
303
|
+
end
|
304
|
+
# only interested in sorted alternatives with conversion
|
305
|
+
sorted = score.alts.select { |alt| alt.measure > 0.0 }.sort_by(&:measure).reverse
|
306
|
+
if sorted.size > 1
|
307
|
+
# start with alternatives that have conversion, from best to worst,
|
308
|
+
# then alternatives with no conversion.
|
309
|
+
sorted |= score.alts
|
310
|
+
# we want a result that's clearly better than 2nd best.
|
311
|
+
best, second = sorted[0], sorted[1]
|
312
|
+
if best.measure > second.measure
|
313
|
+
diff = ((best.measure - second.measure) / second.measure * 100).round
|
314
|
+
better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
|
315
|
+
claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
|
316
|
+
if best.probability >= 90
|
317
|
+
claims << "With %d%% probability this result is statistically significant." % score.best.probability
|
318
|
+
else
|
319
|
+
claims << "This result is not statistically significant, suggest you continue this experiment."
|
320
|
+
end
|
321
|
+
sorted.delete best
|
322
|
+
end
|
323
|
+
sorted.each do |alt|
|
324
|
+
if alt.measure > 0.0
|
325
|
+
claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
|
326
|
+
else
|
327
|
+
claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
|
328
|
+
end
|
329
|
+
end
|
330
|
+
else
|
331
|
+
claims << "This experiment did not run long enough to find a clear winner."
|
332
|
+
end
|
333
|
+
claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
|
334
|
+
claims
|
335
|
+
end
|
336
|
+
|
337
|
+
|
338
|
+
# -- Completion --
|
339
|
+
|
340
|
+
# Defines how the experiment can choose the optimal outcome on completion.
|
341
|
+
#
|
342
|
+
# By default, Vanity will take the best alternative (highest conversion
|
343
|
+
# rate) and use that as the outcome. You experiment may have different
|
344
|
+
# needs, maybe you want the least performing alternative, or factor cost
|
345
|
+
# in the equation?
|
346
|
+
#
|
347
|
+
# The default implementation reads like this:
|
348
|
+
# outcome_is do
|
349
|
+
# a, b = alternatives
|
350
|
+
# # a is expensive, only choose a if it performs 2x better than b
|
351
|
+
# a.measure > b.measure * 2 ? a : b
|
352
|
+
# end
|
353
|
+
def outcome_is(&block)
|
354
|
+
raise ArgumentError, "Missing block" unless block
|
355
|
+
raise "outcome_is already called on this experiment" if @outcome_is
|
356
|
+
@outcome_is = block
|
357
|
+
end
|
358
|
+
|
359
|
+
# Alternative chosen when this experiment completed.
|
360
|
+
def outcome
|
361
|
+
outcome = redis[key("outcome")]
|
362
|
+
outcome && alternatives[outcome.to_i]
|
363
|
+
end
|
364
|
+
|
365
|
+
def complete!
|
366
|
+
return unless active?
|
367
|
+
super
|
368
|
+
if @outcome_is
|
369
|
+
begin
|
370
|
+
result = @outcome_is.call
|
371
|
+
outcome = result.id if result && result.experiment == self
|
372
|
+
rescue
|
373
|
+
# TODO: logging
|
374
|
+
end
|
375
|
+
else
|
376
|
+
best = score.best
|
377
|
+
outcome = best.id if best
|
378
|
+
end
|
379
|
+
# TODO: logging
|
380
|
+
redis.setnx key("outcome"), outcome || 0
|
381
|
+
end
|
382
|
+
|
383
|
+
|
384
|
+
# -- Store/validate --
|
385
|
+
|
386
|
+
def destroy
|
387
|
+
@alternatives.size.times do |i|
|
388
|
+
redis.del key("alts:#{i}:participants")
|
389
|
+
redis.del key("alts:#{i}:converted")
|
390
|
+
redis.del key("alts:#{i}:conversions")
|
391
|
+
end
|
392
|
+
redis.del key(:outcome)
|
393
|
+
super
|
394
|
+
end
|
395
|
+
|
396
|
+
def save
|
397
|
+
fail "Experiment #{name} needs at least two alternatives" unless alternatives.size >= 2
|
398
|
+
super
|
399
|
+
if @metrics.nil? || @metrics.empty?
|
400
|
+
warn "Please use metrics method to explicitly state which metric you are measuring against."
|
401
|
+
metric = @playground.metrics[id] ||= Vanity::Metric.new(@playground, name)
|
402
|
+
@metrics = [metric]
|
403
|
+
end
|
404
|
+
@metrics.each do |metric|
|
405
|
+
metric.hook &method(:track!)
|
406
|
+
end
|
407
|
+
end
|
408
|
+
|
409
|
+
# Called when tracking associated metric.
|
410
|
+
def track!(metric_id, timestamp, count, *args)
|
411
|
+
return unless active?
|
412
|
+
identity = identity() rescue nil
|
413
|
+
if identity
|
414
|
+
return if redis[key("participants:#{identity}:show")]
|
415
|
+
index = alternative_for(identity)
|
416
|
+
redis.sadd key("alts:#{index}:converted"), identity if redis.sismember(key("alts:#{index}:participants"), identity)
|
417
|
+
redis.incrby key("alts:#{index}:conversions"), count
|
418
|
+
check_completion!
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
# If you are not embarrassed by the first version of your product, you’ve
|
423
|
+
# launched too late.
|
424
|
+
# -- Reid Hoffman, founder of LinkedIn
|
425
|
+
|
426
|
+
protected
|
427
|
+
|
428
|
+
# Used for testing.
|
429
|
+
def fake(values)
|
430
|
+
values.each do |value, (participants, conversions)|
|
431
|
+
conversions ||= participants
|
432
|
+
participants.times do |identity|
|
433
|
+
index = @alternatives.index(value)
|
434
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
435
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
436
|
+
end
|
437
|
+
conversions.times do |identity|
|
438
|
+
index = @alternatives.index(value)
|
439
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
440
|
+
redis.sadd key("alts:#{index}:converted"), identity
|
441
|
+
redis.incr key("alts:#{index}:conversions")
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
# Chooses an alternative for the identity and returns its index. This
|
447
|
+
# method always returns the same alternative for a given experiment and
|
448
|
+
# identity, and randomly distributed alternatives for each identity (in the
|
449
|
+
# same experiment).
|
450
|
+
def alternative_for(identity)
|
451
|
+
Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
|
452
|
+
end
|
453
|
+
|
454
|
+
begin
|
455
|
+
a = 50.0
|
456
|
+
# Returns array of [z-score, percentage]
|
457
|
+
norm_dist = []
|
458
|
+
(0.0..3.1).step(0.01) { |x| norm_dist << [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
|
459
|
+
# We're really only interested in 90%, 95%, 99% and 99.9%.
|
460
|
+
Z_TO_PROBABILITY = [90, 95, 99, 99.9].map { |pct| [norm_dist.find { |x,a| a >= pct }.first, pct] }.reverse
|
461
|
+
end
|
462
|
+
|
463
|
+
end
|
464
|
+
|
465
|
+
|
466
|
+
module Definition
|
467
|
+
# Define an A/B test with the given name. For example:
|
468
|
+
# ab_test "New Banner" do
|
469
|
+
# alternatives :red, :green, :blue
|
470
|
+
# end
|
471
|
+
def ab_test(name, &block)
|
472
|
+
define name, :ab_test, &block
|
473
|
+
end
|
474
|
+
end
|
475
|
+
|
476
|
+
end
|
477
|
+
end
|