mikeg-vanity 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +153 -0
- data/MIT-LICENSE +21 -0
- data/README.rdoc +83 -0
- data/bin/vanity +53 -0
- data/lib/vanity.rb +38 -0
- data/lib/vanity/backport.rb +43 -0
- data/lib/vanity/commands.rb +2 -0
- data/lib/vanity/commands/list.rb +21 -0
- data/lib/vanity/commands/report.rb +60 -0
- data/lib/vanity/experiment/ab_test.rb +477 -0
- data/lib/vanity/experiment/base.rb +212 -0
- data/lib/vanity/helpers.rb +59 -0
- data/lib/vanity/metric/active_record.rb +77 -0
- data/lib/vanity/metric/base.rb +221 -0
- data/lib/vanity/metric/google_analytics.rb +70 -0
- data/lib/vanity/mock_redis.rb +76 -0
- data/lib/vanity/playground.rb +197 -0
- data/lib/vanity/rails.rb +22 -0
- data/lib/vanity/rails/dashboard.rb +24 -0
- data/lib/vanity/rails/helpers.rb +158 -0
- data/lib/vanity/rails/testing.rb +11 -0
- data/lib/vanity/templates/_ab_test.erb +26 -0
- data/lib/vanity/templates/_experiment.erb +5 -0
- data/lib/vanity/templates/_experiments.erb +7 -0
- data/lib/vanity/templates/_metric.erb +14 -0
- data/lib/vanity/templates/_metrics.erb +13 -0
- data/lib/vanity/templates/_report.erb +27 -0
- data/lib/vanity/templates/flot.min.js +1 -0
- data/lib/vanity/templates/jquery.min.js +19 -0
- data/lib/vanity/templates/vanity.css +26 -0
- data/lib/vanity/templates/vanity.js +82 -0
- data/test/ab_test_test.rb +656 -0
- data/test/experiment_test.rb +136 -0
- data/test/experiments/age_and_zipcode.rb +19 -0
- data/test/experiments/metrics/cheers.rb +3 -0
- data/test/experiments/metrics/signups.rb +2 -0
- data/test/experiments/metrics/yawns.rb +3 -0
- data/test/experiments/null_abc.rb +5 -0
- data/test/metric_test.rb +518 -0
- data/test/playground_test.rb +10 -0
- data/test/rails_test.rb +104 -0
- data/test/test_helper.rb +135 -0
- data/vanity.gemspec +18 -0
- data/vendor/redis-rb/LICENSE +20 -0
- data/vendor/redis-rb/README.markdown +36 -0
- data/vendor/redis-rb/Rakefile +62 -0
- data/vendor/redis-rb/bench.rb +44 -0
- data/vendor/redis-rb/benchmarking/suite.rb +24 -0
- data/vendor/redis-rb/benchmarking/worker.rb +71 -0
- data/vendor/redis-rb/bin/distredis +33 -0
- data/vendor/redis-rb/examples/basic.rb +16 -0
- data/vendor/redis-rb/examples/incr-decr.rb +18 -0
- data/vendor/redis-rb/examples/list.rb +26 -0
- data/vendor/redis-rb/examples/sets.rb +36 -0
- data/vendor/redis-rb/lib/dist_redis.rb +124 -0
- data/vendor/redis-rb/lib/hash_ring.rb +128 -0
- data/vendor/redis-rb/lib/pipeline.rb +21 -0
- data/vendor/redis-rb/lib/redis.rb +370 -0
- data/vendor/redis-rb/lib/redis/raketasks.rb +1 -0
- data/vendor/redis-rb/profile.rb +22 -0
- data/vendor/redis-rb/redis-rb.gemspec +30 -0
- data/vendor/redis-rb/spec/redis_spec.rb +637 -0
- data/vendor/redis-rb/spec/spec_helper.rb +4 -0
- data/vendor/redis-rb/speed.rb +16 -0
- data/vendor/redis-rb/tasks/redis.tasks.rb +140 -0
- metadata +125 -0
@@ -0,0 +1,477 @@
|
|
1
|
+
require "digest/md5"
|
2
|
+
|
3
|
+
module Vanity
|
4
|
+
module Experiment
|
5
|
+
|
6
|
+
# One of several alternatives in an A/B test (see AbTest#alternatives).
|
7
|
+
class Alternative
|
8
|
+
|
9
|
+
def initialize(experiment, id, value, participants, converted, conversions)
|
10
|
+
@experiment = experiment
|
11
|
+
@id = id
|
12
|
+
@name = "option #{(@id + 65).chr}"
|
13
|
+
@value = value
|
14
|
+
@participants, @converted, @conversions = participants, converted, conversions
|
15
|
+
end
|
16
|
+
|
17
|
+
# Alternative id, only unique for this experiment.
|
18
|
+
attr_reader :id
|
19
|
+
|
20
|
+
# Alternative name (option A, option B, etc).
|
21
|
+
attr_reader :name
|
22
|
+
|
23
|
+
# Alternative value.
|
24
|
+
attr_reader :value
|
25
|
+
|
26
|
+
# Experiment this alternative belongs to.
|
27
|
+
attr_reader :experiment
|
28
|
+
|
29
|
+
# Number of participants who viewed this alternative.
|
30
|
+
attr_reader :participants
|
31
|
+
|
32
|
+
# Number of participants who converted on this alternative (a participant is counted only once).
|
33
|
+
attr_reader :converted
|
34
|
+
|
35
|
+
# Number of conversions for this alternative (same participant may be counted more than once).
|
36
|
+
attr_reader :conversions
|
37
|
+
|
38
|
+
# Z-score for this alternative, related to 2nd-best performing alternative. Populated by AbTest#score.
|
39
|
+
attr_accessor :z_score
|
40
|
+
|
41
|
+
# Probability derived from z-score. Populated by AbTest#score.
|
42
|
+
attr_accessor :probability
|
43
|
+
|
44
|
+
# Difference from least performing alternative. Populated by AbTest#score.
|
45
|
+
attr_accessor :difference
|
46
|
+
|
47
|
+
# Conversion rate calculated as converted/participants, rounded to 3 places.
|
48
|
+
def conversion_rate
|
49
|
+
@conversion_rate ||= (participants > 0 ? (converted.to_f/participants.to_f * 1000).round / 1000.0 : 0.0)
|
50
|
+
end
|
51
|
+
|
52
|
+
# The measure we use to order (sort) alternatives and decide which one is better (by calculating z-score).
|
53
|
+
# Defaults to conversion rate.
|
54
|
+
def measure
|
55
|
+
conversion_rate
|
56
|
+
end
|
57
|
+
|
58
|
+
def <=>(other)
|
59
|
+
measure <=> other.measure
|
60
|
+
end
|
61
|
+
|
62
|
+
def ==(other)
|
63
|
+
other && id == other.id && experiment == other.experiment
|
64
|
+
end
|
65
|
+
|
66
|
+
def to_s
|
67
|
+
name
|
68
|
+
end
|
69
|
+
|
70
|
+
def inspect
|
71
|
+
"#{name}: #{value} #{converted}/#{participants}"
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
# The meat.
|
78
|
+
class AbTest < Base
|
79
|
+
class << self
|
80
|
+
|
81
|
+
# Convert z-score to probability.
|
82
|
+
def probability(score)
|
83
|
+
score = score.abs
|
84
|
+
probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
|
85
|
+
probability ? probability.last : 0
|
86
|
+
end
|
87
|
+
|
88
|
+
def friendly_name
|
89
|
+
"A/B Test"
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
def initialize(*args)
|
95
|
+
super
|
96
|
+
@alternatives = [false, true]
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
# -- Metric --
|
101
|
+
|
102
|
+
# Tells A/B test which metric we're measuring, or returns metric in use.
|
103
|
+
#
|
104
|
+
# @example Define A/B test against coolness metric
|
105
|
+
# ab_test "Background color" do
|
106
|
+
# metrics :coolness
|
107
|
+
# alternatives "red", "blue", "orange"
|
108
|
+
# end
|
109
|
+
# @example Find metric for A/B test
|
110
|
+
# puts "Measures: " + experiment(:background_color).metrics.map(&:name)
|
111
|
+
def metrics(*args)
|
112
|
+
@metrics = args.map { |id| @playground.metric(id) } unless args.empty?
|
113
|
+
@metrics
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
# -- Alternatives --
|
118
|
+
|
119
|
+
# Call this method once to set alternative values for this experiment
|
120
|
+
# (requires at least two values). Call without arguments to obtain
|
121
|
+
# current list of alternatives.
|
122
|
+
#
|
123
|
+
# @example Define A/B test with three alternatives
|
124
|
+
# ab_test "Background color" do
|
125
|
+
# metrics :coolness
|
126
|
+
# alternatives "red", "blue", "orange"
|
127
|
+
# end
|
128
|
+
#
|
129
|
+
# @example Find out which alternatives this test uses
|
130
|
+
# alts = experiment(:background_color).alternatives
|
131
|
+
# puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
|
132
|
+
def alternatives(*args)
|
133
|
+
unless args.empty?
|
134
|
+
@alternatives = args.clone
|
135
|
+
end
|
136
|
+
class << self
|
137
|
+
define_method :alternatives, instance_method(:_alternatives)
|
138
|
+
end
|
139
|
+
alternatives
|
140
|
+
end
|
141
|
+
|
142
|
+
def _alternatives
|
143
|
+
alts = []
|
144
|
+
@alternatives.each_with_index do |value, i|
|
145
|
+
participants = redis.scard(key("alts:#{i}:participants")).to_i
|
146
|
+
converted = redis.scard(key("alts:#{i}:converted")).to_i
|
147
|
+
conversions = redis[key("alts:#{i}:conversions")].to_i
|
148
|
+
alts << Alternative.new(self, i, value, participants, converted, conversions)
|
149
|
+
end
|
150
|
+
alts
|
151
|
+
end
|
152
|
+
private :_alternatives
|
153
|
+
|
154
|
+
# Returns an Alternative with the specified value.
|
155
|
+
#
|
156
|
+
# @example
|
157
|
+
# alternative(:red) == alternatives[0]
|
158
|
+
# alternative(:blue) == alternatives[2]
|
159
|
+
def alternative(value)
|
160
|
+
alternatives.find { |alt| alt.value == value }
|
161
|
+
end
|
162
|
+
|
163
|
+
# Defines an A/B test with two alternatives: false and true. This is the
|
164
|
+
# default pair of alternatives, so just syntactic sugar for those who love
|
165
|
+
# being explicit.
|
166
|
+
#
|
167
|
+
# @example
|
168
|
+
# ab_test "More bacon" do
|
169
|
+
# metrics :yummyness
|
170
|
+
# false_true
|
171
|
+
# end
|
172
|
+
#
|
173
|
+
def false_true
|
174
|
+
alternatives false, true
|
175
|
+
end
|
176
|
+
alias true_false false_true
|
177
|
+
|
178
|
+
# Chooses a value for this experiment. You probably want to use the
|
179
|
+
# Rails helper method ab_test instead.
|
180
|
+
#
|
181
|
+
# This method picks an alternative for the current identity and returns
|
182
|
+
# the alternative's value. It will consistently choose the same
|
183
|
+
# alternative for the same identity, and randomly split alternatives
|
184
|
+
# between different identities.
|
185
|
+
#
|
186
|
+
# @example
|
187
|
+
# color = experiment(:which_blue).choose
|
188
|
+
def choose
|
189
|
+
if active?
|
190
|
+
identity = identity()
|
191
|
+
index = redis[key("participant:#{identity}:show")]
|
192
|
+
unless index
|
193
|
+
index = alternative_for(identity)
|
194
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
195
|
+
check_completion!
|
196
|
+
end
|
197
|
+
else
|
198
|
+
index = redis[key("outcome")] || alternative_for(identify)
|
199
|
+
end
|
200
|
+
@alternatives[index.to_i]
|
201
|
+
end
|
202
|
+
|
203
|
+
# Returns fingerprint (hash) for given alternative. Can be used to lookup
|
204
|
+
# alternative for experiment without revealing what values are available
|
205
|
+
# (e.g. choosing alternative from HTTP query parameter).
|
206
|
+
def fingerprint(alternative)
|
207
|
+
Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
|
208
|
+
end
|
209
|
+
|
210
|
+
|
211
|
+
# -- Testing --
|
212
|
+
|
213
|
+
# Forces this experiment to use a particular alternative. You'll want to
|
214
|
+
# use this from your test cases to test for the different alternatives.
|
215
|
+
#
|
216
|
+
# @example Setup test to red button
|
217
|
+
# setup do
|
218
|
+
# experiment(:button_color).select(:red)
|
219
|
+
# end
|
220
|
+
#
|
221
|
+
# def test_shows_red_button
|
222
|
+
# . . .
|
223
|
+
# end
|
224
|
+
#
|
225
|
+
# @example Use nil to clear selection
|
226
|
+
# teardown do
|
227
|
+
# experiment(:green_button).select(nil)
|
228
|
+
# end
|
229
|
+
def chooses(value)
|
230
|
+
if value.nil?
|
231
|
+
redis.del key("participant:#{identity}:show")
|
232
|
+
else
|
233
|
+
index = @alternatives.index(value)
|
234
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
235
|
+
redis[key("participant:#{identity}:show")] = index
|
236
|
+
end
|
237
|
+
self
|
238
|
+
end
|
239
|
+
|
240
|
+
# True if this alternative is currently showing (see #chooses).
|
241
|
+
def showing?(alternative)
|
242
|
+
identity = identity()
|
243
|
+
index = redis[key("participant:#{identity}:show")]
|
244
|
+
index && index.to_i == alternative.id
|
245
|
+
end
|
246
|
+
|
247
|
+
|
248
|
+
# -- Reporting --
|
249
|
+
|
250
|
+
# Scores alternatives based on the current tracking data. This method
|
251
|
+
# returns a structure with the following attributes:
|
252
|
+
# [:alts] Ordered list of alternatives, populated with scoring info.
|
253
|
+
# [:base] Second best performing alternative.
|
254
|
+
# [:least] Least performing alternative (but more than zero conversion).
|
255
|
+
# [:choice] Choice alterntive, either the outcome or best alternative.
|
256
|
+
#
|
257
|
+
# Alternatives returned by this method are populated with the following
|
258
|
+
# attributes:
|
259
|
+
# [:z_score] Z-score (relative to the base alternative).
|
260
|
+
# [:probability] Probability (z-score mapped to 0, 90, 95, 99 or 99.9%).
|
261
|
+
# [:difference] Difference from the least performant altenative.
|
262
|
+
#
|
263
|
+
# The choice alternative is set only if its probability is higher or
|
264
|
+
# equal to the specified probability (default is 90%).
|
265
|
+
def score(probability = 90)
|
266
|
+
alts = alternatives
|
267
|
+
# sort by conversion rate to find second best and 2nd best
|
268
|
+
sorted = alts.sort_by(&:measure)
|
269
|
+
base = sorted[-2]
|
270
|
+
# calculate z-score
|
271
|
+
pc = base.measure
|
272
|
+
nc = base.participants
|
273
|
+
alts.each do |alt|
|
274
|
+
p = alt.measure
|
275
|
+
n = alt.participants
|
276
|
+
alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
|
277
|
+
alt.probability = AbTest.probability(alt.z_score)
|
278
|
+
end
|
279
|
+
# difference is measured from least performant
|
280
|
+
if least = sorted.find { |alt| alt.measure > 0 }
|
281
|
+
alts.each do |alt|
|
282
|
+
if alt.measure > least.measure
|
283
|
+
alt.difference = (alt.measure - least.measure) / least.measure * 100
|
284
|
+
end
|
285
|
+
end
|
286
|
+
end
|
287
|
+
# best alternative is one with highest conversion rate (best shot).
|
288
|
+
# choice alternative can only pick best if we have high probability (>90%).
|
289
|
+
best = sorted.last if sorted.last.measure > 0.0
|
290
|
+
choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
|
291
|
+
Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
|
292
|
+
end
|
293
|
+
|
294
|
+
# Use the result of #score to derive a conclusion. Returns an
|
295
|
+
# array of claims.
|
296
|
+
def conclusion(score = score)
|
297
|
+
claims = []
|
298
|
+
participants = score.alts.inject(0) { |t,alt| t + alt.participants }
|
299
|
+
claims << case participants
|
300
|
+
when 0 ; "There are no participants in this experiment yet."
|
301
|
+
when 1 ; "There is one participant in this experiment."
|
302
|
+
else ; "There are #{participants} participants in this experiment."
|
303
|
+
end
|
304
|
+
# only interested in sorted alternatives with conversion
|
305
|
+
sorted = score.alts.select { |alt| alt.measure > 0.0 }.sort_by(&:measure).reverse
|
306
|
+
if sorted.size > 1
|
307
|
+
# start with alternatives that have conversion, from best to worst,
|
308
|
+
# then alternatives with no conversion.
|
309
|
+
sorted |= score.alts
|
310
|
+
# we want a result that's clearly better than 2nd best.
|
311
|
+
best, second = sorted[0], sorted[1]
|
312
|
+
if best.measure > second.measure
|
313
|
+
diff = ((best.measure - second.measure) / second.measure * 100).round
|
314
|
+
better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
|
315
|
+
claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
|
316
|
+
if best.probability >= 90
|
317
|
+
claims << "With %d%% probability this result is statistically significant." % score.best.probability
|
318
|
+
else
|
319
|
+
claims << "This result is not statistically significant, suggest you continue this experiment."
|
320
|
+
end
|
321
|
+
sorted.delete best
|
322
|
+
end
|
323
|
+
sorted.each do |alt|
|
324
|
+
if alt.measure > 0.0
|
325
|
+
claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
|
326
|
+
else
|
327
|
+
claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
|
328
|
+
end
|
329
|
+
end
|
330
|
+
else
|
331
|
+
claims << "This experiment did not run long enough to find a clear winner."
|
332
|
+
end
|
333
|
+
claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
|
334
|
+
claims
|
335
|
+
end
|
336
|
+
|
337
|
+
|
338
|
+
# -- Completion --
|
339
|
+
|
340
|
+
# Defines how the experiment can choose the optimal outcome on completion.
|
341
|
+
#
|
342
|
+
# By default, Vanity will take the best alternative (highest conversion
|
343
|
+
# rate) and use that as the outcome. You experiment may have different
|
344
|
+
# needs, maybe you want the least performing alternative, or factor cost
|
345
|
+
# in the equation?
|
346
|
+
#
|
347
|
+
# The default implementation reads like this:
|
348
|
+
# outcome_is do
|
349
|
+
# a, b = alternatives
|
350
|
+
# # a is expensive, only choose a if it performs 2x better than b
|
351
|
+
# a.measure > b.measure * 2 ? a : b
|
352
|
+
# end
|
353
|
+
def outcome_is(&block)
|
354
|
+
raise ArgumentError, "Missing block" unless block
|
355
|
+
raise "outcome_is already called on this experiment" if @outcome_is
|
356
|
+
@outcome_is = block
|
357
|
+
end
|
358
|
+
|
359
|
+
# Alternative chosen when this experiment completed.
|
360
|
+
def outcome
|
361
|
+
outcome = redis[key("outcome")]
|
362
|
+
outcome && alternatives[outcome.to_i]
|
363
|
+
end
|
364
|
+
|
365
|
+
def complete!
|
366
|
+
return unless active?
|
367
|
+
super
|
368
|
+
if @outcome_is
|
369
|
+
begin
|
370
|
+
result = @outcome_is.call
|
371
|
+
outcome = result.id if result && result.experiment == self
|
372
|
+
rescue
|
373
|
+
# TODO: logging
|
374
|
+
end
|
375
|
+
else
|
376
|
+
best = score.best
|
377
|
+
outcome = best.id if best
|
378
|
+
end
|
379
|
+
# TODO: logging
|
380
|
+
redis.setnx key("outcome"), outcome || 0
|
381
|
+
end
|
382
|
+
|
383
|
+
|
384
|
+
# -- Store/validate --
|
385
|
+
|
386
|
+
def destroy
|
387
|
+
@alternatives.size.times do |i|
|
388
|
+
redis.del key("alts:#{i}:participants")
|
389
|
+
redis.del key("alts:#{i}:converted")
|
390
|
+
redis.del key("alts:#{i}:conversions")
|
391
|
+
end
|
392
|
+
redis.del key(:outcome)
|
393
|
+
super
|
394
|
+
end
|
395
|
+
|
396
|
+
def save
|
397
|
+
fail "Experiment #{name} needs at least two alternatives" unless alternatives.size >= 2
|
398
|
+
super
|
399
|
+
if @metrics.nil? || @metrics.empty?
|
400
|
+
warn "Please use metrics method to explicitly state which metric you are measuring against."
|
401
|
+
metric = @playground.metrics[id] ||= Vanity::Metric.new(@playground, name)
|
402
|
+
@metrics = [metric]
|
403
|
+
end
|
404
|
+
@metrics.each do |metric|
|
405
|
+
metric.hook &method(:track!)
|
406
|
+
end
|
407
|
+
end
|
408
|
+
|
409
|
+
# Called when tracking associated metric.
|
410
|
+
def track!(metric_id, timestamp, count, *args)
|
411
|
+
return unless active?
|
412
|
+
identity = identity() rescue nil
|
413
|
+
if identity
|
414
|
+
return if redis[key("participants:#{identity}:show")]
|
415
|
+
index = alternative_for(identity)
|
416
|
+
redis.sadd key("alts:#{index}:converted"), identity if redis.sismember(key("alts:#{index}:participants"), identity)
|
417
|
+
redis.incrby key("alts:#{index}:conversions"), count
|
418
|
+
check_completion!
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
# If you are not embarrassed by the first version of your product, you’ve
|
423
|
+
# launched too late.
|
424
|
+
# -- Reid Hoffman, founder of LinkedIn
|
425
|
+
|
426
|
+
protected
|
427
|
+
|
428
|
+
# Used for testing.
|
429
|
+
def fake(values)
|
430
|
+
values.each do |value, (participants, conversions)|
|
431
|
+
conversions ||= participants
|
432
|
+
participants.times do |identity|
|
433
|
+
index = @alternatives.index(value)
|
434
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
435
|
+
redis.sadd key("alts:#{index}:participants"), identity
|
436
|
+
end
|
437
|
+
conversions.times do |identity|
|
438
|
+
index = @alternatives.index(value)
|
439
|
+
raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
|
440
|
+
redis.sadd key("alts:#{index}:converted"), identity
|
441
|
+
redis.incr key("alts:#{index}:conversions")
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
# Chooses an alternative for the identity and returns its index. This
|
447
|
+
# method always returns the same alternative for a given experiment and
|
448
|
+
# identity, and randomly distributed alternatives for each identity (in the
|
449
|
+
# same experiment).
|
450
|
+
def alternative_for(identity)
|
451
|
+
Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
|
452
|
+
end
|
453
|
+
|
454
|
+
begin
|
455
|
+
a = 50.0
|
456
|
+
# Returns array of [z-score, percentage]
|
457
|
+
norm_dist = []
|
458
|
+
(0.0..3.1).step(0.01) { |x| norm_dist << [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
|
459
|
+
# We're really only interested in 90%, 95%, 99% and 99.9%.
|
460
|
+
Z_TO_PROBABILITY = [90, 95, 99, 99.9].map { |pct| [norm_dist.find { |x,a| a >= pct }.first, pct] }.reverse
|
461
|
+
end
|
462
|
+
|
463
|
+
end
|
464
|
+
|
465
|
+
|
466
|
+
module Definition
|
467
|
+
# Define an A/B test with the given name. For example:
|
468
|
+
# ab_test "New Banner" do
|
469
|
+
# alternatives :red, :green, :blue
|
470
|
+
# end
|
471
|
+
def ab_test(name, &block)
|
472
|
+
define name, :ab_test, &block
|
473
|
+
end
|
474
|
+
end
|
475
|
+
|
476
|
+
end
|
477
|
+
end
|