mikeg-vanity 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. data/CHANGELOG +153 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README.rdoc +83 -0
  4. data/bin/vanity +53 -0
  5. data/lib/vanity.rb +38 -0
  6. data/lib/vanity/backport.rb +43 -0
  7. data/lib/vanity/commands.rb +2 -0
  8. data/lib/vanity/commands/list.rb +21 -0
  9. data/lib/vanity/commands/report.rb +60 -0
  10. data/lib/vanity/experiment/ab_test.rb +477 -0
  11. data/lib/vanity/experiment/base.rb +212 -0
  12. data/lib/vanity/helpers.rb +59 -0
  13. data/lib/vanity/metric/active_record.rb +77 -0
  14. data/lib/vanity/metric/base.rb +221 -0
  15. data/lib/vanity/metric/google_analytics.rb +70 -0
  16. data/lib/vanity/mock_redis.rb +76 -0
  17. data/lib/vanity/playground.rb +197 -0
  18. data/lib/vanity/rails.rb +22 -0
  19. data/lib/vanity/rails/dashboard.rb +24 -0
  20. data/lib/vanity/rails/helpers.rb +158 -0
  21. data/lib/vanity/rails/testing.rb +11 -0
  22. data/lib/vanity/templates/_ab_test.erb +26 -0
  23. data/lib/vanity/templates/_experiment.erb +5 -0
  24. data/lib/vanity/templates/_experiments.erb +7 -0
  25. data/lib/vanity/templates/_metric.erb +14 -0
  26. data/lib/vanity/templates/_metrics.erb +13 -0
  27. data/lib/vanity/templates/_report.erb +27 -0
  28. data/lib/vanity/templates/flot.min.js +1 -0
  29. data/lib/vanity/templates/jquery.min.js +19 -0
  30. data/lib/vanity/templates/vanity.css +26 -0
  31. data/lib/vanity/templates/vanity.js +82 -0
  32. data/test/ab_test_test.rb +656 -0
  33. data/test/experiment_test.rb +136 -0
  34. data/test/experiments/age_and_zipcode.rb +19 -0
  35. data/test/experiments/metrics/cheers.rb +3 -0
  36. data/test/experiments/metrics/signups.rb +2 -0
  37. data/test/experiments/metrics/yawns.rb +3 -0
  38. data/test/experiments/null_abc.rb +5 -0
  39. data/test/metric_test.rb +518 -0
  40. data/test/playground_test.rb +10 -0
  41. data/test/rails_test.rb +104 -0
  42. data/test/test_helper.rb +135 -0
  43. data/vanity.gemspec +18 -0
  44. data/vendor/redis-rb/LICENSE +20 -0
  45. data/vendor/redis-rb/README.markdown +36 -0
  46. data/vendor/redis-rb/Rakefile +62 -0
  47. data/vendor/redis-rb/bench.rb +44 -0
  48. data/vendor/redis-rb/benchmarking/suite.rb +24 -0
  49. data/vendor/redis-rb/benchmarking/worker.rb +71 -0
  50. data/vendor/redis-rb/bin/distredis +33 -0
  51. data/vendor/redis-rb/examples/basic.rb +16 -0
  52. data/vendor/redis-rb/examples/incr-decr.rb +18 -0
  53. data/vendor/redis-rb/examples/list.rb +26 -0
  54. data/vendor/redis-rb/examples/sets.rb +36 -0
  55. data/vendor/redis-rb/lib/dist_redis.rb +124 -0
  56. data/vendor/redis-rb/lib/hash_ring.rb +128 -0
  57. data/vendor/redis-rb/lib/pipeline.rb +21 -0
  58. data/vendor/redis-rb/lib/redis.rb +370 -0
  59. data/vendor/redis-rb/lib/redis/raketasks.rb +1 -0
  60. data/vendor/redis-rb/profile.rb +22 -0
  61. data/vendor/redis-rb/redis-rb.gemspec +30 -0
  62. data/vendor/redis-rb/spec/redis_spec.rb +637 -0
  63. data/vendor/redis-rb/spec/spec_helper.rb +4 -0
  64. data/vendor/redis-rb/speed.rb +16 -0
  65. data/vendor/redis-rb/tasks/redis.tasks.rb +140 -0
  66. metadata +125 -0
@@ -0,0 +1,477 @@
1
+ require "digest/md5"
2
+
3
+ module Vanity
4
+ module Experiment
5
+
6
+ # One of several alternatives in an A/B test (see AbTest#alternatives).
7
+ class Alternative
8
+
9
+ def initialize(experiment, id, value, participants, converted, conversions)
10
+ @experiment = experiment
11
+ @id = id
12
+ @name = "option #{(@id + 65).chr}"
13
+ @value = value
14
+ @participants, @converted, @conversions = participants, converted, conversions
15
+ end
16
+
17
+ # Alternative id, only unique for this experiment.
18
+ attr_reader :id
19
+
20
+ # Alternative name (option A, option B, etc).
21
+ attr_reader :name
22
+
23
+ # Alternative value.
24
+ attr_reader :value
25
+
26
+ # Experiment this alternative belongs to.
27
+ attr_reader :experiment
28
+
29
+ # Number of participants who viewed this alternative.
30
+ attr_reader :participants
31
+
32
+ # Number of participants who converted on this alternative (a participant is counted only once).
33
+ attr_reader :converted
34
+
35
+ # Number of conversions for this alternative (same participant may be counted more than once).
36
+ attr_reader :conversions
37
+
38
+ # Z-score for this alternative, related to 2nd-best performing alternative. Populated by AbTest#score.
39
+ attr_accessor :z_score
40
+
41
+ # Probability derived from z-score. Populated by AbTest#score.
42
+ attr_accessor :probability
43
+
44
+ # Difference from least performing alternative. Populated by AbTest#score.
45
+ attr_accessor :difference
46
+
47
+ # Conversion rate calculated as converted/participants, rounded to 3 places.
48
+ def conversion_rate
49
+ @conversion_rate ||= (participants > 0 ? (converted.to_f/participants.to_f * 1000).round / 1000.0 : 0.0)
50
+ end
51
+
52
+ # The measure we use to order (sort) alternatives and decide which one is better (by calculating z-score).
53
+ # Defaults to conversion rate.
54
+ def measure
55
+ conversion_rate
56
+ end
57
+
58
+ def <=>(other)
59
+ measure <=> other.measure
60
+ end
61
+
62
+ def ==(other)
63
+ other && id == other.id && experiment == other.experiment
64
+ end
65
+
66
+ def to_s
67
+ name
68
+ end
69
+
70
+ def inspect
71
+ "#{name}: #{value} #{converted}/#{participants}"
72
+ end
73
+
74
+ end
75
+
76
+
77
+ # The meat.
78
+ class AbTest < Base
79
+ class << self
80
+
81
+ # Convert z-score to probability.
82
+ def probability(score)
83
+ score = score.abs
84
+ probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
85
+ probability ? probability.last : 0
86
+ end
87
+
88
+ def friendly_name
89
+ "A/B Test"
90
+ end
91
+
92
+ end
93
+
94
+ def initialize(*args)
95
+ super
96
+ @alternatives = [false, true]
97
+ end
98
+
99
+
100
+ # -- Metric --
101
+
102
+ # Tells A/B test which metric we're measuring, or returns metric in use.
103
+ #
104
+ # @example Define A/B test against coolness metric
105
+ # ab_test "Background color" do
106
+ # metrics :coolness
107
+ # alternatives "red", "blue", "orange"
108
+ # end
109
+ # @example Find metric for A/B test
110
+ # puts "Measures: " + experiment(:background_color).metrics.map(&:name)
111
+ def metrics(*args)
112
+ @metrics = args.map { |id| @playground.metric(id) } unless args.empty?
113
+ @metrics
114
+ end
115
+
116
+
117
+ # -- Alternatives --
118
+
119
+ # Call this method once to set alternative values for this experiment
120
+ # (requires at least two values). Call without arguments to obtain
121
+ # current list of alternatives.
122
+ #
123
+ # @example Define A/B test with three alternatives
124
+ # ab_test "Background color" do
125
+ # metrics :coolness
126
+ # alternatives "red", "blue", "orange"
127
+ # end
128
+ #
129
+ # @example Find out which alternatives this test uses
130
+ # alts = experiment(:background_color).alternatives
131
+ # puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
132
+ def alternatives(*args)
133
+ unless args.empty?
134
+ @alternatives = args.clone
135
+ end
136
+ class << self
137
+ define_method :alternatives, instance_method(:_alternatives)
138
+ end
139
+ alternatives
140
+ end
141
+
142
+ def _alternatives
143
+ alts = []
144
+ @alternatives.each_with_index do |value, i|
145
+ participants = redis.scard(key("alts:#{i}:participants")).to_i
146
+ converted = redis.scard(key("alts:#{i}:converted")).to_i
147
+ conversions = redis[key("alts:#{i}:conversions")].to_i
148
+ alts << Alternative.new(self, i, value, participants, converted, conversions)
149
+ end
150
+ alts
151
+ end
152
+ private :_alternatives
153
+
154
+ # Returns an Alternative with the specified value.
155
+ #
156
+ # @example
157
+ # alternative(:red) == alternatives[0]
158
+ # alternative(:blue) == alternatives[2]
159
+ def alternative(value)
160
+ alternatives.find { |alt| alt.value == value }
161
+ end
162
+
163
+ # Defines an A/B test with two alternatives: false and true. This is the
164
+ # default pair of alternatives, so just syntactic sugar for those who love
165
+ # being explicit.
166
+ #
167
+ # @example
168
+ # ab_test "More bacon" do
169
+ # metrics :yummyness
170
+ # false_true
171
+ # end
172
+ #
173
+ def false_true
174
+ alternatives false, true
175
+ end
176
+ alias true_false false_true
177
+
178
+ # Chooses a value for this experiment. You probably want to use the
179
+ # Rails helper method ab_test instead.
180
+ #
181
+ # This method picks an alternative for the current identity and returns
182
+ # the alternative's value. It will consistently choose the same
183
+ # alternative for the same identity, and randomly split alternatives
184
+ # between different identities.
185
+ #
186
+ # @example
187
+ # color = experiment(:which_blue).choose
188
+ def choose
189
+ if active?
190
+ identity = identity()
191
+ index = redis[key("participant:#{identity}:show")]
192
+ unless index
193
+ index = alternative_for(identity)
194
+ redis.sadd key("alts:#{index}:participants"), identity
195
+ check_completion!
196
+ end
197
+ else
198
+ index = redis[key("outcome")] || alternative_for(identify)
199
+ end
200
+ @alternatives[index.to_i]
201
+ end
202
+
203
+ # Returns fingerprint (hash) for given alternative. Can be used to lookup
204
+ # alternative for experiment without revealing what values are available
205
+ # (e.g. choosing alternative from HTTP query parameter).
206
+ def fingerprint(alternative)
207
+ Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
208
+ end
209
+
210
+
211
+ # -- Testing --
212
+
213
+ # Forces this experiment to use a particular alternative. You'll want to
214
+ # use this from your test cases to test for the different alternatives.
215
+ #
216
+ # @example Setup test to red button
217
+ # setup do
218
+ # experiment(:button_color).select(:red)
219
+ # end
220
+ #
221
+ # def test_shows_red_button
222
+ # . . .
223
+ # end
224
+ #
225
+ # @example Use nil to clear selection
226
+ # teardown do
227
+ # experiment(:green_button).select(nil)
228
+ # end
229
+ def chooses(value)
230
+ if value.nil?
231
+ redis.del key("participant:#{identity}:show")
232
+ else
233
+ index = @alternatives.index(value)
234
+ raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
235
+ redis[key("participant:#{identity}:show")] = index
236
+ end
237
+ self
238
+ end
239
+
240
+ # True if this alternative is currently showing (see #chooses).
241
+ def showing?(alternative)
242
+ identity = identity()
243
+ index = redis[key("participant:#{identity}:show")]
244
+ index && index.to_i == alternative.id
245
+ end
246
+
247
+
248
+ # -- Reporting --
249
+
250
+ # Scores alternatives based on the current tracking data. This method
251
+ # returns a structure with the following attributes:
252
+ # [:alts] Ordered list of alternatives, populated with scoring info.
253
+ # [:base] Second best performing alternative.
254
+ # [:least] Least performing alternative (but more than zero conversion).
255
+ # [:choice] Choice alterntive, either the outcome or best alternative.
256
+ #
257
+ # Alternatives returned by this method are populated with the following
258
+ # attributes:
259
+ # [:z_score] Z-score (relative to the base alternative).
260
+ # [:probability] Probability (z-score mapped to 0, 90, 95, 99 or 99.9%).
261
+ # [:difference] Difference from the least performant altenative.
262
+ #
263
+ # The choice alternative is set only if its probability is higher or
264
+ # equal to the specified probability (default is 90%).
265
+ def score(probability = 90)
266
+ alts = alternatives
267
+ # sort by conversion rate to find second best and 2nd best
268
+ sorted = alts.sort_by(&:measure)
269
+ base = sorted[-2]
270
+ # calculate z-score
271
+ pc = base.measure
272
+ nc = base.participants
273
+ alts.each do |alt|
274
+ p = alt.measure
275
+ n = alt.participants
276
+ alt.z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
277
+ alt.probability = AbTest.probability(alt.z_score)
278
+ end
279
+ # difference is measured from least performant
280
+ if least = sorted.find { |alt| alt.measure > 0 }
281
+ alts.each do |alt|
282
+ if alt.measure > least.measure
283
+ alt.difference = (alt.measure - least.measure) / least.measure * 100
284
+ end
285
+ end
286
+ end
287
+ # best alternative is one with highest conversion rate (best shot).
288
+ # choice alternative can only pick best if we have high probability (>90%).
289
+ best = sorted.last if sorted.last.measure > 0.0
290
+ choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
291
+ Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
292
+ end
293
+
294
+ # Use the result of #score to derive a conclusion. Returns an
295
+ # array of claims.
296
+ def conclusion(score = score)
297
+ claims = []
298
+ participants = score.alts.inject(0) { |t,alt| t + alt.participants }
299
+ claims << case participants
300
+ when 0 ; "There are no participants in this experiment yet."
301
+ when 1 ; "There is one participant in this experiment."
302
+ else ; "There are #{participants} participants in this experiment."
303
+ end
304
+ # only interested in sorted alternatives with conversion
305
+ sorted = score.alts.select { |alt| alt.measure > 0.0 }.sort_by(&:measure).reverse
306
+ if sorted.size > 1
307
+ # start with alternatives that have conversion, from best to worst,
308
+ # then alternatives with no conversion.
309
+ sorted |= score.alts
310
+ # we want a result that's clearly better than 2nd best.
311
+ best, second = sorted[0], sorted[1]
312
+ if best.measure > second.measure
313
+ diff = ((best.measure - second.measure) / second.measure * 100).round
314
+ better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
315
+ claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
316
+ if best.probability >= 90
317
+ claims << "With %d%% probability this result is statistically significant." % score.best.probability
318
+ else
319
+ claims << "This result is not statistically significant, suggest you continue this experiment."
320
+ end
321
+ sorted.delete best
322
+ end
323
+ sorted.each do |alt|
324
+ if alt.measure > 0.0
325
+ claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
326
+ else
327
+ claims << "%s did not convert." % alt.name.gsub(/^o/, "O")
328
+ end
329
+ end
330
+ else
331
+ claims << "This experiment did not run long enough to find a clear winner."
332
+ end
333
+ claims << "#{score.choice.name.gsub(/^o/, "O")} selected as the best alternative." if score.choice
334
+ claims
335
+ end
336
+
337
+
338
+ # -- Completion --
339
+
340
+ # Defines how the experiment can choose the optimal outcome on completion.
341
+ #
342
+ # By default, Vanity will take the best alternative (highest conversion
343
+ # rate) and use that as the outcome. You experiment may have different
344
+ # needs, maybe you want the least performing alternative, or factor cost
345
+ # in the equation?
346
+ #
347
+ # The default implementation reads like this:
348
+ # outcome_is do
349
+ # a, b = alternatives
350
+ # # a is expensive, only choose a if it performs 2x better than b
351
+ # a.measure > b.measure * 2 ? a : b
352
+ # end
353
+ def outcome_is(&block)
354
+ raise ArgumentError, "Missing block" unless block
355
+ raise "outcome_is already called on this experiment" if @outcome_is
356
+ @outcome_is = block
357
+ end
358
+
359
+ # Alternative chosen when this experiment completed.
360
+ def outcome
361
+ outcome = redis[key("outcome")]
362
+ outcome && alternatives[outcome.to_i]
363
+ end
364
+
365
+ def complete!
366
+ return unless active?
367
+ super
368
+ if @outcome_is
369
+ begin
370
+ result = @outcome_is.call
371
+ outcome = result.id if result && result.experiment == self
372
+ rescue
373
+ # TODO: logging
374
+ end
375
+ else
376
+ best = score.best
377
+ outcome = best.id if best
378
+ end
379
+ # TODO: logging
380
+ redis.setnx key("outcome"), outcome || 0
381
+ end
382
+
383
+
384
+ # -- Store/validate --
385
+
386
+ def destroy
387
+ @alternatives.size.times do |i|
388
+ redis.del key("alts:#{i}:participants")
389
+ redis.del key("alts:#{i}:converted")
390
+ redis.del key("alts:#{i}:conversions")
391
+ end
392
+ redis.del key(:outcome)
393
+ super
394
+ end
395
+
396
+ def save
397
+ fail "Experiment #{name} needs at least two alternatives" unless alternatives.size >= 2
398
+ super
399
+ if @metrics.nil? || @metrics.empty?
400
+ warn "Please use metrics method to explicitly state which metric you are measuring against."
401
+ metric = @playground.metrics[id] ||= Vanity::Metric.new(@playground, name)
402
+ @metrics = [metric]
403
+ end
404
+ @metrics.each do |metric|
405
+ metric.hook &method(:track!)
406
+ end
407
+ end
408
+
409
+ # Called when tracking associated metric.
410
+ def track!(metric_id, timestamp, count, *args)
411
+ return unless active?
412
+ identity = identity() rescue nil
413
+ if identity
414
+ return if redis[key("participants:#{identity}:show")]
415
+ index = alternative_for(identity)
416
+ redis.sadd key("alts:#{index}:converted"), identity if redis.sismember(key("alts:#{index}:participants"), identity)
417
+ redis.incrby key("alts:#{index}:conversions"), count
418
+ check_completion!
419
+ end
420
+ end
421
+
422
+ # If you are not embarrassed by the first version of your product, you’ve
423
+ # launched too late.
424
+ # -- Reid Hoffman, founder of LinkedIn
425
+
426
+ protected
427
+
428
+ # Used for testing.
429
+ def fake(values)
430
+ values.each do |value, (participants, conversions)|
431
+ conversions ||= participants
432
+ participants.times do |identity|
433
+ index = @alternatives.index(value)
434
+ raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
435
+ redis.sadd key("alts:#{index}:participants"), identity
436
+ end
437
+ conversions.times do |identity|
438
+ index = @alternatives.index(value)
439
+ raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
440
+ redis.sadd key("alts:#{index}:converted"), identity
441
+ redis.incr key("alts:#{index}:conversions")
442
+ end
443
+ end
444
+ end
445
+
446
+ # Chooses an alternative for the identity and returns its index. This
447
+ # method always returns the same alternative for a given experiment and
448
+ # identity, and randomly distributed alternatives for each identity (in the
449
+ # same experiment).
450
+ def alternative_for(identity)
451
+ Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
452
+ end
453
+
454
+ begin
455
+ a = 50.0
456
+ # Returns array of [z-score, percentage]
457
+ norm_dist = []
458
+ (0.0..3.1).step(0.01) { |x| norm_dist << [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
459
+ # We're really only interested in 90%, 95%, 99% and 99.9%.
460
+ Z_TO_PROBABILITY = [90, 95, 99, 99.9].map { |pct| [norm_dist.find { |x,a| a >= pct }.first, pct] }.reverse
461
+ end
462
+
463
+ end
464
+
465
+
466
+ module Definition
467
+ # Define an A/B test with the given name. For example:
468
+ # ab_test "New Banner" do
469
+ # alternatives :red, :green, :blue
470
+ # end
471
+ def ab_test(name, &block)
472
+ define name, :ab_test, &block
473
+ end
474
+ end
475
+
476
+ end
477
+ end