vanity 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/CHANGELOG +34 -0
  2. data/Gemfile +16 -0
  3. data/MIT-LICENSE +1 -1
  4. data/README.rdoc +10 -5
  5. data/Rakefile +119 -0
  6. data/bin/vanity +23 -18
  7. data/lib/vanity.rb +12 -4
  8. data/lib/vanity/commands.rb +1 -0
  9. data/lib/vanity/commands/list.rb +21 -0
  10. data/lib/vanity/experiment/ab_test.rb +8 -1
  11. data/lib/vanity/experiment/base.rb +40 -30
  12. data/lib/vanity/frameworks/rails.rb +222 -0
  13. data/lib/vanity/metric/active_record.rb +77 -0
  14. data/lib/vanity/{metric.rb → metric/base.rb} +6 -71
  15. data/lib/vanity/metric/google_analytics.rb +76 -0
  16. data/lib/vanity/playground.rb +93 -44
  17. data/lib/vanity/templates/_metric.erb +12 -7
  18. data/lib/vanity/templates/vanity.css +1 -0
  19. data/test/ab_test_test.rb +69 -48
  20. data/test/experiment_test.rb +29 -15
  21. data/test/metric_test.rb +104 -0
  22. data/test/myapp/app/controllers/application_controller.rb +2 -0
  23. data/test/myapp/app/controllers/main_controller.rb +7 -0
  24. data/test/myapp/config/boot.rb +110 -0
  25. data/test/myapp/config/environment.rb +10 -0
  26. data/test/myapp/config/environments/production.rb +0 -0
  27. data/test/myapp/config/routes.rb +3 -0
  28. data/test/myapp/log/production.log +80 -0
  29. data/test/passenger_test.rb +34 -0
  30. data/test/rails_test.rb +129 -1
  31. data/test/test_helper.rb +12 -4
  32. data/vanity.gemspec +2 -2
  33. data/vendor/cache/RedCloth-4.2.2.gem +0 -0
  34. data/vendor/cache/actionmailer-2.3.5.gem +0 -0
  35. data/vendor/cache/actionpack-2.3.5.gem +0 -0
  36. data/vendor/cache/activerecord-2.3.5.gem +0 -0
  37. data/vendor/cache/activeresource-2.3.5.gem +0 -0
  38. data/vendor/cache/activesupport-2.3.5.gem +0 -0
  39. data/vendor/cache/autotest-4.2.7.gem +0 -0
  40. data/vendor/cache/autotest-fsevent-0.2.1.gem +0 -0
  41. data/vendor/cache/autotest-growl-0.2.0.gem +0 -0
  42. data/vendor/cache/bundler-0.9.7.gem +0 -0
  43. data/vendor/cache/classifier-1.3.1.gem +0 -0
  44. data/vendor/cache/directory_watcher-1.3.1.gem +0 -0
  45. data/vendor/cache/fastthread-1.0.7.gem +0 -0
  46. data/vendor/cache/garb-0.7.0.gem +0 -0
  47. data/vendor/cache/happymapper-0.3.0.gem +0 -0
  48. data/vendor/cache/jekyll-0.5.7.gem +0 -0
  49. data/vendor/cache/libxml-ruby-1.1.3.gem +0 -0
  50. data/vendor/cache/liquid-2.0.0.gem +0 -0
  51. data/vendor/cache/maruku-0.6.0.gem +0 -0
  52. data/vendor/cache/mocha-0.9.8.gem +0 -0
  53. data/vendor/cache/open4-1.0.1.gem +0 -0
  54. data/vendor/cache/passenger-2.2.9.gem +0 -0
  55. data/vendor/cache/rack-1.0.1.gem +0 -0
  56. data/vendor/cache/rails-2.3.5.gem +0 -0
  57. data/vendor/cache/rake-0.8.7.gem +0 -0
  58. data/vendor/cache/rubygems-update-1.3.5.gem +0 -0
  59. data/vendor/cache/shoulda-2.10.3.gem +0 -0
  60. data/vendor/cache/sqlite3-ruby-1.2.5.gem +0 -0
  61. data/vendor/cache/stemmer-1.0.1.gem +0 -0
  62. data/vendor/cache/syntax-1.0.0.gem +0 -0
  63. data/vendor/cache/sys-uname-0.8.4.gem +0 -0
  64. data/vendor/cache/timecop-0.3.4.gem +0 -0
  65. metadata +60 -11
  66. data/lib/vanity/rails.rb +0 -22
  67. data/lib/vanity/rails/dashboard.rb +0 -24
  68. data/lib/vanity/rails/helpers.rb +0 -101
  69. data/lib/vanity/rails/testing.rb +0 -11
@@ -7,35 +7,45 @@ module Vanity
7
7
  # puts Vanity.playground.map(&:name)
8
8
  class Playground
9
9
 
10
- DEFAULTS = { :host=>"127.0.0.1", :port=>6379, :db=>0, :load_path=>"experiments" }
10
+ DEFAULTS = {
11
+ :connection=>"localhost:6379",
12
+ :load_path=>"experiments",
13
+ :namespace=>"vanity:#{Vanity::Version::MAJOR}"
14
+ }
15
+
16
+ # Created new Playground. Unless you need to, use the global
17
+ # Vanity.playground.
18
+ #
19
+ # First argument is connection specification (see #redis=), last argument is
20
+ # a set of options, both are optional. Supported options are:
21
+ # - connection -- Connection specification
22
+ # - namespace -- Namespace to use
23
+ # - load_path -- Path to load experiments/metrics from
24
+ # - logger -- Logger to use
25
+ def initialize(*args)
26
+ options = args.pop if Hash === args.last
27
+ @options = DEFAULTS.merge(options || {})
28
+ if @options.values_at(:host, :port, :db).any?
29
+ warn "Deprecated: please specify Redis connection as single argument (\"host:port\")"
30
+ @connection_spec = "#{@options[:host]}:#{@options[:port]}:#{@options[:db]}"
31
+ elsif @options[:redis]
32
+ @redis = @options[:redis]
33
+ else
34
+ @connection_spec = args.shift || @options[:connection]
35
+ end
11
36
 
12
- # Created new Playground. Unless you need to, use the global Vanity.playground.
13
- def initialize(options = {})
14
- @host, @port, @db, @load_path = DEFAULTS.merge(options).values_at(:host, :port, :db, :load_path)
15
- @namespace = "vanity:#{Vanity::Version::MAJOR}"
16
- @logger = options[:logger]
37
+ @namespace = @options[:namespace] || DEFAULTS[:namespace]
38
+ @load_path = @options[:load_path] || DEFAULTS[:load_path]
39
+ @logger = @options[:logger]
17
40
  unless @logger
18
41
  @logger = Logger.new(STDOUT)
19
42
  @logger.level = Logger::ERROR
20
43
  end
21
- @redis = options[:redis]
22
44
  @loading = []
23
45
  end
24
-
25
- # Redis host name. Default is 127.0.0.1
26
- attr_accessor :host
27
-
28
- # Redis port number. Default is 6379.
29
- attr_accessor :port
30
-
31
- # Redis database number. Default is 0.
32
- attr_accessor :db
33
-
34
- # Redis database password.
35
- attr_accessor :password
36
-
37
- # Namespace for database keys. Default is vanity:n, where n is the major release number, e.g. vanity:1 for 1.0.3.
38
- attr_accessor :namespace
46
+
47
+ # Deprecated. Use redis.server instead.
48
+ attr_accessor :host, :port, :db, :password, :namespace
39
49
 
40
50
  # Path to load experiment files from.
41
51
  attr_accessor :load_path
@@ -48,6 +58,7 @@ module Vanity
48
58
  #
49
59
  # @see Vanity::Experiment
50
60
  def define(name, type, options = {}, &block)
61
+ warn "Deprecated: if you need this functionality let's make a better API"
51
62
  id = name.to_s.downcase.gsub(/\W/, "_").to_sym
52
63
  raise "Experiment #{id} already defined once" if experiments[id]
53
64
  klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
@@ -64,7 +75,7 @@ module Vanity
64
75
  def experiment(name)
65
76
  id = name.to_s.downcase.gsub(/\W/, "_").to_sym
66
77
  warn "Deprecated: pleae call experiment method with experiment identifier (a Ruby symbol)" unless id == name
67
- experiments[id] ||= Experiment::Base.load(self, @loading, File.expand_path(load_path), id)
78
+ experiments[id.to_sym] or raise NameError, "No experiment #{id}"
68
79
  end
69
80
 
70
81
  # Returns hash of experiments (key is experiment id).
@@ -75,8 +86,7 @@ module Vanity
75
86
  @experiments = {}
76
87
  @logger.info "Vanity: loading experiments from #{load_path}"
77
88
  Dir[File.join(load_path, "*.rb")].each do |file|
78
- id = File.basename(file).gsub(/.rb$/, "")
79
- experiment id.to_sym
89
+ Experiment::Base.load self, @loading, file
80
90
  end
81
91
  end
82
92
  @experiments
@@ -97,25 +107,6 @@ module Vanity
97
107
  metrics
98
108
  end
99
109
 
100
- # Use this instance to access the Redis database.
101
- def redis
102
- @redis ||= Redis.new(:host=>self.host, :port=>self.port, :db=>self.db,
103
- :password=>self.password, :logger=>self.logger)
104
- class << self ; self ; end.send(:define_method, :redis) { @redis }
105
- @redis
106
- end
107
-
108
- # Switches playground to use MockRedis instead of a live server.
109
- # Particularly useful for testing, e.g. if you can't access Redis on your CI
110
- # server. This method has no affect after playground accesses live Redis
111
- # server.
112
- #
113
- # @example Put this in config/environments/test.rb
114
- # config.after_initialize { Vanity.playground.mock! }
115
- def mock!
116
- @redis ||= MockRedis.new
117
- end
118
-
119
110
  # Returns a metric (raises NameError if no metric with that identifier).
120
111
  #
121
112
  # @see Vanity::Metric
@@ -148,6 +139,64 @@ module Vanity
148
139
  def track!(id, count = 1)
149
140
  metric(id).track! count
150
141
  end
142
+
143
+
144
+ # -- Connection management --
145
+
146
+ # Tells the playground where to find Redis. Accepts one of the following:
147
+ # - "hostname:port"
148
+ # - ":port"
149
+ # - "hostname:port:db"
150
+ # - Instance of Redis connection.
151
+ def redis=(spec_or_connection)
152
+ case spec_or_connection
153
+ when String
154
+ @connection_spec = spec_or_connection
155
+ host, port, db = spec_or_connection.split(':').map { |x| x unless x.empty? }
156
+ @redis = Redis.new(:host=>host, :port=>port, :thread_safe=>true, :db=>db, :thread_safe=>true)
157
+ when Redis
158
+ @connection_spec = nil
159
+ @redis = spec_or_connection
160
+ when :mock
161
+ @connection_spec = nil
162
+ @redis = MockRedis.new
163
+ else
164
+ raise "I don't know what to do with #{spec_or_connection.inspect}"
165
+ end
166
+ end
167
+
168
+ def redis
169
+ self.redis = @connection_spec unless @redis
170
+ @redis
171
+ end
172
+
173
+ def connected?
174
+ !@redis.nil?
175
+ end
176
+
177
+ def disconnect!
178
+ @redis.quit if connected? rescue nil
179
+ @redis = nil
180
+ end
181
+
182
+ def reconnect!
183
+ raise "Connect reconnect without connection specification" unless String === @connection_spec
184
+ disconnect! rescue nil
185
+ end
186
+
187
+ def mock!
188
+ warn "Deprecated: use Vanity.playground.test!"
189
+ test!
190
+ end
191
+
192
+ # Use this when testing to disable Redis (e.g. if your CI server doesn't
193
+ # have Redis).
194
+ #
195
+ # @example Put this in config/environments/test.rb
196
+ # config.after_initialize { Vanity.playground.test! }
197
+ def test!
198
+ self.redis = :mock
199
+ end
151
200
  end
152
201
 
153
202
  @playground = Playground.new
@@ -1,9 +1,14 @@
1
1
  <h3><%=h metric.name %></h3>
2
2
  <%= simple_format h(Vanity::Metric.description(metric).to_s), :class=>"description" %>
3
- <% data = Vanity::Metric.data(metric)
4
- min, max = data.map(&:last).minmax
5
- js = data.map { |date,value| "['#{date.to_time.httpdate}',#{value}]" }.join(",") %>
6
- <div class="chart"></div>
7
- <script type="text/javascript">
8
- $(function () { Vanity.metric("<%= id %>").plot([{ label: "<%=h metric.name %>", data: [<%= js %>] }]); });
9
- </script>
3
+ <%=
4
+ begin
5
+ data = Vanity::Metric.data(metric)
6
+ min, max = data.map(&:last).minmax
7
+ js = data.map { |date,value| "['#{date.to_time.httpdate}',#{value}]" }.join(",")
8
+ %{<div class="chart"></div>
9
+ <script type="text/javascript">
10
+ $(function(){Vanity.metric("#{h id.to_s}").plot([{label:"#{h metric.name}", data: [#{js}]}])})
11
+ </script>}
12
+ rescue Exception=>ex
13
+ %{<div class="error">#{h ex.message}</div>}
14
+ end %>
@@ -2,6 +2,7 @@
2
2
  .vanity .meta { color: #666 }
3
3
  .vanity .footer { margin-top: 2em; text-align: right; font-size: 90% }
4
4
  .vanity h2, .vanity h3 { margin: 0 0 .3em 0 }
5
+ .vanity .error { color: #ff2020; border: 1px solid #ff2020; background: #fff; padding: .3em }
5
6
 
6
7
  .vanity .experiments { list-style: none; margin: 0; padding: 0 }
7
8
  .vanity .experiment { margin: 1em 0 2em 0; border-bottom: 1px solid #ddd }
@@ -35,23 +35,23 @@ class AbTestTest < ActionController::TestCase
35
35
 
36
36
  def test_requires_at_least_two_alternatives_per_experiment
37
37
  assert_raises RuntimeError do
38
- ab_test :none do
38
+ new_ab_test :none do
39
39
  alternatives []
40
40
  end
41
41
  end
42
42
  assert_raises RuntimeError do
43
- ab_test :one do
43
+ new_ab_test :one do
44
44
  alternatives "foo"
45
45
  end
46
46
  end
47
- ab_test :two do
47
+ new_ab_test :two do
48
48
  alternatives "foo", "bar"
49
49
  metrics :coolness
50
50
  end
51
51
  end
52
52
 
53
53
  def test_returning_alternative_by_value
54
- ab_test :abcd do
54
+ new_ab_test :abcd do
55
55
  alternatives :a, :b, :c, :d
56
56
  metrics :coolness
57
57
  end
@@ -60,7 +60,7 @@ class AbTestTest < ActionController::TestCase
60
60
  end
61
61
 
62
62
  def test_alternative_name
63
- ab_test :abcd do
63
+ new_ab_test :abcd do
64
64
  alternatives :a, :b
65
65
  metrics :coolness
66
66
  end
@@ -68,25 +68,50 @@ class AbTestTest < ActionController::TestCase
68
68
  assert_equal "option B", experiment(:abcd).alternative(:b).name
69
69
  end
70
70
 
71
+ def test_alternative_fingerprint_is_unique
72
+ new_ab_test :ab do
73
+ metrics :coolness
74
+ alternatives :a, :b
75
+ end
76
+ new_ab_test :cd do
77
+ metrics :coolness
78
+ alternatives :a, :b
79
+ end
80
+ fingerprints = Vanity.playground.experiments.map { |id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
81
+ assert_equal 4, fingerprints.uniq.size
82
+ end
83
+
84
+ def test_alternative_fingerprint_is_consistent
85
+ new_ab_test :ab do
86
+ alternatives :a, :b
87
+ metrics :coolness
88
+ end
89
+ fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
90
+ fingerprints.each do |fingerprint|
91
+ assert_match /^[0-9a-f]{10}$/i, fingerprint
92
+ end
93
+ assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
94
+ end
95
+
71
96
 
72
97
  # -- Experiment metric --
73
98
 
74
99
  def test_explicit_metric
75
- ab_test :abcd do
100
+ new_ab_test :abcd do
76
101
  metrics :coolness
77
102
  end
78
103
  assert_equal [Vanity.playground.metric(:coolness)], experiment(:abcd).metrics
79
104
  end
80
105
 
81
106
  def test_implicit_metric
82
- ab_test :abcd do
107
+ new_ab_test :abcd do
83
108
  end
84
109
  assert_equal [Vanity.playground.metric(:abcd)], experiment(:abcd).metrics
85
110
  end
86
111
 
87
112
  def test_metric_tracking_into_alternative
88
113
  metric "Coolness"
89
- ab_test :abcd do
114
+ new_ab_test :abcd do
90
115
  metrics :coolness
91
116
  end
92
117
  Vanity.playground.track! :coolness
@@ -97,7 +122,7 @@ class AbTestTest < ActionController::TestCase
97
122
  # -- Running experiment --
98
123
 
99
124
  def test_returns_the_same_alternative_consistently
100
- ab_test :foobar do
125
+ new_ab_test :foobar do
101
126
  alternatives "foo", "bar"
102
127
  identify { "6e98ec" }
103
128
  metrics :coolness
@@ -110,7 +135,7 @@ class AbTestTest < ActionController::TestCase
110
135
  end
111
136
 
112
137
  def test_returns_different_alternatives_for_each_participant
113
- ab_test :foobar do
138
+ new_ab_test :foobar do
114
139
  alternatives "foo", "bar"
115
140
  identify { rand }
116
141
  metrics :coolness
@@ -122,7 +147,7 @@ class AbTestTest < ActionController::TestCase
122
147
 
123
148
  def test_records_all_participants_in_each_alternative
124
149
  ids = (Array.new(200) { |i| i } * 5).shuffle
125
- ab_test :foobar do
150
+ new_ab_test :foobar do
126
151
  alternatives "foo", "bar"
127
152
  identify { ids.pop }
128
153
  metrics :coolness
@@ -135,7 +160,7 @@ class AbTestTest < ActionController::TestCase
135
160
 
136
161
  def test_records_each_converted_participant_only_once
137
162
  ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
138
- ab_test :foobar do
163
+ new_ab_test :foobar do
139
164
  alternatives "foo", "bar"
140
165
  identify { ids.pop }
141
166
  metrics :coolness
@@ -150,7 +175,7 @@ class AbTestTest < ActionController::TestCase
150
175
 
151
176
  def test_records_conversion_only_for_participants
152
177
  ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
153
- ab_test :foobar do
178
+ new_ab_test :foobar do
154
179
  alternatives "foo", "bar"
155
180
  identify { ids.pop }
156
181
  metrics :coolness
@@ -166,7 +191,7 @@ class AbTestTest < ActionController::TestCase
166
191
 
167
192
 
168
193
  def test_destroy_experiment
169
- ab_test :simple do
194
+ new_ab_test :simple do
170
195
  identify { "me" }
171
196
  metrics :coolness
172
197
  complete_if { alternatives.map(&:converted).sum >= 1 }
@@ -196,7 +221,7 @@ class AbTestTest < ActionController::TestCase
196
221
  end
197
222
 
198
223
  def test_ab_test_chooses_in_render
199
- ab_test :simple do
224
+ new_ab_test :simple do
200
225
  metrics :coolness
201
226
  end
202
227
  responses = Array.new(100) do
@@ -208,7 +233,7 @@ class AbTestTest < ActionController::TestCase
208
233
  end
209
234
 
210
235
  def test_ab_test_chooses_view_helper
211
- ab_test :simple do
236
+ new_ab_test :simple do
212
237
  metrics :coolness
213
238
  end
214
239
  responses = Array.new(100) do
@@ -220,7 +245,7 @@ class AbTestTest < ActionController::TestCase
220
245
  end
221
246
 
222
247
  def test_ab_test_with_capture
223
- ab_test :simple do
248
+ new_ab_test :simple do
224
249
  metrics :coolness
225
250
  end
226
251
  responses = Array.new(100) do
@@ -232,7 +257,7 @@ class AbTestTest < ActionController::TestCase
232
257
  end
233
258
 
234
259
  def test_ab_test_track
235
- ab_test :simple do
260
+ new_ab_test :simple do
236
261
  metrics :coolness
237
262
  end
238
263
  responses = Array.new(100) do
@@ -246,7 +271,7 @@ class AbTestTest < ActionController::TestCase
246
271
  # -- Testing with tests --
247
272
 
248
273
  def test_with_given_choice
249
- ab_test :simple do
274
+ new_ab_test :simple do
250
275
  alternatives :a, :b, :c
251
276
  metrics :coolness
252
277
  end
@@ -259,7 +284,7 @@ class AbTestTest < ActionController::TestCase
259
284
  end
260
285
 
261
286
  def test_which_chooses_non_existent_alternative
262
- ab_test :simple do
287
+ new_ab_test :simple do
263
288
  metrics :coolness
264
289
  end
265
290
  assert_raises ArgumentError do
@@ -268,7 +293,7 @@ class AbTestTest < ActionController::TestCase
268
293
  end
269
294
 
270
295
  def test_chooses_cleared_with_nil
271
- ab_test :simple do
296
+ new_ab_test :simple do
272
297
  identify { rand }
273
298
  alternatives :a, :b, :c
274
299
  metrics :coolness
@@ -287,7 +312,7 @@ class AbTestTest < ActionController::TestCase
287
312
  # -- Scoring --
288
313
 
289
314
  def test_scoring
290
- ab_test :abcd do
315
+ new_ab_test :abcd do
291
316
  alternatives :a, :b, :c, :d
292
317
  metrics :coolness
293
318
  end
@@ -313,7 +338,7 @@ class AbTestTest < ActionController::TestCase
313
338
  end
314
339
 
315
340
  def test_scoring_with_no_performers
316
- ab_test :abcd do
341
+ new_ab_test :abcd do
317
342
  alternatives :a, :b, :c, :d
318
343
  metrics :coolness
319
344
  end
@@ -326,7 +351,7 @@ class AbTestTest < ActionController::TestCase
326
351
  end
327
352
 
328
353
  def test_scoring_with_one_performer
329
- ab_test :abcd do
354
+ new_ab_test :abcd do
330
355
  alternatives :a, :b, :c, :d
331
356
  metrics :coolness
332
357
  end
@@ -341,7 +366,7 @@ class AbTestTest < ActionController::TestCase
341
366
  end
342
367
 
343
368
  def test_scoring_with_some_performers
344
- ab_test :abcd do
369
+ new_ab_test :abcd do
345
370
  alternatives :a, :b, :c, :d
346
371
  metrics :coolness
347
372
  end
@@ -360,7 +385,7 @@ class AbTestTest < ActionController::TestCase
360
385
  end
361
386
 
362
387
  def test_scoring_with_different_probability
363
- ab_test :abcd do
388
+ new_ab_test :abcd do
364
389
  alternatives :a, :b, :c, :d
365
390
  metrics :coolness
366
391
  end
@@ -375,7 +400,7 @@ class AbTestTest < ActionController::TestCase
375
400
  # -- Conclusion --
376
401
 
377
402
  def test_conclusion
378
- ab_test :abcd do
403
+ new_ab_test :abcd do
379
404
  alternatives :a, :b, :c, :d
380
405
  metrics :coolness
381
406
  end
@@ -398,7 +423,7 @@ Option D selected as the best alternative.
398
423
  end
399
424
 
400
425
  def test_conclusion_with_some_performers
401
- ab_test :abcd do
426
+ new_ab_test :abcd do
402
427
  alternatives :a, :b, :c, :d
403
428
  metrics :coolness
404
429
  end
@@ -416,7 +441,7 @@ Option D selected as the best alternative.
416
441
  end
417
442
 
418
443
  def test_conclusion_without_clear_winner
419
- ab_test :abcd do
444
+ new_ab_test :abcd do
420
445
  alternatives :a, :b, :c, :d
421
446
  metrics :coolness
422
447
  end
@@ -433,7 +458,7 @@ Option C did not convert.
433
458
  end
434
459
 
435
460
  def test_conclusion_without_close_performers
436
- ab_test :abcd do
461
+ new_ab_test :abcd do
437
462
  alternatives :a, :b, :c, :d
438
463
  metrics :coolness
439
464
  end
@@ -450,7 +475,7 @@ Option C did not convert.
450
475
  end
451
476
 
452
477
  def test_conclusion_without_equal_performers
453
- ab_test :abcd do
478
+ new_ab_test :abcd do
454
479
  alternatives :a, :b, :c, :d
455
480
  metrics :coolness
456
481
  end
@@ -466,7 +491,7 @@ Option C did not convert.
466
491
  end
467
492
 
468
493
  def test_conclusion_with_one_performers
469
- ab_test :abcd do
494
+ new_ab_test :abcd do
470
495
  alternatives :a, :b, :c, :d
471
496
  metrics :coolness
472
497
  end
@@ -479,7 +504,7 @@ This experiment did not run long enough to find a clear winner.
479
504
  end
480
505
 
481
506
  def test_conclusion_with_no_performers
482
- ab_test :abcd do
507
+ new_ab_test :abcd do
483
508
  alternatives :a, :b, :c, :d
484
509
  metrics :coolness
485
510
  end
@@ -493,7 +518,7 @@ This experiment did not run long enough to find a clear winner.
493
518
  # -- Completion --
494
519
 
495
520
  def test_completion_if
496
- ab_test :simple do
521
+ new_ab_test :simple do
497
522
  identify { rand }
498
523
  complete_if { true }
499
524
  metrics :coolness
@@ -503,7 +528,7 @@ This experiment did not run long enough to find a clear winner.
503
528
  end
504
529
 
505
530
  def test_completion_if_fails
506
- ab_test :simple do
531
+ new_ab_test :simple do
507
532
  identify { rand }
508
533
  complete_if { fail }
509
534
  metrics :coolness
@@ -514,7 +539,7 @@ This experiment did not run long enough to find a clear winner.
514
539
 
515
540
  def test_completion
516
541
  ids = Array.new(100) { |i| i.to_s }.shuffle
517
- ab_test :simple do
542
+ new_ab_test :simple do
518
543
  identify { ids.pop }
519
544
  complete_if { alternatives.map(&:participants).sum >= 100 }
520
545
  metrics :coolness
@@ -530,7 +555,7 @@ This experiment did not run long enough to find a clear winner.
530
555
 
531
556
  def test_ab_methods_after_completion
532
557
  ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
533
- ab_test :simple do
558
+ new_ab_test :simple do
534
559
  identify { ids.pop }
535
560
  complete_if { alternatives.map(&:participants).sum >= 100 }
536
561
  outcome_is { alternatives[1] }
@@ -559,7 +584,7 @@ This experiment did not run long enough to find a clear winner.
559
584
  # -- Outcome --
560
585
 
561
586
  def test_completion_outcome
562
- ab_test :quick do
587
+ new_ab_test :quick do
563
588
  outcome_is { alternatives[1] }
564
589
  metrics :coolness
565
590
  end
@@ -568,7 +593,7 @@ This experiment did not run long enough to find a clear winner.
568
593
  end
569
594
 
570
595
  def test_outcome_is_returns_nil
571
- ab_test :quick do
596
+ new_ab_test :quick do
572
597
  outcome_is { nil }
573
598
  metrics :coolness
574
599
  end
@@ -577,7 +602,7 @@ This experiment did not run long enough to find a clear winner.
577
602
  end
578
603
 
579
604
  def test_outcome_is_returns_something_else
580
- ab_test :quick do
605
+ new_ab_test :quick do
581
606
  outcome_is { "error" }
582
607
  metrics :coolness
583
608
  end
@@ -586,7 +611,7 @@ This experiment did not run long enough to find a clear winner.
586
611
  end
587
612
 
588
613
  def test_outcome_is_fails
589
- ab_test :quick do
614
+ new_ab_test :quick do
590
615
  outcome_is { fail }
591
616
  metrics :coolness
592
617
  end
@@ -595,7 +620,7 @@ This experiment did not run long enough to find a clear winner.
595
620
  end
596
621
 
597
622
  def test_outcome_choosing_best_alternative
598
- ab_test :quick do
623
+ new_ab_test :quick do
599
624
  metrics :coolness
600
625
  end
601
626
  fake :quick, false=>[2,0], true=>10
@@ -604,7 +629,7 @@ This experiment did not run long enough to find a clear winner.
604
629
  end
605
630
 
606
631
  def test_outcome_only_performing_alternative
607
- ab_test :quick do
632
+ new_ab_test :quick do
608
633
  metrics :coolness
609
634
  end
610
635
  fake :quick, true=>2
@@ -613,7 +638,7 @@ This experiment did not run long enough to find a clear winner.
613
638
  end
614
639
 
615
640
  def test_outcome_choosing_equal_alternatives
616
- ab_test :quick do
641
+ new_ab_test :quick do
617
642
  metrics :coolness
618
643
  end
619
644
  fake :quick, false=>8, true=>8
@@ -624,10 +649,6 @@ This experiment did not run long enough to find a clear winner.
624
649
 
625
650
  # -- Helper methods --
626
651
 
627
- def ab_test(name, &block)
628
- Vanity.playground.define name, :ab_test, &block
629
- end
630
-
631
652
  def fake(name, args)
632
653
  experiment(name).instance_eval { fake args }
633
654
  end