lookout-vanity 1.8.2

data/test/dummy/config/environments/production.rb

@@ -0,0 +1,49 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb
+
+  # The production environment is meant for finished, "live" apps.
+  # Code is not reloaded between requests
+  config.cache_classes = true
+
+  # Full error reports are disabled and caching is turned on
+  config.consider_all_requests_local       = false
+  config.action_controller.perform_caching = true
+
+  # Specifies the header that your server uses for sending files
+  config.action_dispatch.x_sendfile_header = "X-Sendfile"
+
+  # For nginx:
+  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect'
+
+  # If you have no front-end server that supports something like X-Sendfile,
+  # just comment this out and Rails will serve the files
+
+  # See everything in the log (default is :info)
+  # config.log_level = :debug
+
+  # Use a different logger for distributed setups
+  # config.logger = SyslogLogger.new
+
+  # Use a different cache store in production
+  # config.cache_store = :mem_cache_store
+
+  # Disable Rails's static asset server
+  # In production, Apache or nginx will already do this
+  config.serve_static_assets = false
+
+  # Enable serving of images, stylesheets, and javascripts from an asset server
+  # config.action_controller.asset_host = "http://assets.example.com"
+
+  # Disable delivery errors, bad email addresses will be ignored
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Enable threaded mode
+  # config.threadsafe!
+
+  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+  # the I18n.default_locale when a translation can not be found)
+  config.i18n.fallbacks = true
+
+  # Send deprecation notices to registered listeners
+  config.active_support.deprecation = :notify
+end

data/test/dummy/config/environments/test.rb

@@ -0,0 +1,35 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb
+
+  # The test environment is used exclusively to run your application's
+  # test suite.  You never need to work with it otherwise.  Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs.  Don't rely on the data there!
+  config.cache_classes = true
+
+  # Log error messages when you accidentally call methods on nil.
+  config.whiny_nils = true
+
+  # Show full error reports and disable caching
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment
+  config.action_controller.allow_forgery_protection    = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  config.action_mailer.delivery_method = :test
+
+  # Use SQL instead of Active Record's schema dumper when creating the test database.
+  # This is necessary if your schema can't be completely dumped by the schema dumper,
+  # like if you have constraints or database-specific column types
+  # config.active_record.schema_format = :sql
+
+  # Print deprecation notices to the stderr
+  config.active_support.deprecation = :stderr
+end

data/test/dummy/config/initializers/backtrace_silencers.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
+
+# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# Rails.backtrace_cleaner.remove_silencers!

data/test/dummy/config/initializers/inflections.rb

@@ -0,0 +1,10 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new inflection rules using the following format
+# (all these examples are active by default):
+# ActiveSupport::Inflector.inflections do |inflect|
+#   inflect.plural /^(ox)$/i, '\1en'
+#   inflect.singular /^(ox)en/i, '\1'
+#   inflect.irregular 'person', 'people'
+#   inflect.uncountable %w( fish sheep )
+# end

data/test/dummy/config/initializers/mime_types.rb

@@ -0,0 +1,5 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new mime types for use in respond_to blocks:
+# Mime::Type.register "text/richtext", :rtf
+# Mime::Type.register_alias "text/html", :iphone

data/test/dummy/config/initializers/secret_token.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+Dummy::Application.config.secret_token = '33ccbc9a29f3b02e87c08904505b1c9a3a1e97dd01f02e598e65ee9e7b96fff2ca4a6d0dd7c4a8d3682d8c64f84d372661e141264e70697dc576c722c72d80d0'

data/test/dummy/config/initializers/session_store.rb

@@ -0,0 +1,8 @@
+# Be sure to restart your server when you modify this file.
+
+Dummy::Application.config.session_store :cookie_store, :key => '_dummy_session'
+
+# Use the database for sessions instead of the cookie-based default,
+# which shouldn't be used to store highly confidential information
+# (create the session table with "rails generate session_migration")
+# Dummy::Application.config.session_store :active_record_store

data/test/dummy/config/locales/en.yml

@@ -0,0 +1,5 @@
+# Sample localization file for English. Add more files in this directory for other locales.
+# See http://github.com/svenfuchs/rails-i18n/tree/master/rails%2Flocale for starting points.
+
+en:
+  hello: "Hello world"

data/test/dummy/config/routes.rb

@@ -0,0 +1,58 @@
+Dummy::Application.routes.draw do
+  # The priority is based upon order of creation:
+  # first created -> highest priority.
+
+  # Sample of regular route:
+  #   match 'products/:id' => 'catalog#view'
+  # Keep in mind you can assign values other than :controller and :action
+
+  # Sample of named route:
+  #   match 'products/:id/purchase' => 'catalog#purchase', :as => :purchase
+  # This route can be invoked with purchase_url(:id => product.id)
+
+  # Sample resource route (maps HTTP verbs to controller actions automatically):
+  #   resources :products
+
+  # Sample resource route with options:
+  #   resources :products do
+  #     member do
+  #       get 'short'
+  #       post 'toggle'
+  #     end
+  #
+  #     collection do
+  #       get 'sold'
+  #     end
+  #   end
+
+  # Sample resource route with sub-resources:
+  #   resources :products do
+  #     resources :comments, :sales
+  #     resource :seller
+  #   end
+
+  # Sample resource route with more complex sub-resources
+  #   resources :products do
+  #     resources :comments
+  #     resources :sales do
+  #       get 'recent', :on => :collection
+  #     end
+  #   end
+
+  # Sample resource route within a namespace:
+  #   namespace :admin do
+  #     # Directs /admin/products/* to Admin::ProductsController
+  #     # (app/controllers/admin/products_controller.rb)
+  #     resources :products
+  #   end
+
+  # You can have the root of your site routed with "root"
+  # just remember to delete public/index.html.
+  # root :to => "welcome#index"
+
+  # See how all your routes lay out with "rake routes"
+
+  # This is a legacy wild controller route that's not recommended for RESTful applications.
+  # Note: This route will make all actions in every controller accessible via GET requests.
+  match ':controller(/:action(/:id(.:format)))'
+end

data/test/dummy/script/rails

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
+
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require File.expand_path('../../config/boot',  __FILE__)
+require 'rails/commands'

data/test/experiment/ab_test.rb

@@ -0,0 +1,859 @@
+require "test/test_helper"
+
+class AbTestController < ActionController::Base
+  use_vanity :current_user
+  attr_accessor :current_user
+
+  def test_render
+    render :text=>ab_test(:simple)
+  end
+
+  def test_view
+    render :inline=>"<%= ab_test(:simple) %>"
+  end
+
+  def test_capture
+    if Rails.version.to_i == 3
+      render :inline=>"<%= ab_test :simple do |value| %><%= value %><% end %>"
+    else
+      render :inline=>"<% ab_test :simple do |value| %><%= value %><% end %>"
+    end
+  end
+
+  def track
+    track! :coolness
+    render :text=>""
+  end
+end
+
+
+class AbTestTest < ActionController::TestCase
+  tests AbTestController
+
+  def setup
+    super
+    metric "Coolness"
+  end
+
+  # --  Experiment definition --
+
+  def test_requires_at_least_two_alternatives_per_experiment
+    assert_raises RuntimeError do
+      new_ab_test :none do
+        alternatives []
+      end
+    end
+    assert_raises RuntimeError do
+      new_ab_test :one do
+        alternatives "foo"
+      end
+    end
+    new_ab_test :two do
+      alternatives "foo", "bar"
+      metrics :coolness
+    end
+  end
+  
+  def test_returning_alternative_by_value
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
+    assert_equal experiment(:abcd).alternatives[3], experiment(:abcd).alternative(:d)
+  end
+
+  def test_alternative_name
+    new_ab_test :abcd do
+      alternatives :a, :b
+      metrics :coolness
+    end
+    assert_equal "option A", experiment(:abcd).alternative(:a).name
+    assert_equal "option B", experiment(:abcd).alternative(:b).name
+  end
+
+  def test_alternative_fingerprint_is_unique
+    new_ab_test :ab do
+      metrics :coolness
+      alternatives :a, :b
+    end
+    new_ab_test :cd do
+      metrics :coolness
+      alternatives :a, :b
+    end
+    fingerprints = Vanity.playground.experiments.map { |id, exp| exp.alternatives.map { |alt| exp.fingerprint(alt) } }.flatten
+    assert_equal 4, fingerprints.uniq.size
+  end
+
+  def test_alternative_fingerprint_is_consistent
+    new_ab_test :ab do
+      alternatives :a, :b
+      metrics :coolness
+    end
+    fingerprints = experiment(:ab).alternatives.map { |alt| experiment(:ab).fingerprint(alt) }
+    fingerprints.each do |fingerprint|
+      assert_match /^[0-9a-f]{10}$/i, fingerprint
+    end
+    assert_equal fingerprints.first, experiment(:ab).fingerprint(experiment(:ab).alternatives.first)
+  end
+
+
+  # -- Experiment metric --
+
+  def test_explicit_metric
+    new_ab_test :abcd do
+      metrics :coolness
+    end
+    assert_equal [Vanity.playground.metric(:coolness)], experiment(:abcd).metrics
+  end
+
+  def test_implicit_metric
+    new_ab_test :abcd do
+    end
+    assert_equal [Vanity.playground.metric(:abcd)], experiment(:abcd).metrics
+  end
+
+  def test_metric_tracking_into_alternative
+    metric "Coolness"
+    new_ab_test :abcd do
+      metrics :coolness
+    end
+    Vanity.playground.track! :coolness
+    assert_equal 1, experiment(:abcd).alternatives.sum(&:conversions)
+  end
+
+  # -- use_js! --
+
+  def test_does_not_record_participant_when_using_js
+    Vanity.playground.use_js!
+    ids = (0...10).to_a
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { ids.pop }
+      metrics :coolness
+    end
+    10.times { experiment(:foobar).choose }
+    alts = experiment(:foobar).alternatives
+    assert_equal 0, alts.map(&:participants).sum
+  end
+
+
+  # -- Running experiment --
+
+  def test_returns_the_same_alternative_consistently
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      metrics :coolness
+    end
+    assert value = experiment(:foobar).choose.value
+    assert_match /foo|bar/, value
+    1000.times do
+      assert_equal value, experiment(:foobar).choose.value
+    end
+  end
+
+  def test_returns_different_alternatives_for_each_participant
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { rand }
+      metrics :coolness
+    end
+    alts = Array.new(1000) { experiment(:foobar).choose.value }
+    assert_equal %w{bar foo}, alts.uniq.sort
+    assert_in_delta alts.select { |a| a == "foo" }.size, 500, 100 # this may fail, such is propability
+  end
+
+  def test_records_all_participants_in_each_alternative
+    ids = (Array.new(200) { |i| i } * 5).shuffle
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { ids.pop }
+      metrics :coolness
+    end
+    1000.times { experiment(:foobar).choose }
+    alts = experiment(:foobar).alternatives
+    assert_equal 200, alts.map(&:participants).sum
+    assert_in_delta alts.first.participants, 100, 20
+  end
+
+  def test_records_each_converted_participant_only_once
+    ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { ids.pop }
+      metrics :coolness
+    end
+    500.times do
+      experiment(:foobar).choose
+      metric(:coolness).track!
+    end
+    alts = experiment(:foobar).alternatives
+    assert_equal 100, alts.map(&:converted).sum
+  end
+
+  def test_records_conversion_only_for_participants
+    ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { ids.pop }
+      metrics :coolness
+    end
+    500.times do
+      experiment(:foobar).choose
+      metric(:coolness).track!
+      metric(:coolness).track!
+    end
+    alts = experiment(:foobar).alternatives
+    assert_equal 100, alts.map(&:converted).sum
+  end
+
+  TEST_SIZE = 10000
+
+  def self.should_choose_correct_proportions(delta=1, test_pct=100, control=nil)
+    should "choose correct proportions" do
+      assert_equal TEST_SIZE*(100-test_pct)/100, @results.count(control) if control
+      expected_size = @alternatives.size + (test_pct == 100 || @alternatives.include?(control) ? 0 : 1)
+      assert_equal expected_size, @experiment.alternatives.size
+      @alternatives.delete(control)
+      @alternatives.each do |alt|
+        assert_in_delta TEST_SIZE*test_pct/100/@alternatives.size, @results.count(alt), delta
+      end
+    end
+  end
+
+  def run_test(experiment)
+    (0..TEST_SIZE-1).collect do |hash|
+      index = experiment.send(:hash_to_alternative, hash) # access protected method
+      experiment.alternatives[index].value
+    end
+  end
+
+  context "An experiment with three alternatives" do
+    setup do
+      @experiment = Vanity::Experiment::AbTest.new(Vanity.playground, :test, "test")
+      @alternatives = [1, 2, 3]
+      @experiment.alternatives(*@alternatives)
+    end
+
+    context "with an out-of-set control value" do
+      setup do
+        @experiment.control_value(222)
+        @experiment.test_percent(15)
+        @results = run_test(@experiment)
+      end
+
+      should_choose_correct_proportions(TEST_SIZE/500, 15, 222)
+    end
+
+    context "with an in-set control value" do
+      setup do
+        @experiment.control_value(3)
+        @experiment.test_percent(10)
+        @results = run_test(@experiment)
+      end
+
+      should_choose_correct_proportions(TEST_SIZE/500, 10, 3)
+    end
+
+    context "with no control value" do
+      setup do
+        @results = run_test(@experiment)
+      end
+
+      should_choose_correct_proportions
+    end
+  end
+
+  context "An experiment setting alternatives after control value" do
+    setup do
+      @experiment = Vanity::Experiment::AbTest.new(Vanity.playground, :test, "test")
+      @experiment.test_percent(25)
+      @alternatives = [1, 2, 3]
+    end
+
+    context "with an out-of-set control value" do
+      setup do
+        @experiment.control_value(22)
+        @experiment.alternatives(*@alternatives)
+        @results = run_test(@experiment)
+      end
+
+      should_choose_correct_proportions(TEST_SIZE/500, 25, 22)
+    end
+
+    context "with an in-set control value" do
+      setup do
+        @experiment.control_value(3)
+        @experiment.alternatives(*@alternatives)
+        @results = run_test(@experiment)
+      end
+
+      should_choose_correct_proportions(TEST_SIZE/500, 25, 3)
+    end
+  end
+
+  def test_destroy_experiment
+    new_ab_test :simple do
+      identify { "me" }
+      metrics :coolness
+      complete_if { alternatives.map(&:converted).sum >= 1 }
+      outcome_is { alternative(true) }
+    end
+    experiment(:simple).choose
+    metric(:coolness).track!
+    assert !experiment(:simple).active?
+    assert_equal true, experiment(:simple).outcome.value
+
+    experiment(:simple).destroy
+    assert experiment(:simple).active?
+    assert_nil experiment(:simple).outcome
+    assert_nil experiment(:simple).completed_at
+    assert_equal 0, experiment(:simple).alternatives.map(&:participants).sum
+    assert_equal 0, experiment(:simple).alternatives.map(&:conversions).sum
+    assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
+  end
+
+
+  # -- A/B helper methods --
+
+  def test_fail_if_no_experiment
+    assert_raise Vanity::NoExperimentError do
+      get :test_render
+    end
+  end
+
+  def test_ab_test_chooses_in_render
+    new_ab_test :simple do
+      metrics :coolness
+    end
+    responses = Array.new(100) do
+      @controller = nil ; setup_controller_request_and_response
+      get :test_render
+      @response.body
+    end
+    assert_equal %w{false true}, responses.uniq.sort
+  end
+
+  def test_ab_test_chooses_view_helper
+    new_ab_test :simple do
+      metrics :coolness
+    end
+    responses = Array.new(100) do
+      @controller = nil ; setup_controller_request_and_response
+      get :test_view
+      @response.body
+    end
+    assert_equal %w{false true}, responses.uniq.sort
+  end
+
+  def test_ab_test_with_capture
+    new_ab_test :simple do
+      metrics :coolness
+    end
+    responses = Array.new(100) do
+      @controller = nil ; setup_controller_request_and_response
+      get :test_capture
+      @response.body
+    end
+    assert_equal %w{false true}, responses.map(&:strip).uniq.sort
+  end
+
+  def test_ab_test_track
+    new_ab_test :simple do
+      metrics :coolness
+    end
+    responses = Array.new(100) do
+      @controller.send(:cookies).each{ |cookie| @controller.send(:cookies).delete(cookie.first) }
+      get :track
+      @response.body
+    end
+  end
+
+
+  # -- Testing with tests --
+  
+  def test_with_given_choice
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    100.times do |i|
+      @controller = nil ; setup_controller_request_and_response
+      experiment(:simple).chooses(:b)
+      get :test_render
+      assert "b", @response.body
+    end
+  end
+
+  def test_which_chooses_non_existent_alternative
+    new_ab_test :simple do
+      metrics :coolness
+    end
+    assert_raises ArgumentError do
+      experiment(:simple).chooses(404)
+    end
+  end
+
+  def test_chooses_cleared_with_nil
+    new_ab_test :simple  do
+      identify { rand }
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    responses = Array.new(100) { |i|
+      @controller = nil ; setup_controller_request_and_response
+      experiment(:simple).chooses(:b)
+      experiment(:simple).chooses(nil)
+      get :test_render
+      @response.body
+    }
+    assert responses.uniq.size == 3
+  end
+
+
+  # -- Scoring --
+  
+  def test_scoring
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    # participating, conversions, rate, z-score
+    # Control:      182	35 19.23%	N/A
+    # Treatment A:  180	45 25.00%	1.33
+    # treatment B:  189	28 14.81%	-1.13
+    # treatment C:  188	61 32.45%	2.94
+    fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
+
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
+    assert_equal %w{-1.33 0.00 -2.46 1.58}, z_scores
+    probabilities = experiment(:abcd).score.alts.map(&:probability)
+    assert_equal [90, 0, 99, 90], probabilities
+
+    diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
+    assert_equal [30, 69, nil, 119], diff
+    assert_equal 3, experiment(:abcd).score.best.id
+    assert_equal 3, experiment(:abcd).score.choice.id
+
+    assert_equal 1, experiment(:abcd).score.base.id
+    assert_equal 2, experiment(:abcd).score.least.id
+  end
+
+  def test_scoring_with_no_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
+    assert_nil experiment(:abcd).score.best
+    assert_nil experiment(:abcd).score.choice
+    assert_nil experiment(:abcd).score.least
+  end
+
+  def test_scoring_with_one_performer
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[10,8]
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
+    assert_equal 1, experiment(:abcd).score.best.id
+    assert_nil experiment(:abcd).score.choice
+    assert_equal 2, experiment(:abcd).score.base.id
+    assert_equal 1, experiment(:abcd).score.least.id
+  end
+
+  def test_scoring_with_some_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[10,8], :d=>[12,5]
+
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }.map(&:downcase)
+    assert_equal %w{nan 2.01 nan 0.00}, z_scores
+    probabilities = experiment(:abcd).score.alts.map(&:probability)
+    assert_equal [0, 95, 0, 0], probabilities
+    diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
+    assert_equal [nil, 92, nil, nil], diff
+    assert_equal 1, experiment(:abcd).score.best.id
+    assert_equal 1, experiment(:abcd).score.choice.id
+    assert_equal 3, experiment(:abcd).score.base.id
+    assert_equal 3, experiment(:abcd).score.least.id
+  end
+
+  def test_scoring_with_different_probability
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[10,8], :d=>[12,5]
+
+    assert_equal 1, experiment(:abcd).score(90).choice.id
+    assert_equal 1, experiment(:abcd).score(95).choice.id
+    assert_nil experiment(:abcd).score(99).choice
+  end
+
+
+  # -- Conclusion --
+
+  def test_conclusion
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    # participating, conversions, rate, z-score
+    # Control:      182	35 19.23%	N/A
+    # Treatment A:  180	45 25.00%	1.33
+    # treatment B:  189	28 14.81%	-1.13
+    # treatment C:  188	61 32.45%	2.94
+    fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 739 participants in this experiment.
+The best choice is option D: it converted at 32.4% (30% better than option B).
+With 90% probability this result is statistically significant.
+Option B converted at 25.0%.
+Option A converted at 19.2%.
+Option C converted at 14.8%.
+Option D selected as the best alternative.
+    TEXT
+  end
+
+  def test_conclusion_with_some_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[180, 45], :d=>[188, 61]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 368 participants in this experiment.
+The best choice is option D: it converted at 32.4% (30% better than option B).
+With 90% probability this result is statistically significant.
+Option B converted at 25.0%.
+Option A did not convert.
+Option C did not convert.
+Option D selected as the best alternative.
+    TEXT
+  end
+
+  def test_conclusion_without_clear_winner
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[180, 58], :d=>[188, 61]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 368 participants in this experiment.
+The best choice is option D: it converted at 32.4% (1% better than option B).
+This result is not statistically significant, suggest you continue this experiment.
+Option B converted at 32.2%.
+Option A did not convert.
+Option C did not convert.
+    TEXT
+  end
+
+  def test_conclusion_without_close_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[186, 60], :d=>[188, 61]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 374 participants in this experiment.
+The best choice is option D: it converted at 32.4% (1% better than option B).
+This result is not statistically significant, suggest you continue this experiment.
+Option B converted at 32.3%.
+Option A did not convert.
+Option C did not convert.
+    TEXT
+  end
+
+  def test_conclusion_without_equal_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[188, 61], :d=>[188, 61]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 376 participants in this experiment.
+Option D converted at 32.4%.
+Option B converted at 32.4%.
+Option A did not convert.
+Option C did not convert.
+    TEXT
+  end
+
+  def test_conclusion_with_one_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    fake :abcd, :b=>[180, 45]
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are 180 participants in this experiment.
+This experiment did not run long enough to find a clear winner.
+    TEXT
+  end
+
+  def test_conclusion_with_no_performers
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+There are no participants in this experiment yet.
+This experiment did not run long enough to find a clear winner.
+    TEXT
+  end
+
+
+  # -- Completion --
+
+  def test_completion_if
+    new_ab_test :simple do
+      identify { rand }
+      complete_if { true }
+      metrics :coolness
+    end
+    experiment(:simple).choose
+    assert !experiment(:simple).active?
+  end
+
+  def test_completion_if_fails
+    new_ab_test :simple do
+      identify { rand }
+      complete_if { fail "Testing complete_if raises exception" }
+      metrics :coolness
+    end
+    experiment(:simple).choose
+    assert experiment(:simple).active?
+  end
+
+  def test_completion
+    ids = Array.new(100) { |i| i.to_s }.shuffle
+    new_ab_test :simple do
+      identify { ids.pop }
+      complete_if { alternatives.map(&:participants).sum >= 100 }
+      metrics :coolness
+    end
+    99.times do |i|
+      experiment(:simple).choose
+      assert experiment(:simple).active?
+    end
+
+    experiment(:simple).choose
+    assert !experiment(:simple).active?
+  end
+
+  def test_ab_methods_after_completion
+    ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
+    new_ab_test :simple do
+      identify { ids.pop }
+      complete_if { alternatives.map(&:participants).sum >= 100 }
+      outcome_is { alternatives[1] }
+      metrics :coolness
+    end
+    # Run experiment to completion (100 participants)
+    results = Set.new
+    100.times do
+      results << experiment(:simple).choose.value
+      metric(:coolness).track!
+    end
+    assert results.include?(true) && results.include?(false)
+    assert !experiment(:simple).active?
+
+    # Test that we always get the same choice (true)
+    100.times do
+      assert_equal true, experiment(:simple).choose.value
+      metric(:coolness).track!
+    end
+    # We don't get to count the 100 participant's conversion, but that's ok.
+    assert_equal 99, experiment(:simple).alternatives.map(&:converted).sum
+    assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
+  end
+
+
+  # -- Outcome --
+  
+  def test_completion_outcome
+    new_ab_test :quick do
+      outcome_is { alternatives[1] }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+  end
+
+  def test_error_in_completion
+    new_ab_test :quick do
+      outcome_is { raise RuntimeError }
+      metrics :coolness
+    end
+    e = experiment(:quick)
+    e.expects(:warn)
+    assert_nothing_raised do
+      e.complete!
+    end
+  end
+
+  def test_outcome_is_returns_nil
+    new_ab_test :quick do
+      outcome_is { nil }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_is_returns_something_else
+    new_ab_test :quick do
+      outcome_is { "error" }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_is_fails
+    new_ab_test :quick do
+      outcome_is { fail "Testing outcome_is raising exception" }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_choosing_best_alternative
+    new_ab_test :quick do
+      metrics :coolness
+    end
+    fake :quick, false=>[2,0], true=>10
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
+  end
+
+  def test_outcome_only_performing_alternative
+    new_ab_test :quick do
+      metrics :coolness
+    end
+    fake :quick, true=>2
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
+  end
+
+  def test_outcome_choosing_equal_alternatives
+    new_ab_test :quick do
+      metrics :coolness
+    end
+    fake :quick, false=>8, true=>8
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
+  end
+
+
+  # -- No collection --
+
+  def test_no_collection_does_not_track
+    not_collecting!
+    metric "Coolness"
+    new_ab_test :abcd do
+      metrics :coolness
+    end
+    Vanity.playground.track! :coolness
+    assert_equal 0, experiment(:abcd).alternatives.sum(&:conversions)
+  end
+
+  def test_no_collection_and_completion
+    not_collecting!
+    new_ab_test :quick do
+      outcome_is { alternatives[1] }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_nil experiment(:quick).outcome
+  end
+
+  def test_chooses_records_participants
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    experiment(:simple).chooses(:b)
+    assert_equal experiment(:simple).alternatives[1].participants, 1
+  end
+
+  def test_chooses_moves_participant_to_new_alternative
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+      identify { "1" }
+    end
+    val = experiment(:simple).choose.value
+    alternative = experiment(:simple).alternatives.detect {|a| a.value != val }
+    experiment(:simple).chooses(alternative.value)
+    assert_equal experiment(:simple).choose.value, alternative.value
+    experiment(:simple).chooses(val)
+    assert_equal experiment(:simple).choose.value, val
+  end
+
+  def test_chooses_records_participants_only_once
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    2.times { experiment(:simple).chooses(:b) }
+    assert_equal experiment(:simple).alternatives[1].participants, 1
+  end
+
+  def test_chooses_records_participants_for_new_alternatives
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    experiment(:simple).chooses(:b)
+    experiment(:simple).chooses(:c)
+    assert_equal experiment(:simple).alternatives[2].participants, 1
+  end
+
+  def test_no_collection_and_chooses
+    not_collecting!
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    assert !experiment(:simple).showing?(experiment(:simple).alternatives[1])
+    experiment(:simple).chooses(:b)
+    assert experiment(:simple).showing?(experiment(:simple).alternatives[1])
+    assert !experiment(:simple).showing?(experiment(:simple).alternatives[2])
+  end
+
+  def test_no_collection_chooses_without_database
+    not_collecting!
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    choice = experiment(:simple).choose.value
+    assert [:a, :b, :c].include?(choice)
+    assert_equal choice, experiment(:simple).choose.value
+  end
+
+
+  # -- Helper methods --
+
+  def fake(name, args)
+    experiment(name).instance_eval { fake args }
+  end
+
+end