vanity 0.2.2 → 0.3.0

data/lib/vanity/experiment/base.rb

@@ -18,8 +18,6 @@ module Vanity
         @playground = playground
         @id, @name = id.to_sym, name
         @namespace = "#{@playground.namespace}:#{@id}"
-        redis.setnx key(:created_at), Time.now.to_i
-        @created_at = Time.at(redis[key(:created_at)].to_i)
         @identify_block = ->(context){ context.vanity_identity }
       end
 
@@ -147,11 +145,13 @@ module Vanity
         @playground.redis
       end
       
-      # Called to save the experiment definition.
-      def save #:nodoc:
+      # Called by Playground to save the experiment definition.
+      def save
+        redis.setnx key(:created_at), Time.now.to_i
+        @created_at = Time.at(redis[key(:created_at)].to_i)
       end
 
-      # Reset experiment.
+      # Reset experiment to its initial state.
       def reset!
         @created_at = Time.now
         redis[key(:created_at)] = @created_at.to_i

data/lib/vanity/playground.rb

@@ -1,5 +1,3 @@
-require "active_support"
-
 module Vanity
 
   # Vanity.playground.configuration
@@ -88,23 +86,33 @@ module Vanity
   end
 
   @playground = Playground.new
-  # Returns the playground instance.
-  def self.playground
-    @playground
-  end
+  class << self
 
-  # Returns the Vanity context.  For example, when using Rails this would be
-  # the current controller, which can be used to get/set the vanity identity.
-  def self.context
-    Thread.current[:vanity_context]
-  end
+    # Returns the playground instance.
+    def playground
+      @playground
+    end
 
-  # Sets the Vanity context.  For example, when using Rails this would be
-  # set by the set_vanity_context before filter (via use_vanity).
-  def self.context=(context)
-    Thread.current[:vanity_context] = context
-  end
+    # Returns the Vanity context.  For example, when using Rails this would be
+    # the current controller, which can be used to get/set the vanity identity.
+    def context
+      Thread.current[:vanity_context]
+    end
 
+    # Sets the Vanity context.  For example, when using Rails this would be
+    # set by the set_vanity_context before filter (via use_vanity).
+    def context=(context)
+      Thread.current[:vanity_context] = context
+    end
+
+    # Path to template.
+    def template(name)
+      path = File.join(File.dirname(__FILE__), "templates/#{name}")
+      path << ".erb" unless name["."]
+      path
+    end
+
+  end
 end
 
 class Object

data/lib/vanity/rails.rb

@@ -1,6 +1,7 @@
-require File.join(File.dirname(__FILE__), "../vanity")
-require File.join(File.dirname(__FILE__), "rails/helpers")
-require File.join(File.dirname(__FILE__), "rails/testing")
+require "vanity"
+require "vanity/rails/helpers"
+require "vanity/rails/testing"
+require "vanity/rails/console"
 
 # Use Rails logger by default.
 Vanity.playground.logger ||= ActionController::Base.logger

data/lib/vanity/rails/console.rb

@@ -0,0 +1,14 @@
+module Vanity
+  module Rails
+    module ConsoleActions
+      def index
+        render Vanity.template("_report"), content_type: Mime::HTML, layout: true
+      end
+
+      def chooses
+        experiment(params[:e]).chooses(experiment(params[:e]).alternatives[params[:a].to_i].value)
+        redirect_to :back
+      end
+    end
+  end
+end

data/lib/vanity/rails/helpers.rb

@@ -59,10 +59,12 @@ module Vanity
             @vanity_identity = block.call(self)
           elsif symbol && object = send(symbol)
             @vanity_identity = object.id
-          else
+          elsif response # everyday use
             @vanity_identity = cookies["vanity_id"] || OpenSSL::Random.random_bytes(16).unpack("H*")[0]
             cookies["vanity_id"] = { value: @vanity_identity, expires: 1.month.from_now }
             @vanity_identity
+          else # during functional testing
+            @vanity_identity = "test"
           end
         end
         define_method :set_vanity_context do

data/lib/vanity/rails/testing.rb

@@ -3,7 +3,7 @@ module ActionController #:nodoc:
     alias :setup_controller_request_and_response_without_vanity :setup_controller_request_and_response
     def setup_controller_request_and_response
       setup_controller_request_and_response_without_vanity 
-      Vanity.context = @request
+      Vanity.context = @controller
     end
   end
 end

data/lib/vanity/templates/_ab_test.erb

@@ -0,0 +1,25 @@
+<% score = experiment.score %>
+<table>
+  <caption><%= experiment.conclusion(score).join(" ") %></caption>
+  <% score.alts.each do |alt| %>
+    <tr class="<%= "choice" if score.choice == alt %>">
+      <td class="option"><%= alt.name.gsub(/^o/, "O") %>:</td>
+      <td class="value"><code><%= CGI.escape_html alt.value.to_s %></code></td>
+      <td>
+        <%= "%.1f%%" % [alt.conversion_rate * 100] %>
+        <%= "(%d%% better than %s)" % [alt.difference, score.least.name] if alt.difference && alt.difference >= 1 %>
+      </td>
+      <td class="action">
+        <% if experiment.active? && respond_to?(:chooses_experiments_url) %>
+          <% if experiment.chosen?(alt) %>
+            showing
+          <% else %>
+            <%= link_to "show", chooses_experiments_url(e: experiment.id, a: alt.id), method: :post,
+                  class: "button", title: "Show me this alternative from now on" %>
+          <% end %>
+        <% end %>
+      </td>
+    </tr>
+  <% end %>
+</table>
+<%=  %>

data/lib/vanity/templates/_experiments.erb

@@ -0,0 +1,12 @@
+<ul class="experiments">
+  <% experiments.sort_by(&:created_at).each do |experiment| %>
+    <li class="experiment" id="experiment_<%= CGI.escape experiment.id.to_s %>">
+      <h2 class="<%= experiment.type %>"><%= CGI.escape_html experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h2>
+      <p class="description"><%= CGI.escape_html experiment.description.to_s %></p>
+      <%= render Vanity.template("ab_test"), experiment: experiment %>
+      <p class="meta">Started <%= experiment.created_at.strftime("%a, %b %-d %Y") %>
+        <%= " | Completed #{experiment.completed_at.strftime("%a, %b %-d %Y")}" unless experiment.active? %>
+      </p>
+    </li>
+  <% end %>
+</ul>

data/lib/vanity/templates/_report.erb

@@ -0,0 +1,16 @@
+<html>
+<head>
+  <title>Experiments (<%= Time.now %>)</title>
+  <style>
+    .vanity { margin: 2em auto; width: 40em; font-family: "Helvetica Neue", "Helvetica", "Verdana", sans-serif }
+    .vanity h1 { margin: 1em 0; border-bottom: 3px solid #ccc }
+  </style>
+  <%= render Vanity.template("vanity.css") %>
+</head>
+<body>
+  <div class="vanity">
+    <h1>Experiments</h1>
+    <%= render Vanity.template("experiments"), experiments: Vanity.playground.experiments %>
+  </div>
+</body>
+</html>

data/lib/vanity/templates/_vanity.css

@@ -0,0 +1,13 @@
+<style>
+  .vanity .experiments { list-style: none; margin: 0; padding: 0 }
+  .vanity .experiment { padding-bottom: 1em; margin: 0 0 1em 0; border-bottom: 1px dashed #ddd }
+  .vanity .experiment .type { margin-left: .3em; color: #bbb; font-size: .8em; font-weight: normal }
+  .vanity .experiment table { border-collapse: collapse; table-layout: fixed; width: 100%; border-bottom: 1px solid #ccc; margin: 1em 0 0 0 }
+  .vanity .experiment td { padding: .5em; border-top: 1px solid #ccc }
+  .vanity .experiment .choice td { font-weight: bold; background: #f0f0f8 }
+  .vanity .experiment td.option { width: 5em; white-space: nowrap; overflow: hidden }
+  .vanity .experiment td.value { width: 8em; white-space: nowrap; overflow: hidden }
+  .vanity .experiment td.action { width: 6em; overflow: hidden; text-align: center }
+  .vanity .experiment caption { caption-side: bottom; padding: .5em; background: transparent; margin-bottom: 1em; text-align: left }
+  .vanity .experiment .meta { color: #444; font-style: italic }
+</style>

data/test/ab_test_test.rb

@@ -5,19 +5,19 @@ class AbTestController < ActionController::Base
   attr_accessor :current_user
 
   def test_render
-    render text: ab_test(:simple_ab)
+    render text: ab_test(:simple)
   end
 
   def test_view
-    render inline: "<%= ab_test(:simple_ab) %>"
+    render inline: "<%= ab_test(:simple) %>"
   end
 
   def test_capture
-    render inline: "<% ab_test :simple_ab do |value| %><%= value %><% end %>"
+    render inline: "<% ab_test :simple do |value| %><%= value %><% end %>"
   end
 
   def goal
-    ab_goal! :simple_ab
+    ab_goal! :simple
     render text: ""
   end
 end
@@ -25,10 +25,6 @@ end
 
 class AbTestTest < ActionController::TestCase
   tests AbTestController
-  def setup
-    experiment(:simple_ab) { }
-  end
-
 
   # --  Experiment definition --
 
@@ -65,8 +61,8 @@ class AbTestTest < ActionController::TestCase
     experiment :abcd do
       alternatives :a, :b
     end
-    assert_equal "option 1", experiment(:abcd).alternative(:a).name
-    assert_equal "option 2", experiment(:abcd).alternative(:b).name
+    assert_equal "option A", experiment(:abcd).alternative(:a).name
+    assert_equal "option B", experiment(:abcd).alternative(:b).name
   end
 
 
@@ -87,7 +83,7 @@ class AbTestTest < ActionController::TestCase
   def test_returns_different_alternatives_for_each_participant
     experiment :foobar do
       alternatives "foo", "bar"
-      identify { rand(1000).to_s }
+      identify { rand }
     end
     alts = Array.new(1000) { experiment(:foobar).choose }
     assert_equal %w{bar foo}, alts.uniq.sort
@@ -95,48 +91,44 @@ class AbTestTest < ActionController::TestCase
   end
 
   def test_records_all_participants_in_each_alternative
-    ids = (Array.new(200) { |i| i.to_s } * 5).shuffle
+    ids = (Array.new(200) { |i| i } * 5).shuffle
     experiment :foobar do
       alternatives "foo", "bar"
       identify { ids.pop }
     end
     1000.times { experiment(:foobar).choose }
     alts = experiment(:foobar).alternatives
-    assert_equal 200, alts.inject(0) { |total,alt| total + alt.participants }
+    assert_equal 200, alts.map(&:participants).sum
     assert_in_delta alts.first.participants, 100, 20
   end
 
   def test_records_each_converted_participant_only_once
-    ids = (Array.new(100) { |i| i.to_s } * 5).shuffle
-    test = self
+    ids = ((1..100).map { |i| [i,i] } * 5).shuffle.flatten # 3,3,1,1,7,7 etc
     experiment :foobar do
       alternatives "foo", "bar"
-      identify { test.identity ||= ids.pop }
+      identify { ids.pop }
     end
     500.times do
-      test.identity = nil
       experiment(:foobar).choose
       experiment(:foobar).conversion!
     end
     alts = experiment(:foobar).alternatives
-    assert_equal 100, alts.inject(0) { |total,alt| total + alt.converted }
+    assert_equal 100, alts.map(&:converted).sum
   end
 
   def test_records_conversion_only_for_participants
-    test = self
+    ids = ((1..100).map { |i| [-i,i,i] } * 5).shuffle.flatten # -3,3,3,-1,1,1,-7,7,7 etc
     experiment :foobar do
       alternatives "foo", "bar"
-      identify { test.identity ||= rand(100).to_s }
+      identify { ids.pop }
     end
-    1000.times do
-      test.identity = nil
+    500.times do
       experiment(:foobar).choose
       experiment(:foobar).conversion!
-      test.identity << "!"
       experiment(:foobar).conversion!
     end
     alts = experiment(:foobar).alternatives
-    assert_equal 100, alts.inject(0) { |t,a| t + a.converted }
+    assert_equal 100, alts.map(&:converted).sum
   end
 
   def test_reset_experiment
@@ -163,13 +155,13 @@ class AbTestTest < ActionController::TestCase
   # -- A/B helper methods --
 
   def test_fail_if_no_experiment
-    new_playground
     assert_raise MissingSourceFile do
       get :test_render
     end
   end
 
   def test_ab_test_chooses_in_render
+    experiment(:simple) { }
     responses = Array.new(100) do
       @controller = nil ; setup_controller_request_and_response
       get :test_render
@@ -179,6 +171,7 @@ class AbTestTest < ActionController::TestCase
   end
 
   def test_ab_test_chooses_view_helper
+    experiment(:simple) { }
     responses = Array.new(100) do
       @controller = nil ; setup_controller_request_and_response
       get :test_view
@@ -188,6 +181,7 @@ class AbTestTest < ActionController::TestCase
   end
 
   def test_ab_test_with_capture
+    experiment(:simple) { }
     responses = Array.new(100) do
       @controller = nil ; setup_controller_request_and_response
       get :test_capture
@@ -197,6 +191,7 @@ class AbTestTest < ActionController::TestCase
   end
 
   def test_ab_test_goal
+    experiment(:simple) { }
     responses = Array.new(100) do
       @controller.send(:cookies).clear
       get :goal
@@ -208,88 +203,95 @@ class AbTestTest < ActionController::TestCase
   # -- Testing with tests --
   
   def test_with_given_choice
-    100.times do
+    experiment(:simple) { alternatives :a, :b, :c }
+    100.times do |i|
       @controller = nil ; setup_controller_request_and_response
-      experiment(:simple_ab).chooses(true)
+      experiment(:simple).chooses(:b)
       get :test_render
-      post :goal
+      assert "b", @response.body
     end
-    alts = experiment(:simple_ab).alternatives
-    assert_equal [0,100], alts.map { |alt| alt.participants }
-    assert_equal [0,100], alts.map { |alt| alt.conversions }
   end
 
   def test_which_chooses_non_existent_alternative
+    experiment(:simple) { }
     assert_raises ArgumentError do
-      experiment(:simple_ab).chooses(404)
+      experiment(:simple).chooses(404)
     end
   end
 
 
   # -- Scoring --
-  
+ 
   def test_scoring
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # participating, conversions, rate, z-score
     # Control:      182	35 19.23%	N/A
-    182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
-    35.times  { |i| experiment(:abcd).alternative(:a).conversion!(i) }
+    182.times { |i| experiment(:abcd).count i, :a, :participant }
+    35.times  { |i| experiment(:abcd).count i, :a, :conversion }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    180.times { |i| experiment(:abcd).count i, :b, :participant }
+    45.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment B:  189	28 14.81%	-1.13
-    189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
-    28.times  { |i| experiment(:abcd).alternative(:c).conversion!(i) }
+    189.times { |i| experiment(:abcd).count i, :c, :participant }
+    28.times  { |i| experiment(:abcd).count i, :c, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
-    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
     assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
-    confidences = experiment(:abcd).score.alts.map(&:conf)
+    confidences = experiment(:abcd).score.alts.map(&:confidence)
     assert_equal [90, 0, 99, 90], confidences
 
-    diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
+    diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
     assert_equal [30, 69, nil, 119], diff
     assert_equal 3, experiment(:abcd).score.best.id
     assert_equal 3, experiment(:abcd).score.choice.id
+
+    assert_equal 1, experiment(:abcd).score.base.id
+    assert_equal 2, experiment(:abcd).score.least.id
   end
 
   def test_scoring_with_no_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
     assert_nil experiment(:abcd).score.best
     assert_nil experiment(:abcd).score.choice
+    assert_nil experiment(:abcd).score.least
   end
 
   def test_scoring_with_one_performer
     experiment(:abcd) { alternatives :a, :b, :c, :d }
-    10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    8.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
-    assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
+    10.times { |i| experiment(:abcd).count i, :b, :participant }
+    8.times  { |i| experiment(:abcd).count i, :b, :conversion }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.confidence == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
     assert 1, experiment(:abcd).score.best.id
     assert_nil experiment(:abcd).score.choice
+    assert 1, experiment(:abcd).score.base.id
+    assert 1, experiment(:abcd).score.least.id
   end
 
   def test_scoring_with_some_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
-    10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    8.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
-    12.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    5.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    10.times { |i| experiment(:abcd).count i, :b, :participant }
+    8.times  { |i| experiment(:abcd).count i, :b, :conversion }
+    12.times { |i| experiment(:abcd).count i, :d, :participant }
+    5.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
-    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
     assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
-    confidences = experiment(:abcd).score.alts.map(&:conf)
+    confidences = experiment(:abcd).score.alts.map(&:confidence)
     assert_equal [0, 95, 0, 0], confidences
-    diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
+    diff = experiment(:abcd).score.alts.map { |alt| alt.difference && alt.difference.round }
     assert_equal [nil, 92, nil, nil], diff
     assert_equal 1, experiment(:abcd).score.best.id
     assert_equal 1, experiment(:abcd).score.choice.id
+    assert_equal 3, experiment(:abcd).score.base.id
+    assert_equal 3, experiment(:abcd).score.least.id
   end
 
 
@@ -299,105 +301,105 @@ class AbTestTest < ActionController::TestCase
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # participating, conversions, rate, z-score
     # Control:      182	35 19.23%	N/A
-    182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
-    35.times  { |i| experiment(:abcd).alternative(:a).conversion!(i) }
+    182.times { |i| experiment(:abcd).count i, :a, :participant }
+    35.times  { |i| experiment(:abcd).count i, :a, :conversion }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    180.times { |i| experiment(:abcd).count i, :b, :participant }
+    45.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment B:  189	28 14.81%	-1.13
-    189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
-    28.times  { |i| experiment(:abcd).alternative(:c).conversion!(i) }
+    189.times { |i| experiment(:abcd).count i, :c, :participant }
+    28.times  { |i| experiment(:abcd).count i, :c, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
-The best choice is option 4: it converted at 32.4% (30% better than option 2).
+The best choice is option D: it converted at 32.4% (30% better than option B).
 With 90% probability this result is statistically significant.
-Option 2 converted at 25.0%.
-Option 1 converted at 19.2%.
-Option 3 converted at 14.8%.
-Option 4 selected as the best alternative.
+Option B converted at 25.0%.
+Option A converted at 19.2%.
+Option C converted at 14.8%.
+Option D selected as the best alternative.
     TEXT
   end
 
   def test_conclusion_with_some_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    180.times { |i| experiment(:abcd).count i, :b, :participant }
+    45.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
-The best choice is option 4: it converted at 32.4% (30% better than option 2).
+The best choice is option D: it converted at 32.4% (30% better than option B).
 With 90% probability this result is statistically significant.
-Option 2 converted at 25.0%.
-Option 1 did not convert.
-Option 3 did not convert.
-Option 4 selected as the best alternative.
+Option B converted at 25.0%.
+Option A did not convert.
+Option C did not convert.
+Option D selected as the best alternative.
     TEXT
   end
 
   def test_conclusion_without_clear_winner
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    58.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    180.times { |i| experiment(:abcd).count i, :b, :participant }
+    58.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
-The best choice is option 4: it converted at 32.4% (1% better than option 2).
+The best choice is option D: it converted at 32.4% (1% better than option B).
 This result is not statistically significant, suggest you continue this experiment.
-Option 2 converted at 32.2%.
-Option 1 did not convert.
-Option 3 did not convert.
+Option B converted at 32.2%.
+Option A did not convert.
+Option C did not convert.
     TEXT
   end
 
   def test_conclusion_without_close_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # Treatment A:  180	45 25.00%	1.33
-    186.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    60.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    186.times { |i| experiment(:abcd).count i, :b, :participant }
+    60.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
-The best choice is option 4: it converted at 32.4%.
+The best choice is option D: it converted at 32.4%.
 This result is not statistically significant, suggest you continue this experiment.
-Option 2 converted at 32.3%.
-Option 1 did not convert.
-Option 3 did not convert.
+Option B converted at 32.3%.
+Option A did not convert.
+Option C did not convert.
     TEXT
   end
 
   def test_conclusion_without_equal_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # Treatment A:  180	45 25.00%	1.33
-    188.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :b, :participant }
+    61.times  { |i| experiment(:abcd).count i, :b, :conversion }
     # treatment C:  188	61 32.45%	2.94
-    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
-    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+    188.times { |i| experiment(:abcd).count i, :d, :participant }
+    61.times  { |i| experiment(:abcd).count i, :d, :conversion }
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
-Option 4 converted at 32.4%.
-Option 2 converted at 32.4%.
-Option 1 did not convert.
-Option 3 did not convert.
+Option D converted at 32.4%.
+Option B converted at 32.4%.
+Option A did not convert.
+Option C did not convert.
     TEXT
   end
 
   def test_conclusion_with_one_performers
     experiment(:abcd) { alternatives :a, :b, :c, :d }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
-    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    180.times { |i| experiment(:abcd).count i, :b, :participant }
+    45.times  { |i| experiment(:abcd).count i, :b, :conversion }
 
     assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
   end
@@ -444,17 +446,15 @@ Option 3 did not convert.
   end
 
   def test_ab_methods_after_completion
-    ids = Array.new(200) { |i| i.to_s }.shuffle
-    test = self
+    ids = Array.new(200) { |i| [i, i] }.shuffle.flatten
     experiment :simple do
-      identify { test.identity ||= ids.pop }
+      identify { ids.pop }
       complete_if { alternatives.map(&:participants).sum >= 100 }
       outcome_is { alternatives[1] }
     end
     # Run experiment to completion (100 participants)
     results = Set.new
     100.times do
-      test.identity = nil
       results << experiment(:simple).choose
       experiment(:simple).conversion!
     end
@@ -463,7 +463,6 @@ Option 3 did not convert.
 
     # Test that we always get the same choice (true)
     100.times do
-      test.identity = nil
       assert_equal true, experiment(:simple).choose
       experiment(:simple).conversion!
     end
@@ -510,41 +509,27 @@ Option 3 did not convert.
   def test_outcome_choosing_best_alternative
     experiment :quick do
     end
-    2.times do |i|
-      experiment(:quick).alternatives[0].participating!(i)
-    end
-    10.times do |i|
-      experiment(:quick).alternatives[1].participating!(i)
-      experiment(:quick).alternatives[1].conversion!(i)
-    end
+    2.times  { |i| experiment(:quick).count i, false, :participant }
+    10.times { |i| experiment(:quick).count i, true }
     experiment(:quick).complete!
-    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
   end
 
   def test_outcome_only_performing_alternative
     experiment :quick do
     end
-    2.times do |i|
-      experiment(:quick).alternatives[1].participating!(i)
-      experiment(:quick).alternatives[1].conversion!(i)
-    end
+    2.times { |i| experiment(:quick).count i, true }
     experiment(:quick).complete!
-    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
   end
 
   def test_outcome_choosing_equal_alternatives
     experiment :quick do
     end
-    8.times do |i|
-      experiment(:quick).alternatives[0].participating!(i)
-      experiment(:quick).alternatives[0].conversion!(i)
-    end
-    8.times do |i|
-      experiment(:quick).alternatives[1].participating!(i)
-      experiment(:quick).alternatives[1].conversion!(i)
-    end
+    8.times { |i| experiment(:quick).count i, false }
+    8.times { |i| experiment(:quick).count i, true }
     experiment(:quick).complete!
-    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+    assert_equal experiment(:quick).alternative(true), experiment(:quick).outcome
   end
 
 end