vanity 1.8.2 → 1.8.3.beta

data/lib/vanity/experiment/alternative.rb

@@ -0,0 +1,93 @@
+module Vanity
+  module Experiment
+
+    # One of several alternatives in an A/B test (see AbTest#alternatives).
+    class Alternative
+
+      def initialize(experiment, id, value) #, participants, converted, conversions)
+      	@experiment = experiment
+      	@id = id
+      	@name = "option #{(@id + 65).chr}"
+      	@value = value
+      end
+
+      # Alternative id, only unique for this experiment.
+      attr_reader :id
+
+      # Alternative name (option A, option B, etc).
+      attr_reader :name
+
+      # Alternative value.
+      attr_reader :value
+
+      # Experiment this alternative belongs to.
+      attr_reader :experiment
+
+      # Number of participants who viewed this alternative.
+      def participants
+      	load_counts unless @participants
+      	@participants
+      end
+
+      # Number of participants who converted on this alternative (a
+      # participant is counted only once).
+      def converted
+      	load_counts unless @converted
+      	@converted
+      end
+
+      # Number of conversions for this alternative (same participant may be
+      # counted more than once).
+      def conversions
+      	load_counts unless @conversions
+      	@conversions
+      end
+
+      # Z-score for this alternative, related to 2nd-best performing
+      # alternative. Populated by AbTest#score if #score_method is :z_score.
+      attr_accessor :z_score
+
+      # Probability alternative is best. Populated by AbTest#score.
+      attr_accessor :probability
+
+      # Difference from least performing alternative. Populated by
+      # AbTest#score.
+      attr_accessor :difference
+
+      # Conversion rate calculated as converted/participants
+      def conversion_rate
+      	@conversion_rate ||= (participants > 0 ? converted.to_f/participants.to_f  : 0.0)
+      end
+
+      # The measure we use to order (sort) alternatives and decide which one
+      # is better (by calculating z-score). Defaults to conversion rate.
+      def measure
+      	conversion_rate
+      end
+
+      def <=>(other)
+      	measure <=> other.measure
+      end
+
+      def ==(other)
+      	other && id == other.id && experiment == other.experiment
+      end
+
+      def to_s
+      	name
+      end
+
+      def inspect
+      	"#{name}: #{value} #{converted}/#{participants}"
+      end
+
+      def load_counts
+      	if @experiment.playground.collecting?
+      	  @participants, @converted, @conversions = @experiment.playground.connection.ab_counts(@experiment.id, id).values_at(:participants, :converted, :conversions)
+      	else
+      	  @participants = @converted = @conversions = 0
+      	end
+      end
+    end
+  end
+end

data/lib/vanity/experiment/base.rb

@@ -1,40 +1,11 @@
+require "vanity/experiment/definition"
+
 module Vanity
   module Experiment
-
-    # These methods are available from experiment definitions (files located in
-    # the experiments directory, automatically loaded by Vanity).  Use these
-    # methods to define you experiments, for example:
-    #   ab_test "New Banner" do
-    #     alternatives :red, :green, :blue
-    #     metrics :signup
-    #   end
-    module Definition
-
-      attr_reader :playground
-
-      # Defines a new experiment, given the experiment's name, type and
-      # definition block.
-      def define(name, type, options = nil, &block)
-        fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
-        klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
-        experiment = klass.new(playground, @experiment_id, name, options)
-        experiment.instance_eval &block
-        experiment.save
-        playground.experiments[@experiment_id] = experiment
-      end
-
-      def new_binding(playground, id)
-        @playground, @experiment_id = playground, id
-        binding
-      end
-
-    end
-
     # Base class that all experiment types are derived from.
     class Base
 
       class << self
-
         # Returns the type of this class as a symbol (e.g. AbTest becomes
         # ab_test).
         def type
@@ -69,7 +40,7 @@ module Vanity
         @id, @name = id.to_sym, name
         @options = options || {}
         @identify_block = method(:default_identify)
-	@on_assignment_block = nil
+        @on_assignment_block = nil
       end
 
       # Human readable experiment name (first argument you pass when creating a
@@ -124,8 +95,8 @@ module Vanity
       #     end
       #   end
       def on_assignment(&block)
-	fail "Missing block" unless block
-	@on_assignment_block = block
+      	fail "Missing block" unless block
+      	@on_assignment_block = block
       end
 
       # -- Reporting --
@@ -155,7 +126,9 @@ module Vanity
       end
 
       # Force experiment to complete.
-      def complete!
+      # @param optional integer id of the alternative that is the decided
+      # outcome of the experiment
+      def complete!(outcome = nil)
         @playground.logger.info "vanity: completed experiment #{id}"
         return unless @playground.collecting?
         connection.set_experiment_completed_at @id, Time.now

data/lib/vanity/experiment/bayesian_bandit_score.rb

@@ -0,0 +1,93 @@
+module Vanity
+  module Experiment
+    class Score
+      attr_accessor :alts, :best, :base, :least, :choice, :method
+    end
+
+    class BayesianBanditScore < Score
+      DEFAULT_PROBABILITY = 90
+
+      def initialize(alternatives, outcome)
+      	@alternatives = alternatives
+      	@outcome = outcome
+      	@method = :bayes_bandit_score
+      end
+
+      def calculate!(probability=DEFAULT_PROBABILITY)
+      	# sort by conversion rate to find second best
+      	@alts = @alternatives.sort_by(&:measure)
+      	@base = @alts[-2]
+
+      	assign_alternatives_bayesian_probability(@alts)
+
+      	@least = assign_alternatives_difference(@alts)
+
+      	# best alternative is one with highest conversion rate (best shot).
+      	@best = @alts.last if @alts.last.measure > 0.0
+      	# choice alternative can only pick best if we have high probability (>90%).
+      	@choice = outcome_or_best_probability(@alternatives, @outcome, @best, probability)
+      	self
+      end
+
+      protected
+
+      def outcome_or_best_probability(alternatives, outcome, best, probability)
+      	if outcome
+      	  alternatives[@outcome.id]
+      	elsif best && best.probability >= probability
+      	  best
+      	else
+      	  nil
+      	end
+      end
+
+      # Assign each alternative's bayesian probability of being the best
+      # alternative to alternative#probability.
+      def assign_alternatives_bayesian_probability(alternatives)
+      	alternative_posteriors = calculate_alternative_posteriors(alternatives)
+      	alternatives.each_with_index do |alternative, i|
+      	  alternative.probability = 100 * probability_alternative_is_best(alternative_posteriors[i], alternative_posteriors)
+      	end
+      end
+
+      def calculate_alternative_posteriors(alternatives)
+      	alternatives.map do |alternative|
+      	  x = alternative.converted
+      	  n = alternative.participants
+      	  Rubystats::BetaDistribution.new(x+1, n-x+1)
+      	end
+      end
+
+      def probability_alternative_is_best(alternative_being_examined, all_alternatives)
+      	Integration.integrate(0, 1, :tolerance=>1e-4) do |z|
+      	  pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
+      	end
+      end
+
+      def pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
+      	# get the pdf for this alternative at z
+      	pdf = alternative_being_examined.pdf(z)
+      	# now multiply by the probability that all the other alternatives are lower
+      	all_alternatives.each do |alternative|
+      	  if alternative != alternative_being_examined
+      	    pdf = pdf * alternative.cdf(z)
+      	  end
+      	end
+      	pdf
+      end
+
+      def assign_alternatives_difference(alternatives)
+      	# difference is measured from least performant
+      	least = alternatives.find { |alternative| alternative.measure > 0 }
+      	if least
+      	  alternatives.each do |alternative|
+      	    if alternative.measure > least.measure
+      	      alternative.difference = (alternative.measure - least.measure) / least.measure * 100
+      	    end
+      	  end
+      	end
+      	least
+      end
+    end
+  end
+end

data/lib/vanity/experiment/definition.rb

@@ -0,0 +1,32 @@
+module Vanity
+  module Experiment
+    # These methods are available from experiment definitions (files located in
+    # the experiments directory, automatically loaded by Vanity).  Use these
+    # methods to define you experiments, for example:
+    #   ab_test "New Banner" do
+    #     alternatives :red, :green, :blue
+    #     metrics :signup
+    #   end
+    module Definition
+
+      attr_reader :playground
+
+      # Defines a new experiment, given the experiment's name, type and
+      # definition block.
+      def define(name, type, options = nil, &block)
+      	fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
+      	klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
+      	experiment = klass.new(playground, @experiment_id, name, options)
+      	experiment.instance_eval &block
+      	experiment.save
+      	playground.experiments[@experiment_id] = experiment
+      end
+
+      def new_binding(playground, id)
+      	@playground, @experiment_id = playground, id
+      	binding
+      end
+
+    end
+  end
+end

data/lib/vanity/frameworks/rails.rb

@@ -284,6 +284,24 @@ module Vanity
         render :file=>Vanity.template("_report"), :content_type=>Mime::HTML, :layout=>false
       end
 
+      def participant
+        render :file=>Vanity.template("_participant"), :locals=>{:participant_id => params[:id], :participant_info => Vanity.playground.participant_info(params[:id])}, :content_type=>Mime::HTML, :layout=>false
+      end
+
+      def complete
+        exp = Vanity.playground.experiment(params[:e].to_sym)
+        alt = exp.alternatives[params[:a].to_i]
+        confirmed = params[:confirmed]
+        # make the user confirm before completing an experiment
+        if confirmed && confirmed.to_i==alt.id && exp.active?
+          exp.complete!(alt.id)
+          render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
+        else
+          @to_confirm = alt.id
+          render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
+        end
+      end
+
       def chooses
         exp = Vanity.playground.experiment(params[:e].to_sym)
         exp.chooses(exp.alternatives[params[:a].to_i].value)

data/lib/vanity/playground.rb

@@ -95,6 +95,22 @@ module Vanity
     end
 
 
+    # -- Participant Information --
+
+    # Returns an array of all experiments this participant is involved in, with their assignment.
+    #  This is done as an array of arrays [[<experiment_1>, <assignment_1>], [<experiment_2>, <assignment_2>]], sorted by experiment name, so that it will give a consistent string
+    #  when converted to_s (so could be used for caching, for example)
+    def participant_info(participant_id)
+      participant_array = []
+      experiments.values.sort_by{|e| e.name}.each do |e|
+        index = connection.ab_assigned(e.id, participant_id)
+        if index
+          participant_array << [e, e.alternatives[index.to_i]]
+        end
+      end
+      return participant_array
+    end
+
     # -- Robot Detection --
 
     # Call to indicate that participants should be added via js

data/lib/vanity/templates/_ab_test.erb

@@ -1,7 +1,8 @@
-<% score = experiment.score %>
+<% score = experiment.calculate_score %>
 <table>
   <caption>
-    <%= experiment.conclusion(score).join(" ") %></caption>
+    <%= experiment.conclusion(score).join(" ") %>
+  </caption>
   <% score.alts.each do |alt| %>
     <tr class="<%= "choice" if score.choice == alt %>">
       <td class="option"><%= alt.name.gsub(/^o/, "O") %>:</td>
@@ -10,19 +11,33 @@
       <td class="value"><%= alt.converted %> converted</td>
       <td>
         <%= "%.1f%%" % [alt.conversion_rate * 100] %>
+        <%= "(%d%% this is best)" % [alt.probability] if score.method == :bayes_score %>
         <%= "(%d%% better than %s)" % [alt.difference, score.least.name] if alt.difference && alt.difference >= 1 %>
       </td>
-      <td class="action">
-        <% if experiment.active? && respond_to?(:url_for) %>
-          <% if experiment.showing?(alt) %>
-            showing
-          <% else %>
-            <a class="button chooses" title="Show me this alternative from now on" href="#"
-              data-id="<%= experiment.id %>" data-url="<%= url_for(:action=>:chooses, :e=>experiment.id, :a=>alt.id) %>">show</a>
-          <% end %>
-        <% end %>
-      </td>
+      <% if experiment.active? && respond_to?(:url_for) %>
+        <td class="action">
+          <small>
+            <% if experiment.showing?(alt) %>
+              currently <br> shown
+            <% else %>
+              <a class="button chooses" title="Show me this alternative from now on" href="#"
+                data-id="<%= experiment.id %>" data-url="<%= url_for(:action=>:chooses, :e=>experiment.id, :a=>alt.id) %>">show me</a>
+            <% end %>
+          </small>
+        </td>
+        <td class="action">
+          <small>
+            <% if @to_confirm == alt.id %>
+             Finish this experiment and assign all current and future participants to <%= alt.name %>?
+             <a class="button chooses" title="Confirm experiment completion and assignment of all current and future participants to <%= alt.name %>" href="#"
+              data-id="<%= experiment.id %>" data-url="<%= url_for(:action=>:complete, :e=>experiment.id, :a=>alt.id, :confirmed=>alt.id) %>">confirm</a>
+            <% else %>
+             <a class="button chooses" title="Complete the experiment and assign all current and future participants to <%= alt.name %>" href="#"
+              data-id="<%= experiment.id %>" data-url="<%= url_for(:action=>:complete, :e=>experiment.id, :a=>alt.id) %>">make permanent</a>
+            <% end %>
+          </small>
+        </td>
+      <% end %>
     </tr>
   <% end %>
 </table>
-<%=  %>

data/lib/vanity/templates/_participant.erb

@@ -0,0 +1,12 @@
+<p>Participant id <%= participant_id %> is taking part in the following experiments:
+<ul class="experiments">
+  <% participant_info.each do |experiment, alt| %>
+    <li class="experiment <%= experiment.type %>" id="experiment_<%=vanity_h experiment.id.to_s %>">
+	<h3><%=vanity_h experiment.name %> <span class="type">(<%= experiment.class.friendly_name %>)</span></h3>
+	<%= experiment.description.to_s.split(/\n\s*\n/).map { |para| vanity_html_safe(%{<p class="description">#{vanity_h para}</p>}) }.join.html_safe %>
+    <br /><span class="option"><%= alt.name.gsub(/^o/, "O") %>:</span>
+    <span class="value"><code><%=vanity_h alt.value.to_s %></code></span>
+    </li>
+  <% end %>
+</ul>
+</p>

data/lib/vanity/version.rb

@@ -1,5 +1,5 @@
 module Vanity
-  VERSION = "1.8.2"
+  VERSION = "1.8.3.beta"
 
   module Version
     version = VERSION.to_s.split(".").map { |i| i.to_i }

data/test/experiment/ab_test.rb

@@ -174,6 +174,106 @@ class AbTestTest < ActionController::TestCase
     end
   end
 
+  # -- ab_assigned --
+
+  def test_ab_assigned
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      metrics :coolness
+    end
+    assert_equal nil, experiment(:foobar).playground.connection.ab_assigned(experiment(:foobar).id, "6e98ec")
+    assert id = experiment(:foobar).choose.id
+    assert_equal id, experiment(:foobar).playground.connection.ab_assigned(experiment(:foobar).id, "6e98ec")
+  end
+
+  # -- Unequal probabilities --
+
+  def test_returns_the_same_alternative_consistently_when_using_probabilities
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      rebalance_frequency 100
+      metrics :coolness
+    end
+    assert value = experiment(:foobar).choose.value
+    assert_match /foo|bar/, value
+    1000.times do
+      assert_equal value, experiment(:foobar).choose.value
+    end
+  end
+
+  def test_uses_probabilities_for_new_assignments
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { rand }
+      rebalance_frequency 10000
+      metrics :coolness
+    end
+    altered_alts = experiment(:foobar).alternatives
+    altered_alts[0].probability=30
+    altered_alts[1].probability=70
+    experiment(:foobar).set_alternative_probabilities altered_alts
+    alts = Array.new(1000) { experiment(:foobar).choose.value }
+    assert_equal %w{bar foo}, alts.uniq.sort
+    assert_in_delta alts.select { |a| a == altered_alts[0].value }.size, 300, 100 # this may fail, such is propability
+  end
+
+  # -- Rebalancing probabilities --
+
+  def test_rebalances_probabilities_after_rebalance_frequency_calls
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { rand }
+      rebalance_frequency 12
+      metrics :coolness
+    end
+    class <<experiment(:foobar)
+      def times_called
+        @times_called || 0
+      end
+      def rebalance!
+        @times_called = times_called + 1
+      end
+    end
+    11.times { experiment(:foobar).choose.value }
+    assert_equal 0, experiment(:foobar).times_called
+    experiment(:foobar).choose.value
+    assert_equal 1, experiment(:foobar).times_called
+    12.times { experiment(:foobar).choose.value }
+    assert_equal 2, experiment(:foobar).times_called
+  end
+
+  def test_rebalance_uses_bayes_score_probabilities_to_update_probabilities
+    new_ab_test :foobar do
+      alternatives "foo", "bar", "baa"
+      identify { rand }
+      rebalance_frequency 12
+      metrics :coolness
+    end
+    corresponding_probabilities = [[experiment(:foobar).alternatives[0], 0.3], [experiment(:foobar).alternatives[1], 0.6], [experiment(:foobar).alternatives[2], 1.0]]
+
+    class <<experiment(:foobar)
+      def was_called
+        @was_called
+      end
+      def bayes_bandit_score(probability=90)
+        @was_called = true
+        altered_alts = Vanity.playground.experiment(:foobar).alternatives
+        altered_alts[0].probability=30
+        altered_alts[1].probability=30
+        altered_alts[2].probability=40
+        Struct.new(:alts,:method).new(altered_alts,:bayes_bandit_score)
+      end
+      def use_probabilities
+        @use_probabilities
+      end
+    end
+    experiment(:foobar).rebalance!
+    assert experiment(:foobar).was_called
+    assert_equal experiment(:foobar).use_probabilities, corresponding_probabilities
+  end
+
   # -- Running experiment --
 
   def test_returns_the_same_alternative_consistently
@@ -365,6 +465,22 @@ class AbTestTest < ActionController::TestCase
 
 
   # -- Scoring --
+  def test_calculate_score
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    score_result = experiment(:abcd).calculate_score
+    assert_equal :score, score_result.method
+
+    new_ab_test :bayes_abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+      score_method :bayes_bandit_score
+    end
+    bayes_score_result = experiment(:bayes_abcd).calculate_score
+    assert_equal :bayes_bandit_score, bayes_score_result.method
+  end
 
   def test_scoring
     new_ab_test :abcd do
@@ -392,6 +508,23 @@ class AbTestTest < ActionController::TestCase
     assert_equal 2, experiment(:abcd).score.least.id
   end
 
+  def test_bayes_scoring
+    new_ab_test :abcd do
+      alternatives :a, :b, :c, :d
+      metrics :coolness
+    end
+    # participating, conversions, rate, z-score
+    # Control:      182	35 19.23%	N/A
+    # Treatment A:  180	45 25.00%	1.33
+    # treatment B:  189	28 14.81%	-1.13
+    # treatment C:  188	61 32.45%	2.94
+    fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
+
+    score_result = experiment(:abcd).bayes_bandit_score
+    probabilities = score_result.alts.map{|a| a.probability.round}
+    assert_equal [0,0,6,94], probabilities
+  end
+
   def test_scoring_with_no_performers
     new_ab_test :abcd do
       alternatives :a, :b, :c, :d
@@ -647,6 +780,14 @@ This experiment did not run long enough to find a clear winner.
     assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
   end
 
+  def test_completion_with_outcome
+    new_ab_test :quick do
+      metrics :coolness
+    end
+    experiment(:quick).complete!(1)
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+  end
+
   def test_error_in_completion
     new_ab_test :quick do
       outcome_is { raise RuntimeError }