vanity 0.2.0 → 0.2.1

data/CHANGELOG

@@ -1,4 +1,10 @@
-0.1.1
+0.2.1 (2009-11-11)
+* Added: z-score and confidence level for A/B test alternatives.
+* Added: test auto-completion and auto-outcome (complete_it, outcome_is).
+* Changed: default alternatives are now false/true, so if can't decide
+outcome, fall back on false.
+
+0.2.0 (2009-11-10)
 * Added: experiment method on object, used to define and access experiments.
 * Added: playground configuration (Vanity.playground.namespace = , etc).
 * Added: use_vanity now accepts block instead of symbol.

data/README.rdoc

@@ -51,7 +51,7 @@ the value.
 Here are some examples:
 
   def index
-    if ab_test(:new_page) # classic true/false test
+    if ab_test(:new_page) # classic false/true test
       render action: "new_page"
     else
       render action: "index"

data/lib/vanity/experiment/ab_test.rb

@@ -31,6 +31,15 @@ module Vanity
         redis.get(key("conversions")).to_i
       end
 
+      # Conversion rate calculated as converted/participants.
+      def conversion_rate
+        converted.to_f / participants.to_f
+      end
+
+      def <=>(other)
+        conversion_rate <=> other.conversion_rate 
+      end
+
       def participating!(identity)
         redis.sadd key("participants"), identity
       end
@@ -42,6 +51,33 @@ module Vanity
         end
       end
 
+      # Z-score this alternativet related to the base alternative.  This
+      # alternative is better than base if it receives a positive z-score,
+      # worse if z-score is negative.  Call #confident if you need confidence
+      # level (percentage).
+      def z_score
+        return 0 if base == self
+        pc = base.conversion_rate
+        nc = base.participants
+        p = conversion_rate
+        n = participants
+        (p - pc) / Math.sqrt((p * (1-p)/n) + (pc * (1-pc)/nc))
+      end
+
+      # How confident are we in this alternative being an improvement over the
+      # base alternative.  Returns 0, 90, 95, 99 or 99.9 (percentage).
+      def confidence
+        score = z_score
+        confidence = AbTest::Z_TO_CONFIDENCE.find { |z,p| score >= z }
+        confidence ? confidence.last : 0
+      end
+
+      def destroy #:nodoc:
+        redis.del key("participants")
+        redis.del key("converted")
+        redis.del key("conversions")
+      end
+
     protected
 
       def key(name)
@@ -52,39 +88,20 @@ module Vanity
         @experiment.redis
       end
 
+      def base
+        @base ||= @experiment.alternatives.first
+      end
+
     end
 
+
     # The meat.
     class AbTest < Base
       def initialize(*args) #:nodoc:
         super
       end
 
-      # Chooses a value for this experiment.
-      #
-      # This method returns different values for different identity (see
-      # #identify), and consistenly the same value for the same
-      # expriment/identity pair.
-      #
-      # For example:
-      #   color = experiment(:which_blue).choose
-      def choose
-        identity = identify
-        alt = alternative_for(identity)
-        alt.participating! identity
-        alt.value
-      end
-
-      # Records a conversion.
-      #
-      # For example:
-      #   experiment(:which_blue).conversion!
-      def conversion!
-        identity = identify
-        alt = alternative_for(identity)
-        alt.conversion! identity
-        alt.id
-      end
+      # -- Alternatives --
 
       # Call this method once to specify values for the A/B test.  At least two
       # values are required.
@@ -99,7 +116,7 @@ module Vanity
       #   alts = experiment(:background_color).alternatives
       #   puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
       def alternatives(*args)
-        args = [true, false] if args.empty?
+        args = [false, true] if args.empty?
         @alternatives = []
         args.each_with_index do |arg, i|
           @alternatives << Alternative.new(self, i, arg)
@@ -108,18 +125,50 @@ module Vanity
         alternatives
       end
 
-      # Sets this test to two alternatives: true and false.
-      def true_false
-        alternatives true, false
+      # Sets this test to two alternatives: false and true.
+      def false_true
+        alternatives false, true
       end
+      alias true_false false_true
 
-      def report
-        alts = alternatives.map { |alt|
-          "<dt>Option #{(65 + alt.id).chr}</dt><dd><code>#{CGI.escape_html alt.value.inspect}</code> viewed #{alt.participants} times, converted #{alt.conversions}<dd>"
-        }
-        %{<dl class="data">#{alts.join}</dl>}
+      # Chooses a value for this experiment.
+      #
+      # This method returns different values for different identity (see
+      # #identify), and consistenly the same value for the same
+      # expriment/identity pair.
+      #
+      # For example:
+      #   color = experiment(:which_blue).choose
+      def choose
+        if active?
+          identity = identify
+          alt = alternative_for(identity)
+          alt.participating! identity
+          check_completion!
+          alt.value
+        elsif alternative = outcome
+          alternative.value
+        else
+          alternatives.first.value
+        end
+      end
+
+      # Records a conversion.
+      #
+      # For example:
+      #   experiment(:which_blue).conversion!
+      def conversion!
+        if active?
+          identity = identify
+          alt = alternative_for(identity)
+          alt.conversion! identity
+          check_completion!
+        end
       end
 
+      
+      # -- Testing --
+     
       # Forces this experiment to use a particular alternative. Useful for
       # tests, e.g.
       #
@@ -142,15 +191,78 @@ module Vanity
         Vanity.context.session[:vanity][id] = alternative.id
       end
 
+
+      # -- Reporting --
+
+      def report
+        alts = alternatives.map { |alt|
+          "<dt>Option #{(65 + alt.id).chr}</dt><dd><code>#{CGI.escape_html alt.value.inspect}</code> viewed #{alt.participants} times, converted #{alt.conversions}, rate #{alt.conversion_rate}, z_score #{alt.z_score}, confidence #{alt.confidence}<dd>"
+        }
+        %{<dl class="data">#{alts.join}</dl>}
+      end
+
       def humanize
         "A/B Test" 
       end
 
+
+      # -- Completion --
+
+      # Defines how the experiment can choose the optimal outcome on completion.
+      #
+      # The default implementation looks for the best (highest conversion rate)
+      # alternative.  If it's certain (95% or more) that this alternative is
+      # better than the first alternative, it switches to that one.  If it has
+      # no such certainty, it starts using the first alternative exclusively.
+      #
+      # The default implementation reads like this:
+      #   outcome_is do
+      #     highest = alternatives.sort.last
+      #     highest.confidence >= 95 ? highest ? alternatives.first
+      #   end
+      def outcome_is(&block)
+        raise ArgumentError, "Missing block" unless block
+        raise "outcome_is already called on this experiment" if @outcome_is
+        @outcome_is = block
+      end
+
+      # Alternative chosen when this experiment was completed.
+      def outcome
+        outcome = redis.get(key("outcome"))
+        outcome && alternatives[outcome.to_i]
+      end
+
+      def complete! #:nodoc:
+        super
+        if @outcome_is
+          begin
+            outcome = alternatives.find_index(@outcome_is.call)
+          rescue
+            # TODO: logging
+          end
+        end
+        unless outcome
+          highest = alternatives.sort.last rescue nil
+          outcome = highest && highest.confidence >= 95 ? highest.id : 0
+        end
+        # TODO: logging
+        redis.setnx key("outcome"), outcome
+      end
+
+      
+      # -- Store/validate --
+
       def save #:nodoc:
         fail "Experiment #{name} needs at least two alternatives" unless alternatives.count >= 2
         super
       end
 
+      def destroy #:nodoc:
+        redis.del key(:outcome)
+        alternatives.each(&:destroy)
+        super
+      end
+
     private
 
       # Chooses an alternative for the identity and returns its index. This
@@ -164,7 +276,14 @@ module Vanity
         alternatives[index]
       end
 
+      begin
+        a = 0
+        # Returns array of [z-score, percentage]
+        norm_dist = (-5.0..3.1).step(0.01).map { |x| [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
+        # We're really only interested in 90%, 95%, 99% and 99.9%.
+        Z_TO_CONFIDENCE = [90, 95, 99, 99.9].map { |pct| [norm_dist.find { |x,a| a >= pct }.first, pct] }.reverse
+      end
+
     end
   end
 end
-

data/lib/vanity/experiment/base.rb

@@ -18,8 +18,8 @@ module Vanity
         @playground = playground
         @id, @name = id.to_sym, name
         @namespace = "#{@playground.namespace}:#{@id}"
-        created = redis.get(key(:created_at)) || (redis.setnx(key(:created_at), Time.now.to_i) ; redis.get(key(:created_at))) 
-        @created_at = Time.at(created.to_i)
+        redis.setnx key(:created_at), Time.now.to_i
+        @created_at = Time.at(redis.get(key(:created_at)).to_i)
         @identify_block = ->(context){ context.vanity_identity }
       end
 
@@ -31,26 +31,10 @@ module Vanity
       
       # Experiment creation timestamp.
       attr_reader :created_at
-     
-      # Sets or returns description. For example
-      #   experiment :simple do
-      #     description "Simple experiment"
-      #   end
-      #
-      #   puts "Just defined: " + experiment(:simple).description
-      def description(text = nil)
-        @description = text if text
-        @description
-      end
-
-      def report
-        fail "Implement me"
-      end
-      
-      # Called to save the experiment definition.
-      def save #:nodoc:
-      end
 
+      # Experiment completion timestamp.
+      attr_reader :completed_at
+     
       # Call this method with no argument or block to return an identity.  Call
       # this method with a block to define how to obtain an identity for the
       # current experiment.
@@ -80,6 +64,70 @@ module Vanity
         end
       end
 
+
+      # -- Reporting --
+
+      # Sets or returns description. For example
+      #   experiment :simple do
+      #     description "Simple experiment"
+      #   end
+      #
+      #   puts "Just defined: " + experiment(:simple).description
+      def description(text = nil)
+        @description = text if text
+        @description
+      end
+
+      def report
+        fail "Implement me"
+      end
+      
+
+      # -- Experiment completion --
+
+      # Define experiment completion condition.  For example:
+      #   complete_if do
+      #     alternatives.all? { |alt| alt.participants >= 100 } &&
+      #     alternatives.any? { |alt| alt.confidence >= 0.95 }
+      #   end
+      def complete_if(&block)
+        raise ArgumentError, "Missing block" unless block
+        raise "complete_if already called on this experiment" if @complete_block
+        @complete_block = block
+      end
+
+      # Derived classes call this after state changes that may lead to
+      # experiment completing.
+      def check_completion!
+        if @complete_block
+          begin
+            complete! if @complete_block.call
+          rescue
+            # TODO: logging
+          end
+        end
+      end
+      protected :check_completion!
+
+      # Force experiment to complete.
+      def complete!
+        redis.setnx key(:completed_at), Time.now.to_i
+        # TODO: logging
+      end
+
+      # Time stamp when experiment was completed.
+      def completed_at
+        Time.at(redis.get(key(:completed_at)).to_i)
+      end
+      
+      # Returns true if experiment active, false if completed.
+      def active?
+        redis.get(key(:completed_at)).nil?
+      end
+
+
+      # -- Store/validate --
+
       # Returns key for this experiment, or with an argument, return a key
       # using the experiment as the namespace.  Examples:
       #   key => "vanity:experiments:green_button"
@@ -92,6 +140,17 @@ module Vanity
       def redis #:nodoc:
         @playground.redis
       end
+      
+      # Called to save the experiment definition.
+      def save #:nodoc:
+      end
+
+      # Get rid of all experiment data.
+      def destroy
+        redis.del key(:created_at)
+        redis.del key(:completed_at)
+      end
+
     end
   end
 end

data/test/ab_test_test.rb

@@ -13,7 +13,7 @@ class AbTestController < ActionController::Base
   end
 
   def test_capture
-    render file: File.join(File.dirname(__FILE__), "ab_test_template.erb")
+    render inline: "<% ab_test :simple_ab do |value| %><%= value %><% end %>"
   end
 
   def goal
@@ -29,7 +29,8 @@ class AbTestTest < ActionController::TestCase
     experiment(:simple_ab) { }
   end
 
-  # Experiment definition
+
+  # --  Experiment definition --
 
   def uses_ab_test_when_type_is_ab_test
     experiment(:ab, type: :ab_test) { }
@@ -52,7 +53,8 @@ class AbTestTest < ActionController::TestCase
     end
   end
 
-  # Running experiment
+
+  # -- Running experiment --
 
   def returns_the_same_alternative_consistently
     experiment :foobar do
@@ -122,7 +124,7 @@ class AbTestTest < ActionController::TestCase
   end
 
 
-  # A/B helper methods
+  # -- A/B helper methods --
 
   def test_fail_if_no_experiment
     new_playground
@@ -167,7 +169,7 @@ class AbTestTest < ActionController::TestCase
   end
 
 
-  # Testing with tests
+  # -- Testing with tests --
   
   def test_with_given_choice
     100.times do
@@ -177,8 +179,8 @@ class AbTestTest < ActionController::TestCase
       post :goal
     end
     alts = experiment(:simple_ab).alternatives
-    assert_equal [100,0], alts.map { |alt| alt.participants }
-    assert_equal [100,0], alts.map { |alt| alt.conversions }
+    assert_equal [0,100], alts.map { |alt| alt.participants }
+    assert_equal [0,100], alts.map { |alt| alt.conversions }
   end
 
   def test_which_chooses_non_existent_alternative
@@ -187,4 +189,162 @@ class AbTestTest < ActionController::TestCase
     end
   end
 
+
+  # -- Z-score --
+  
+  def test_z_score
+    experiment :abcd do
+      alternatives :a, :b, :c, :d
+    end
+    alts = experiment(:abcd).alternatives
+    # participating, conversions, rate, z-score
+    # Control:      182	35 19.23%	N/A
+    182.times { |i| alts[0].participating!(i) }
+    35.times { |i| alts[0].conversion!(i) }
+    # Treatment A:  180	45 25.00%	1.33
+    180.times { |i| alts[1].participating!(i + 200) }
+    45.times { |i| alts[1].conversion!(i + 200) }
+    # Treatment B:  189	28 14.81%	-1.13
+    189.times { |i| alts[2].participating!(i + 400) }
+    28.times { |i| alts[2].conversion!(i + 400) }
+    # Treatment C:  188	61 32.45%	2.94
+    188.times { |i| alts[3].participating!(i + 600) }
+    61.times { |i| alts[3].conversion!(i + 600) }
+
+    z_scores = alts.map { |alt| sprintf("%4.2f", alt.z_score) }
+    assert_equal %w{0.00 1.33 -1.13 2.94}, z_scores
+
+    confidences = alts.map { |alt| alt.confidence }
+    assert_equal [0, 90, 0, 99], confidences
+  end
+  
+
+  # -- Completion --
+
+  def test_completion_if
+    experiment :simple do
+      identify { rand }
+      complete_if { true }
+    end
+    experiment(:simple).choose
+    refute experiment(:simple).active?
+  end
+
+  def test_completion_if_fails
+    experiment :simple do
+      identify { rand }
+      complete_if { fail }
+    end
+    experiment(:simple).choose
+    assert experiment(:simple).active?
+  end
+
+  def test_completion
+    ids = Array.new(100) { |i| i.to_s }.shuffle
+    experiment :simple do
+      identify { ids.pop }
+      complete_if { alternatives.map(&:participants).sum >= 100 }
+    end
+    99.times do |i|
+      experiment(:simple).choose
+      assert experiment(:simple).active?
+    end
+
+    experiment(:simple).choose
+    refute experiment(:simple).active?
+  end
+
+  def test_ab_methods_after_completion
+    ids = Array.new(200) { |i| i.to_s }.shuffle
+    test = self
+    experiment :simple do
+      identify { test.identity ||= ids.pop }
+      complete_if { alternatives.map(&:participants).sum >= 100 }
+      outcome_is { alternatives[1] }
+    end
+    # Run experiment to completion (100 participants)
+    results = Set.new
+    100.times do
+      test.identity = nil
+      results << experiment(:simple).choose
+      experiment(:simple).conversion!
+    end
+    assert results.include?(true) && results.include?(false)
+    refute experiment(:simple).active?
+
+    # Test that we always get the same choice (true)
+    100.times do
+      test.identity = nil
+      assert_equal true, experiment(:simple).choose
+      experiment(:simple).conversion!
+    end
+    # We don't get to count the 100 participant's conversion, but that's ok.
+    assert_equal 99, experiment(:simple).alternatives.map(&:converted).sum
+    assert_equal 99, experiment(:simple).alternatives.map(&:conversions).sum
+  end
+
+
+  # -- Outcome --
+  
+  def test_completion_outcome
+    experiment :quick do
+      outcome_is { alternatives[1] }
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+  end
+
+  def test_outcome_is_returns_nil
+    experiment :quick do
+      outcome_is { nil }
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_is_returns_something_else
+    experiment :quick do
+      outcome_is { "error" }
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_is_fails
+    experiment :quick do
+      outcome_is { fail }
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives.first, experiment(:quick).outcome
+  end
+
+  def test_outcome_choosing_best_alternative
+    experiment :quick do
+    end
+    2.times do |i|
+      experiment(:quick).alternatives[0].participating!(i)
+    end
+    10.times do |i|
+      experiment(:quick).alternatives[1].participating!(i)
+      experiment(:quick).alternatives[1].conversion!(i)
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+  end
+
+  def test_outcome_choosing_first_alternative
+    experiment :quick do
+    end
+    8.times do |i|
+      experiment(:quick).alternatives[0].participating!(i)
+      experiment(:quick).alternatives[0].conversion!(i)
+    end
+    7.times do |i|
+      experiment(:quick).alternatives[1].participating!(i)
+      experiment(:quick).alternatives[1].conversion!(i)
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives[0], experiment(:quick).outcome
+  end
+
 end

data/test/experiment_test.rb

@@ -27,11 +27,13 @@ class ExperimentTest < MiniTest::Spec
   end
 
   it "keeps creation timestamp across definitions" do
-    early = Time.now - 1.day
+    early, late = Time.now - 1.day, Time.now
     Time.expects(:now).once.returns(early)
     experiment(:simple) { }
     assert_equal early.to_i, experiment(:simple).created_at.to_i
+
     new_playground
+    Time.expects(:now).once.returns(late)
     experiment(:simple) { }
     assert_equal early.to_i, experiment(:simple).created_at.to_i
   end
@@ -42,4 +44,5 @@ class ExperimentTest < MiniTest::Spec
     end
     assert_equal "Simple experiment", experiment(:simple).description
   end
+
 end

data/vanity.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name           = "vanity"
-  spec.version        = "0.2.0"
+  spec.version        = "0.2.1"
   spec.author         = "Assaf Arkin"
   spec.email          = "assaf@labnotes.org"
   spec.homepage       = "http://github.com/assaf/vanity"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: vanity
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Assaf Arkin
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-10 00:00:00 -08:00
+date: 2009-11-11 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -40,7 +40,6 @@ files:
 - lib/vanity/rails.rb
 - lib/vanity/report.erb
 - lib/vanity.rb
-- test/ab_test_template.erb
 - test/ab_test_test.rb
 - test/experiment_test.rb
 - test/playground_test.rb
@@ -56,7 +55,7 @@ licenses: []
 post_install_message: 
 rdoc_options: 
 - --title
-- Vanity 0.2.0
+- Vanity 0.2.1
 - --main
 - README.rdoc
 - --webcvs

data/test/ab_test_template.erb

@@ -1,3 +0,0 @@
-<% ab_test :simple_ab do |value| %>
-  <%= value %>
-<% end %>