vanity 0.2.1 → 0.2.2

data/CHANGELOG

@@ -1,3 +1,12 @@
+0.2.2 (2009-11-12)
+* Added: vanity binary, with single command for generating a report.
+* Added: return alternative by value from experiment.alternative(val) method.
+* Added: reset an experiment by calling reset!.
+* Added: experiment alternative name (option 1, option 2, etc).
+* Added: new scoring algorithm: use experiment.score instead of
+alternative.z_score/confidence.
+* Added: experiment.conclusion for plain English results.
+
 0.2.1 (2009-11-11)
 * Added: z-score and confidence level for A/B test alternatives.
 * Added: test auto-completion and auto-outcome (complete_it, outcome_is).

data/bin/vanity

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+path = File.expand_path("../lib", File.dirname(__FILE__))
+$LOAD_PATH.unshift path unless $LOAD_PATH.include?(path)
+
+require "vanity"
+require "optparse"
+
+playground = Vanity.playground
+options = Struct.new(:output).new
+OptionParser.new("", 24, "  ") do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options]\n"
+
+  opts.separator ""
+  opts.separator "General options:"
+  opts.on("--path PATH", "Path to experiments directory (default: #{playground.load_path})")  { |v| playground.load_path = v }
+  opts.on("--output FILE", "Write report to this file (default: stdout)")                     { |v| options.output = v }
+
+  opts.separator ""
+  opts.separator "Redis options:"
+  opts.on("--host HOST", "Redis server host (default: #{playground.host})")       { |v| playground.host = v }
+  opts.on("--port PORT", "Redis server port (default: #{playground.port})")       { |v| playground.port = v }
+  opts.on("--db DB", "Redis database (default: #{playground.db})")                { |v| playground.db = v }
+  opts.on("--password PWD", "Redis database password")                            { |v| playground.password = v }
+  opts.on("--namespace NS", "Redis namespace (default: #{playground.namespace})") { |v| playground.namespace = v }
+
+  opts.separator ""
+  opts.separator "Common options:"
+  opts.on_tail "-h", "-H", "--help", "Show this message" do
+    puts opts.to_s.gsub(/^.*DEPRECATED.*$/s, '')
+    exit
+  end
+  opts.on_tail "-v", "--version", "Show version"  do
+    puts "Vanity #{Vanity::Version::STRING}"
+    exit
+  end
+end.parse!(ARGV)
+
+cmds = ARGV.empty? ? ["report"] : ARGV
+cmds.each do |cmd|
+  case cmd
+  when "report"
+    Vanity::Commands.report options.output
+  else fail "No such command: #{cmd}"
+  end
+end

data/lib/vanity/commands/report.rb

@@ -0,0 +1,27 @@
+require "erb"
+require "cgi"
+
+module Vanity
+  module Commands
+    class << self
+
+      # Generate a report with all available tests.  Outputs to the named file,
+      # or stdout with no arguments.
+      def report(output = nil)
+        require "erb"
+        erb = ERB.new(File.read("lib/vanity/report.erb"), nil, '<')
+        experiments = Vanity.playground.experiments
+        html = erb.result(binding)
+        if output
+          File.open output, 'w' do |file|
+            file.write html
+          end
+          puts "New report available in #{output}"
+        else
+          $stdout.write html
+        end
+      end
+
+    end
+  end
+end

data/lib/vanity/commands.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), "commands/report")

data/lib/vanity/experiment/ab_test.rb

@@ -7,12 +7,16 @@ module Vanity
       def initialize(experiment, id, value) #:nodoc:
         @experiment = experiment
         @id = id
+        @name = "option #{(@id + 1)}"
         @value = value
       end
 
       # Alternative id, only unique for this experiment.
       attr_reader :id
      
+      # Alternative name (option A, option B, etc).
+      attr_reader :name
+
       # Alternative value.
       attr_reader :value
 
@@ -28,12 +32,13 @@ module Vanity
 
       # Number of conversions for this alternative (same participant may be counted more than once).
       def conversions
-        redis.get(key("conversions")).to_i
+        redis[key("conversions")].to_i
       end
 
       # Conversion rate calculated as converted/participants.
       def conversion_rate
-        converted.to_f / participants.to_f
+        c, p = converted.to_f, participants.to_f
+        p > 0 ? c/p : 0.0
       end
 
       def <=>(other)
@@ -51,33 +56,20 @@ module Vanity
         end
       end
 
-      # Z-score this alternativet related to the base alternative.  This
-      # alternative is better than base if it receives a positive z-score,
-      # worse if z-score is negative.  Call #confident if you need confidence
-      # level (percentage).
-      def z_score
-        return 0 if base == self
-        pc = base.conversion_rate
-        nc = base.participants
-        p = conversion_rate
-        n = participants
-        (p - pc) / Math.sqrt((p * (1-p)/n) + (pc * (1-pc)/nc))
-      end
-
-      # How confident are we in this alternative being an improvement over the
-      # base alternative.  Returns 0, 90, 95, 99 or 99.9 (percentage).
-      def confidence
-        score = z_score
-        confidence = AbTest::Z_TO_CONFIDENCE.find { |z,p| score >= z }
-        confidence ? confidence.last : 0
-      end
-
       def destroy #:nodoc:
         redis.del key("participants")
         redis.del key("converted")
         redis.del key("conversions")
       end
 
+      def to_s #:nodoc:
+        name
+      end
+
+      def inspect #:nodoc:
+        "#{name}: #{value} #{converted}/#{participants}"
+      end
+
     protected
 
       def key(name)
@@ -97,6 +89,15 @@ module Vanity
 
     # The meat.
     class AbTest < Base
+      class << self
+
+        def confidence(score) #:nodoc:
+          score = score.abs
+          confidence = AbTest::Z_TO_CONFIDENCE.find { |z,p| score >= z }
+          confidence ? confidence.last : 0
+        end
+      end
+
       def initialize(*args) #:nodoc:
         super
       end
@@ -125,6 +126,11 @@ module Vanity
         alternatives
       end
 
+      # Returns an Alternative with the specified value.
+      def alternative(value)
+        alternatives.find { |alt| alt.value == value }
+      end
+
       # Sets this test to two alternatives: false and true.
       def false_true
         alternatives false, true
@@ -194,11 +200,84 @@ module Vanity
 
       # -- Reporting --
 
-      def report
-        alts = alternatives.map { |alt|
-          "<dt>Option #{(65 + alt.id).chr}</dt><dd><code>#{CGI.escape_html alt.value.inspect}</code> viewed #{alt.participants} times, converted #{alt.conversions}, rate #{alt.conversion_rate}, z_score #{alt.z_score}, confidence #{alt.confidence}<dd>"
-        }
-        %{<dl class="data">#{alts.join}</dl>}
+      # Returns an object with the following attributes:
+      # [:alts]  List of alternatives as structures (see below).
+      # [:best]  Best alternative.
+      # [:base]  Second best alternative.
+      # [:choice]  Choice alterntive, either selected outcome or best alternative (with confidence).
+      #
+      # Each alternative is an object with the following attributes:
+      # [:id]    Identifier.
+      # [:conv]  Conversion rate (0.0 to 1.0, rounded to 3 places).
+      # [:pop]   Population size (participants).
+      # [:diff]  Difference from least performant altenative (percentage).
+      # [:z]     Z-score compared to base (above).
+      # [:conf]  Confidence based on z-score (0, 90, 95, 99, 99.9).
+      def score
+        struct = Struct.new(:id, :conv, :pop, :diff, :z, :conf)
+        alts = alternatives.map { |alt| struct.new(alt.id, alt.conversion_rate.round(3), alt.participants) }
+        # sort by conversion rate to find second best and 2nd best
+        sorted = alts.sort_by(&:conv)
+        base = sorted[-2]
+        # calculate z-score
+        pc = base.conv
+        nc = base.pop
+        alts.each do |alt|
+          p = alt.conv
+          n = alt.pop
+          alt.z = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
+          alt.conf = AbTest.confidence(alt.z)
+        end
+        # difference is measured from least performant
+        if least = sorted.find { |alt| alt.conv > 0 }
+          alts.each do |alt|
+            alt.diff = (alt.conv - least.conv) / least.conv * 100 if alt.conv > least.conv
+          end
+        end
+        # best alternative is one with highest conversion rate (best shot).
+        # choice alternative can only pick best if we have high confidence (>90%).
+        best = sorted.last if sorted.last.conv > 0
+        choice = outcome ? alts[outcome.id] : (best && best.conf >= 90 ? best : nil)
+        Struct.new(:alts, :best, :base, :choice).new(alts, best, base, choice)
+      end
+
+      # Use the score returned by #score to derive a conclusion.  Returns an
+      # array of claims.
+      def conclusion(score = score)
+        claims = []
+        # find name form alt structure returned from score
+        name = ->(alt){ alternatives[alt.id].name }
+        # only interested in sorted alternatives with conversion
+        sorted = score.alts.select { |alt| alt.conv > 0.0 }.sort_by(&:conv).reverse
+        if sorted.size > 1
+          # start with alternatives that have conversion, from best to worst,
+          # then alternatives with no conversion.
+          sorted |= score.alts
+          # we want a result that's clearly better than 2nd best.
+          best, second = sorted[0], sorted[1]
+          if best.conv > second.conv
+            diff = ((best.conv - second.conv) / second.conv * 100).round
+            better = " (%d%% better than %s)" % [diff, name[second]] if diff > 0
+            claims << "The best choice is %s: it converted at %.1f%%%s." % [name[best], best.conv * 100, better]
+            if best.conf >= 90
+              claims << "With %d%% probability this result is statistically significant." % score.best.conf
+            else
+              claims << "This result is not statistically significant, suggest you continue this experiment."
+            end
+            sorted.delete best
+          end
+          sorted.each do |alt|
+            if alt.conv > 0.0
+              claims << "%s converted at %.1f%%." % [name[alt].capitalize, alt.conv * 100]
+            else
+              claims << "%s did not convert." % name[alt].capitalize
+            end
+          end
+        else
+          claims << "This experiment did not run long enough to find a clear winner."
+        end
+        claims << "#{name[score.choice].capitalize} selected as the best alternative." if score.choice
+        claims
       end
 
       def humanize
@@ -228,7 +307,7 @@ module Vanity
 
       # Alternative chosen when this experiment was completed.
       def outcome
-        outcome = redis.get(key("outcome"))
+        outcome = redis[key("outcome")]
         outcome && alternatives[outcome.to_i]
       end
 
@@ -242,8 +321,8 @@ module Vanity
           end
         end
         unless outcome
-          highest = alternatives.sort.last rescue nil
-          outcome = highest && highest.confidence >= 95 ? highest.id : 0
+          best = score.best
+          outcome = best.id if best
         end
         # TODO: logging
         redis.setnx key("outcome"), outcome
@@ -257,6 +336,12 @@ module Vanity
         super
       end
 
+      def reset! #:nodoc:
+        redis.del key(:outcome)
+        alternatives.each(&:destroy)
+        super
+      end
+
       def destroy #:nodoc:
         redis.del key(:outcome)
         alternatives.each(&:destroy)

data/lib/vanity/experiment/base.rb

@@ -19,7 +19,7 @@ module Vanity
         @id, @name = id.to_sym, name
         @namespace = "#{@playground.namespace}:#{@id}"
         redis.setnx key(:created_at), Time.now.to_i
-        @created_at = Time.at(redis.get(key(:created_at)).to_i)
+        @created_at = Time.at(redis[key(:created_at)].to_i)
         @identify_block = ->(context){ context.vanity_identity }
       end
 
@@ -34,6 +34,11 @@ module Vanity
 
       # Experiment completion timestamp.
       attr_reader :completed_at
+
+      # Returns the type of this class as a symbol (e.g. ab_test).
+      def type
+        self.class.type
+      end
      
       # Call this method with no argument or block to return an identity.  Call
       # this method with a block to define how to obtain an identity for the
@@ -117,12 +122,13 @@ module Vanity
 
       # Time stamp when experiment was completed.
       def completed_at
-        Time.at(redis.get(key(:completed_at)).to_i)
+        time = redis[key(:completed_at)]
+        time && Time.at(time.to_i)
       end
       
       # Returns true if experiment active, false if completed.
       def active?
-        redis.get(key(:completed_at)).nil?
+        redis[key(:completed_at)].nil?
       end
 
 
@@ -145,6 +151,13 @@ module Vanity
       def save #:nodoc:
       end
 
+      # Reset experiment.
+      def reset!
+        @created_at = Time.now
+        redis[key(:created_at)] = @created_at.to_i
+        redis.del key(:completed_at)
+      end
+
       # Get rid of all experiment data.
       def destroy
         redis.del key(:created_at)

data/lib/vanity/playground.rb

@@ -15,6 +15,7 @@ module Vanity
     # Created new Playground. Unless you need to, use the global Vanity.playground.
     def initialize
       @experiments = {}
+      @host, @port, @db = "127.0.0.1", 6379, 0
       @namespace = "vanity:#{Vanity::Version::MAJOR}"
       @load_path = "experiments"
     end

data/lib/vanity/report.erb

@@ -15,7 +15,7 @@
     <li class="experiment" id="experiment_<%= CGI.escape exp.id.to_s %>">
       <h3><%= CGI.escape_html exp.name %></h3>
       <blockquote><%= CGI.escape_html exp.description.to_s %></blockquote>
-      <%= exp.report %>
+      <%= exp.conclusion.join(" ") %>
       <p class="meta"><%= exp.humanize %> started <%= exp.created_at.strftime("%a, %b %-d %Y") %></p>
     </li>
   <% end %>

data/lib/vanity.rb

@@ -21,3 +21,4 @@ require File.join(File.dirname(__FILE__), "vanity/playground")
 require File.join(File.dirname(__FILE__), "vanity/experiment/base")
 require File.join(File.dirname(__FILE__), "vanity/experiment/ab_test")
 require File.join(File.dirname(__FILE__), "vanity/rails") if defined?(Rails)
+Vanity.autoload :Commands, File.join(File.dirname(__FILE__), "vanity/commands")

data/test/ab_test_test.rb

@@ -32,12 +32,12 @@ class AbTestTest < ActionController::TestCase
 
   # --  Experiment definition --
 
-  def uses_ab_test_when_type_is_ab_test
+  def test_uses_ab_test_when_type_is_ab_test
     experiment(:ab, type: :ab_test) { }
     assert_instance_of Vanity::Experiment::AbTest, experiment(:ab)
   end
 
-  def requires_at_least_two_alternatives_per_experiment
+  def test_requires_at_least_two_alternatives_per_experiment
     assert_raises RuntimeError do
       experiment :none, type: :ab_test do
         alternatives []
@@ -52,11 +52,27 @@ class AbTestTest < ActionController::TestCase
       alternatives "foo", "bar"
     end
   end
+  
+  def test_returning_alternative_by_value
+    experiment :abcd do
+      alternatives :a, :b, :c, :d
+    end
+    assert_equal experiment(:abcd).alternatives[1], experiment(:abcd).alternative(:b)
+    assert_equal experiment(:abcd).alternatives[3], experiment(:abcd).alternative(:d)
+  end
+
+  def test_alternative_name
+    experiment :abcd do
+      alternatives :a, :b
+    end
+    assert_equal "option 1", experiment(:abcd).alternative(:a).name
+    assert_equal "option 2", experiment(:abcd).alternative(:b).name
+  end
 
 
   # -- Running experiment --
 
-  def returns_the_same_alternative_consistently
+  def test_returns_the_same_alternative_consistently
     experiment :foobar do
       alternatives "foo", "bar"
       identify { "6e98ec" }
@@ -68,7 +84,7 @@ class AbTestTest < ActionController::TestCase
     end
   end
 
-  def returns_different_alternatives_for_each_participant
+  def test_returns_different_alternatives_for_each_participant
     experiment :foobar do
       alternatives "foo", "bar"
       identify { rand(1000).to_s }
@@ -78,7 +94,7 @@ class AbTestTest < ActionController::TestCase
     assert_in_delta alts.select { |a| a == "foo" }.count, 500, 100 # this may fail, such is propability
   end
 
-  def records_all_participants_in_each_alternative
+  def test_records_all_participants_in_each_alternative
     ids = (Array.new(200) { |i| i.to_s } * 5).shuffle
     experiment :foobar do
       alternatives "foo", "bar"
@@ -90,7 +106,7 @@ class AbTestTest < ActionController::TestCase
     assert_in_delta alts.first.participants, 100, 20
   end
 
-  def records_each_converted_participant_only_once
+  def test_records_each_converted_participant_only_once
     ids = (Array.new(100) { |i| i.to_s } * 5).shuffle
     test = self
     experiment :foobar do
@@ -123,6 +139,26 @@ class AbTestTest < ActionController::TestCase
     assert_equal 100, alts.inject(0) { |t,a| t + a.converted }
   end
 
+  def test_reset_experiment
+    experiment :simple do
+      identify { "me" }
+      complete_if { alternatives.map(&:converted).sum >= 1 }
+      outcome_is { alternative(true) }
+    end
+    experiment(:simple).choose
+    experiment(:simple).conversion!
+    refute experiment(:simple).active?
+    assert_equal true, experiment(:simple).outcome.value
+
+    experiment(:simple).reset!
+    assert experiment(:simple).active?
+    assert_nil experiment(:simple).outcome
+    assert_nil experiment(:simple).completed_at
+    assert_equal 0, experiment(:simple).alternatives.map(&:participants).sum
+    assert_equal 0, experiment(:simple).alternatives.map(&:conversions).sum
+    assert_equal 0, experiment(:simple).alternatives.map(&:converted).sum
+  end
+
 
   # -- A/B helper methods --
 
@@ -190,34 +226,187 @@ class AbTestTest < ActionController::TestCase
   end
 
 
-  # -- Z-score --
+  # -- Scoring --
   
-  def test_z_score
-    experiment :abcd do
-      alternatives :a, :b, :c, :d
-    end
-    alts = experiment(:abcd).alternatives
+  def test_scoring
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
     # participating, conversions, rate, z-score
     # Control:      182	35 19.23%	N/A
-    182.times { |i| alts[0].participating!(i) }
-    35.times { |i| alts[0].conversion!(i) }
+    182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
+    35.times  { |i| experiment(:abcd).alternative(:a).conversion!(i) }
     # Treatment A:  180	45 25.00%	1.33
-    180.times { |i| alts[1].participating!(i + 200) }
-    45.times { |i| alts[1].conversion!(i + 200) }
-    # Treatment B:  189	28 14.81%	-1.13
-    189.times { |i| alts[2].participating!(i + 400) }
-    28.times { |i| alts[2].conversion!(i + 400) }
-    # Treatment C:  188	61 32.45%	2.94
-    188.times { |i| alts[3].participating!(i + 600) }
-    61.times { |i| alts[3].conversion!(i + 600) }
+    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment B:  189	28 14.81%	-1.13
+    189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
+    28.times  { |i| experiment(:abcd).alternative(:c).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
+    assert_equal %w{-1.33 0.00 -2.47 1.58}, z_scores
+    confidences = experiment(:abcd).score.alts.map(&:conf)
+    assert_equal [90, 0, 99, 90], confidences
+
+    diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
+    assert_equal [30, 69, nil, 119], diff
+    assert_equal 3, experiment(:abcd).score.best.id
+    assert_equal 3, experiment(:abcd).score.choice.id
+  end
 
-    z_scores = alts.map { |alt| sprintf("%4.2f", alt.z_score) }
-    assert_equal %w{0.00 1.33 -1.13 2.94}, z_scores
+  def test_scoring_with_no_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
+    assert_nil experiment(:abcd).score.best
+    assert_nil experiment(:abcd).score.choice
+  end
 
-    confidences = alts.map { |alt| alt.confidence }
-    assert_equal [0, 90, 0, 99], confidences
+  def test_scoring_with_one_performer
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    8.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.z.nan? }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.conf == 0 }
+    assert experiment(:abcd).score.alts.all? { |alt| alt.diff.nil? }
+    assert 1, experiment(:abcd).score.best.id
+    assert_nil experiment(:abcd).score.choice
+  end
+
+  def test_scoring_with_some_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    10.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    8.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    12.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    5.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z }
+    assert_equal %w{NaN 2.01 NaN 0.00}, z_scores
+    confidences = experiment(:abcd).score.alts.map(&:conf)
+    assert_equal [0, 95, 0, 0], confidences
+    diff = experiment(:abcd).score.alts.map { |alt| alt.diff && alt.diff.round }
+    assert_equal [nil, 92, nil, nil], diff
+    assert_equal 1, experiment(:abcd).score.best.id
+    assert_equal 1, experiment(:abcd).score.choice.id
+  end
+
+
+  # -- Conclusion --
+
+  def test_conclusion
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # participating, conversions, rate, z-score
+    # Control:      182	35 19.23%	N/A
+    182.times { |i| experiment(:abcd).alternative(:a).participating!(i) }
+    35.times  { |i| experiment(:abcd).alternative(:a).conversion!(i) }
+    # Treatment A:  180	45 25.00%	1.33
+    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment B:  189	28 14.81%	-1.13
+    189.times { |i| experiment(:abcd).alternative(:c).participating!(i) }
+    28.times  { |i| experiment(:abcd).alternative(:c).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+The best choice is option 4: it converted at 32.4% (30% better than option 2).
+With 90% probability this result is statistically significant.
+Option 2 converted at 25.0%.
+Option 1 converted at 19.2%.
+Option 3 converted at 14.8%.
+Option 4 selected as the best alternative.
+    TEXT
   end
-  
+
+  def test_conclusion_with_some_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # Treatment A:  180	45 25.00%	1.33
+    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+The best choice is option 4: it converted at 32.4% (30% better than option 2).
+With 90% probability this result is statistically significant.
+Option 2 converted at 25.0%.
+Option 1 did not convert.
+Option 3 did not convert.
+Option 4 selected as the best alternative.
+    TEXT
+  end
+
+  def test_conclusion_without_clear_winner
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # Treatment A:  180	45 25.00%	1.33
+    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    58.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+The best choice is option 4: it converted at 32.4% (1% better than option 2).
+This result is not statistically significant, suggest you continue this experiment.
+Option 2 converted at 32.2%.
+Option 1 did not convert.
+Option 3 did not convert.
+    TEXT
+  end
+
+  def test_conclusion_without_close_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # Treatment A:  180	45 25.00%	1.33
+    186.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    60.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+The best choice is option 4: it converted at 32.4%.
+This result is not statistically significant, suggest you continue this experiment.
+Option 2 converted at 32.3%.
+Option 1 did not convert.
+Option 3 did not convert.
+    TEXT
+  end
+
+  def test_conclusion_without_equal_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # Treatment A:  180	45 25.00%	1.33
+    188.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+    # treatment C:  188	61 32.45%	2.94
+    188.times { |i| experiment(:abcd).alternative(:d).participating!(i) }
+    61.times  { |i| experiment(:abcd).alternative(:d).conversion!(i) }
+
+    assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
+Option 4 converted at 32.4%.
+Option 2 converted at 32.4%.
+Option 1 did not convert.
+Option 3 did not convert.
+    TEXT
+  end
+
+  def test_conclusion_with_one_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    # Treatment A:  180	45 25.00%	1.33
+    180.times { |i| experiment(:abcd).alternative(:b).participating!(i) }
+    45.times  { |i| experiment(:abcd).alternative(:b).conversion!(i) }
+
+    assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
+  end
+
+  def test_conclusion_with_no_performers
+    experiment(:abcd) { alternatives :a, :b, :c, :d }
+    assert_equal "This experiment did not run long enough to find a clear winner.", experiment(:abcd).conclusion.join("\n")
+  end
+
 
   # -- Completion --
 
@@ -332,19 +521,30 @@ class AbTestTest < ActionController::TestCase
     assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
   end
 
-  def test_outcome_choosing_first_alternative
+  def test_outcome_only_performing_alternative
+    experiment :quick do
+    end
+    2.times do |i|
+      experiment(:quick).alternatives[1].participating!(i)
+      experiment(:quick).alternatives[1].conversion!(i)
+    end
+    experiment(:quick).complete!
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
+  end
+
+  def test_outcome_choosing_equal_alternatives
     experiment :quick do
     end
     8.times do |i|
       experiment(:quick).alternatives[0].participating!(i)
       experiment(:quick).alternatives[0].conversion!(i)
     end
-    7.times do |i|
+    8.times do |i|
       experiment(:quick).alternatives[1].participating!(i)
       experiment(:quick).alternatives[1].conversion!(i)
     end
     experiment(:quick).complete!
-    assert_equal experiment(:quick).alternatives[0], experiment(:quick).outcome
+    assert_equal experiment(:quick).alternatives[1], experiment(:quick).outcome
   end
 
 end

data/test/experiments/null_abc.rb

@@ -0,0 +1,4 @@
+experiment "Null/ABC" do
+  description "Testing A, B, C alternatives against current feature (0)"
+  alternatives 0, :a, :b, :c
+end

data/vanity.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name           = "vanity"
-  spec.version        = "0.2.1"
+  spec.version        = "0.2.2"
   spec.author         = "Assaf Arkin"
   spec.email          = "assaf@labnotes.org"
   spec.homepage       = "http://github.com/assaf/vanity"
@@ -9,6 +9,7 @@ Gem::Specification.new do |spec|
   #spec.post_install_message = "To get started run vanity --help"
 
   spec.files          = Dir["{bin,lib,rails,test}/**/*", "CHANGELOG", "README.rdoc", "vanity.gemspec"]
+  spec.executable     = "vanity"
 
   spec.has_rdoc         = true
   spec.extra_rdoc_files = "README.rdoc", "CHANGELOG"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: vanity
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors: 
 - Assaf Arkin
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-11 00:00:00 -08:00
+date: 2009-11-12 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -24,14 +24,17 @@ dependencies:
     version: 
 description: ""
 email: assaf@labnotes.org
-executables: []
-
+executables: 
+- vanity
 extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
 - CHANGELOG
 files: 
+- bin/vanity
+- lib/vanity/commands/report.rb
+- lib/vanity/commands.rb
 - lib/vanity/experiment/ab_test.rb
 - lib/vanity/experiment/base.rb
 - lib/vanity/playground.rb
@@ -42,6 +45,7 @@ files:
 - lib/vanity.rb
 - test/ab_test_test.rb
 - test/experiment_test.rb
+- test/experiments/null_abc.rb
 - test/playground_test.rb
 - test/rails_test.rb
 - test/test_helper.rb
@@ -55,7 +59,7 @@ licenses: []
 post_install_message: 
 rdoc_options: 
 - --title
-- Vanity 0.2.1
+- Vanity 0.2.2
 - --main
 - README.rdoc
 - --webcvs