vanity 1.0.0 → 1.1.0

data/CHANGELOG

@@ -1,3 +1,38 @@
+== 1.1.0 (2009-12-4)
+This release introduces metrics.  Metrics are the gateway drug to better software.
+
+It’s as simple as defining a metric:
+
+	metric "Cheers" do
+		description "They love us, don't they?"
+	end
+
+Tracking it from your code:
+
+	track! :cheers
+
+And watching the graph from the Dashboard.
+
+You can (should) also use metrics with your A/B tests, for example:
+
+  ab_test "Pricing options" do
+	  metrics :signup
+		alternatives 15, 25, 29
+	end
+
+This new usage may become requirement in a future release.
+
+Much thanks to Ian Sefferman for fixing issues with Ruby 1.8.7 and Rails support.
+
+* Added: Metrics.
+* Added: Use Vanity.playground.mock! when running tests and you'd rather not access a live Redis server.
+* Changed: A/B tests now using metrics for tracking.
+* Changed: Now throwing NameError instead of LoadError when failing to load experiment/metric.  NameError can be rescued on same line. 
+* Changed: New, easier URL mapping for Dashboard: map.vanity "/vanity", :controller=>:vanity.
+* Changed: All tests are green on Ruby 1.8.6, 1.8.7 and 1.9.1.
+* Changed: Switched to redis-rb from http://github.com/ezmobius/redis-rb.
+* Deprecated: Please call experiment method with experiment identifier (a symbol) and not experiment name.
+
 == 1.0.0 (2009-11-19)
 This release changes the way you define a new experiment.  You can use a method suitable for the type of experiment you want to define, or call the generic define method (previously: experiment method).  For example:
 

data/README.rdoc

@@ -1,13 +1,13 @@
 Vanity is an Experiment Driven Development framework for Rails.
 
-* Documentation: http://vanity.labnotes.org
+* All about Vanity: http://vanity.labnotes.org
 * On github: http://github.com/assaf/vanity
-* Vanity requires Ruby 1.9.1 or later, Redis 1.0 or later.
+* Vanity requires Redis 1.0 or later.
 
 http://farm3.static.flickr.com/2540/4099665871_497f274f68_o.jpg
 
 
-== A/B Testing with Rails (in 5 easy steps)
+== A/B Testing With Rails (In 5 Easy Steps)
 
 <b>Step 1:</b> Start using Vanity in your Rails application:
 
@@ -24,6 +24,7 @@ And:
   ab_test "Price options" do
     description "Mirror, mirror on the wall, who's the better price of all?"
     alternatives 19, 25, 29
+    metrics :signups
   end
 
 <b>Step 3:</b> Present the different options to your users:
@@ -36,7 +37,7 @@ And:
     def signup
       @account = Account.new(params[:account])
       if @account.save
-        track! :pricing_options # <- here be conversion!
+        track! :signups
         redirect_to @acccount
       else
         render action: :offer
@@ -49,8 +50,34 @@ And:
   vanity --output vanity.html
 
 
+== Building From Source
+
+To run the test suite for the first time:
+
+  $ gem install rails mocha timecop
+  $ rake
+
+You can also +rake test+ if you insist on being explicit.
+
+To build the documentation:
+ 
+  $ gem install yardoc jekyll
+  $ rake docs
+  $ open html/index.html
+
+To clean up after yourself:
+
+  $ rake clobber
+
+To package Vanity as a gem and install on your machine:
+
+  $ rake install
+
+
 == Credits/License
 
-Idea behind Experiment Driven Development: Nathaniel Talbott (http://blog.talbott.ws).
+Original code, copyright of Assaf Arkin, released under the MIT license.  
+
+Documentation available under the Creative Commons Attribution license.
 
-Copyright (C) 2009 Assaf Arkin, released under the MIT license.
+For full list of credits and licenses: http://vanity.labnotes.org/credits.html.

data/lib/vanity.rb

@@ -1,11 +1,14 @@
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "../vendor/redis-0.1/lib")
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "../vendor/redis-rb/lib")
 require "redis"
 require "openssl"
+require "date"
+require "logger"
 
-# All the cool stuff happens in other places:
-# - Vanity::Helpers
-# - Vanity::Playground
-# - Experiment::AbTest
+# All the cool stuff happens in other places.
+# @see Vanity::Rails
+# @see Vanity::Playground
+# @see Vanity::Metric
+# @see Vanity::Experiment
 module Vanity
   # Version number.
   module Version
@@ -17,8 +20,11 @@ module Vanity
   end
 end
 
-require "vanity/playground"
+require "vanity/backport" if RUBY_VERSION < "1.9"
+require "vanity/metric"
 require "vanity/experiment/base"
 require "vanity/experiment/ab_test"
-require "vanity/rails" if defined?(Rails)
+require "vanity/playground"
+Vanity.autoload :MockRedis, "vanity/mock_redis"
 Vanity.autoload :Commands, "vanity/commands"
+require "vanity/rails" if defined?(Rails)

data/lib/vanity/backport.rb

@@ -0,0 +1,43 @@
+class Time
+  unless method_defined?(:to_date)
+    # Backported from Ruby 1.9.
+    def to_date
+      jd = Date.__send__(:civil_to_jd, year, mon, mday, Date::ITALY)
+      Date.new!(Date.__send__(:jd_to_ajd, jd, 0, 0), 0, Date::ITALY)
+    end
+  end
+end
+
+class Date
+  unless method_defined?(:to_date)
+    # Backported from Ruby 1.9.
+    def to_date
+      self
+    end
+  end
+
+  unless method_defined?(:to_time)
+    # Backported from Ruby 1.9.
+    def to_time
+      Time.local(year, mon, mday) 
+    end
+  end
+end
+
+class Symbol
+  unless method_defined?(:to_proc)
+    # Backported from Ruby 1.9.
+    def to_proc
+      Proc.new { |*args| args.shift.__send__(self, *args) }
+    end
+  end
+end
+
+class Array
+  unless method_defined?(:minmax)
+    # Backported from Ruby 1.9.
+    def minmax
+      [min, max]
+    end
+  end
+end 

data/lib/vanity/commands/report.rb

@@ -15,15 +15,25 @@ module Vanity
       struct.send :include, Render
       locals = struct.new(*locals.values_at(*keys))
       dir, base = File.split(path)
-      path = File.read(File.join(dir, "_#{base}"))
-      ERB.new(path, nil, '<').result(locals.instance_eval { binding })
+      path = File.join(dir, "_#{base}")
+      erb = ERB.new(File.read(path), nil, '<>')
+      erb.filename = path
+      erb.result(locals.instance_eval { binding })
     end
 
     # Escape HTML.
     def h(html)
-      CGI.escape_html(html)
+      CGI.escapeHTML(html)
     end
 
+    # Dumbed down from Rails' simple_format.
+    def simple_format(text, options={})
+      open = "<p #{options.map { |k,v| "#{k}=\"#{CGI.escapeHTML v}\"" }.join(" ")}>"
+      text = open + text.gsub(/\r\n?/, "\n").   # \r\n and \r -> \n
+        gsub(/\n\n+/, "</p>\n\n#{open}").       # 2+ newline  -> paragraph
+        gsub(/([^\n]\n)(?=[^\n])/, '\1<br />') + # 1 newline   -> br
+        "</p>"
+    end
   end
 
   # Commands available when running Vanity from the command line (see bin/vanity).

data/lib/vanity/experiment/ab_test.rb

@@ -44,7 +44,7 @@ module Vanity
 
       # Conversion rate calculated as converted/participants, rounded to 3 places.
       def conversion_rate
-        @conversion_rate ||= (participants > 0 ? (converted.to_f/participants.to_f).round(3) : 0.0)
+        @conversion_rate ||= (participants > 0 ? (converted.to_f/participants.to_f * 1000).round / 1000.0 : 0.0)
       end
 
       # The measure we use to order (sort) alternatives and decide which one is better (by calculating z-score).
@@ -94,19 +94,39 @@ module Vanity
         @alternatives = [false, true]
       end
 
+
+      # -- Metric --
+    
+      # Tells A/B test which metric we're measuring, or returns metric in use. 
+      #
+      # @example Define A/B test against coolness metric
+      #   ab_test "Background color" do
+      #     metrics :coolness
+      #     alternatives "red", "blue", "orange"
+      #   end
+      # @example Find metric for A/B test
+      #   puts "Measures: " + experiment(:background_color).metrics.map(&:name)
+      def metrics(*args)
+        @metrics = args.map { |id| @playground.metric(id) } unless args.empty?
+        @metrics
+      end
+
+
       # -- Alternatives --
 
-      # Call this method once to set alternative values for this experiment.
-      # Require at least two values.  For example:
+      # Call this method once to set alternative values for this experiment
+      # (requires at least two values).  Call without arguments to obtain
+      # current list of alternatives.
+      #
+      # @example Define A/B test with three alternatives
       #   ab_test "Background color" do
+      #     metrics :coolness
       #     alternatives "red", "blue", "orange"
       #   end
       # 
-      # Call without arguments to obtain current list of alternatives.  For example:
+      # @example Find out which alternatives this test uses
       #   alts = experiment(:background_color).alternatives
       #   puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
-      #
-      # If you want to know how well each alternative is doing, use #score.
       def alternatives(*args)
         unless args.empty?
           @alternatives = args.clone
@@ -129,24 +149,25 @@ module Vanity
       end
       private :_alternatives
 
-      # Returns an Alternative with the specified value.  For example, given:
-      #   ab_test "Which color" do
-      #     alternatives :red, :green, :blue
-      #   end
-      # Then:
+      # Returns an Alternative with the specified value.
+      #
+      # @example
       #   alternative(:red) == alternatives[0]
       #   alternative(:blue) == alternatives[2]
       def alternative(value)
         alternatives.find { |alt| alt.value == value }
       end
 
-      # Defines an A/B test with two alternatives: false and true.  For example:
+      # Defines an A/B test with two alternatives: false and true.  This is the
+      # default pair of alternatives, so just syntactic sugar for those who love
+      # being explicit.
+      #
+      # @example
       #   ab_test "More bacon" do
+      #     metrics :yummyness 
       #     false_true
       #   end
       #
-      # This is the default pair of alternatives, so just syntactic sugar for
-      # those who love being explicit.
       def false_true
         alternatives false, true
       end
@@ -160,7 +181,7 @@ module Vanity
       # alternative for the same identity, and randomly split alternatives
       # between different identities.
       #
-      # For example:
+      # @example
       #   color = experiment(:which_blue).choose
       def choose
         if active?
@@ -177,38 +198,22 @@ module Vanity
         @alternatives[index.to_i]
       end
 
-      # Tracks a conversion.  You probably want to use the Rails helper method
-      # track! instead.
-      # 
-      # For example:
-      #   experiment(:which_blue).track!
-      def track!
-        return unless active?
-        identity = identity()
-        return if redis[key("participants:#{identity}:show")]
-        index = alternative_for(identity)
-        if redis.sismember(key("alts:#{index}:participants"), identity)
-          redis.sadd key("alts:#{index}:converted"), identity
-          redis.incr key("alts:#{index}:conversions")
-        end
-        check_completion!
-      end
-
       
       # -- Testing --
      
       # Forces this experiment to use a particular alternative.  You'll want to
       # use this from your test cases to test for the different alternatives.
-      # For example:
+      #
+      # @example Setup test to red button
       #   setup do
-      #     experiment(:green_button).select(true)
+      #     experiment(:button_color).select(:red)
       #   end
       #
-      #   def test_shows_green_button
+      #   def test_shows_red_button
       #     . . .
       #   end
       #
-      # Use nil to clear out selection:
+      # @example Use nil to clear selection
       #   teardown do
       #     experiment(:green_button).select(nil)
       #   end
@@ -324,7 +329,7 @@ module Vanity
       # -- Completion --
 
       # Defines how the experiment can choose the optimal outcome on completion.
-     
+      #
       # By default, Vanity will take the best alternative (highest conversion
       # rate) and use that as the outcome.  You experiment may have different
       # needs, maybe you want the least performing alternative, or factor cost
@@ -370,7 +375,7 @@ module Vanity
       # -- Store/validate --
 
       def destroy
-        @alternatives.count.times do |i|
+        @alternatives.size.times do |i|
           redis.del key("alts:#{i}:participants")
           redis.del key("alts:#{i}:converted")
           redis.del key("alts:#{i}:conversions")
@@ -380,55 +385,82 @@ module Vanity
       end
 
       def save
-        fail "Experiment #{name} needs at least two alternatives" unless alternatives.count >= 2
+        fail "Experiment #{name} needs at least two alternatives" unless alternatives.size >= 2
         super
+        if @metrics.nil? || @metrics.empty?
+          warn "Please use metrics method to explicitly state which metric you are measuring against."
+          metric = @playground.metrics[id] ||= Vanity::Metric.new(@playground, name)
+          @metrics = [metric]
+        end
+        @metrics.each do |metric|
+          metric.hook &method(:track!)
+        end
+      end
+
+      # Called when tracking associated metric.
+      def track!(metric_id, timestamp, count, *args)
+        return unless active?
+        identity = identity()
+        return if redis[key("participants:#{identity}:show")]
+        index = alternative_for(identity)
+        redis.sadd key("alts:#{index}:converted"), identity if redis.sismember(key("alts:#{index}:participants"), identity)
+        redis.incrby key("alts:#{index}:conversions"), count
+        check_completion!
       end
 
+      # If you are not embarrassed by the first version of your product, you’ve
+      # launched too late.
+      #   -- Reid Hoffman, founder of LinkedIn
+
     protected
 
+      # Used for testing.
+      def fake(values)
+        values.each do |value, (participants, conversions)|
+          conversions ||= participants
+          participants.times do |identity|
+            index = @alternatives.index(value)
+            raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
+            redis.sadd key("alts:#{index}:participants"), identity
+          end
+          conversions.times do |identity|
+            index = @alternatives.index(value)
+            raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
+            redis.sadd key("alts:#{index}:converted"), identity
+            redis.incr key("alts:#{index}:conversions")
+          end
+        end
+      end
+
       # Chooses an alternative for the identity and returns its index. This
       # method always returns the same alternative for a given experiment and
       # identity, and randomly distributed alternatives for each identity (in the
       # same experiment).
       def alternative_for(identity)
-        Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.count
-      end
-
-      # Used for testing Vanity.
-      def count_participant(identity, value)
-        index = @alternatives.index(value)
-        raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
-        redis.sadd key("alts:#{index}:participants"), identity
-        self
-      end
-
-      # Used for testing Vanity.
-      def count_conversion(identity, value)
-        index = @alternatives.index(value)
-        raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
-        redis.sadd key("alts:#{index}:converted"), identity
-        redis.incr key("alts:#{index}:conversions")
-        self
+        Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
       end
 
       begin
         a = 50.0
         # Returns array of [z-score, percentage]
-        norm_dist = (0.0..3.1).step(0.01).map { |x| [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
+        norm_dist = []
+        (0.0..3.1).step(0.01) { |x| norm_dist << [x, a += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
         # We're really only interested in 90%, 95%, 99% and 99.9%.
         Z_TO_PROBABILITY = [90, 95, 99, 99.9].map { |pct| [norm_dist.find { |x,a| a >= pct }.first, pct] }.reverse
       end
 
     end
-  end
 
-  module Definition
-    # Define an A/B test with the given name.  For example:
-    #   ab_test "New Banner" do
-    #     alternatives :red, :green, :blue
-    #   end
-    def ab_test(name, &block)
-      define name, :ab_test, &block
+
+    module Definition
+      # Define an A/B test with the given name.  For example:
+      #   ab_test "New Banner" do
+      #     alternatives :red, :green, :blue
+      #   end
+      def ab_test(name, &block)
+        define name, :ab_test, &block
+      end
     end
+
   end
 end