vanity 1.3.0 → 1.4.0.beta

data/lib/vanity/metric/google_analytics.rb

@@ -51,6 +51,13 @@ module Vanity
         @ga_resource
       end
 
+      # Unkown (for now).
+      def last_update_at
+      end
+
+      def track!(args = nil)
+      end
+
       class Resource
         # GA profile used for this report.  Populated after calling results. 
         attr_reader :profile

data/lib/vanity/metric/remote.rb

@@ -0,0 +1,53 @@
+require "net/http"
+require "cgi"
+
+module Vanity
+  class Metric
+
+    # Specifies the base URL to use for a remote metric. For example:
+    #   metric :sandbox do
+    #     remote "http://api.vanitydash.com/metrics/sandbox"
+    #   end
+    def remote(url = nil)
+      @remote_url = URI.parse(url) if url
+      @mutex ||= Mutex.new
+      extend Remote
+      @remote_url
+    end
+
+    # To update a remote metric, make a POST request to the metric URL with the
+    # content type "application/x-www-form-urlencoded" and the following
+    # fields:
+    # - The +metric+ identifier,
+    # - The +timestamp+ must be RFC 2616 formatted (in Ruby just call +httpdate+
+    #   on the Time object),
+    # - The +identity+ (optional),
+    # - Pass consecutive values using the field +values[]+, or
+    # - Set values by their index using +values[0]+, +values[1]+, etc or
+    # - Set values by series name using +values[foo]+, +values[bar]+, etc.
+    module Remote
+
+      def track!(args = nil)
+        return unless @playground.collecting?
+        timestamp, identity, values = track_args(args)
+        params = ["metric=#{CGI.escape @id.to_s}", "timestamp=#{CGI.escape timestamp.httpdate}"]
+        params << "identity=#{CGI.escape identity.to_s}" if identity
+        params.concat values.map { |v| "values[]=#{v.to_i}" }
+        params << @remote_url.query if @remote_url.query
+        @mutex.synchronize do
+          @http ||= Net::HTTP.start(@remote_url.host, @remote_url.port)
+          @http.request Net::HTTP::Post.new(@remote_url.path, "Content-Type"=>"application/x-www-form-urlencoded"), params.join("&")
+        end
+      rescue Timeout::Error, StandardError
+        @playground.logger.error "Error sending data for metric #{name}: #{$!}"
+        @http = nil
+      ensure
+        call_hooks timestamp, identity, values
+      end
+
+      # "Don't worry, be crappy. Revolutionary means you ship and then test."
+      # -- Guy Kawazaki
+
+    end
+  end
+end

data/lib/vanity/playground.rb

@@ -1,3 +1,5 @@
+require "uri"
+
 module Vanity
 
   # Playground catalogs all your experiments, holds the Vanity configuration.
@@ -7,11 +9,7 @@ module Vanity
   #   puts Vanity.playground.map(&:name)
   class Playground
 
-    DEFAULTS = {
-      :connection=>"localhost:6379",
-      :load_path=>"experiments",
-      :namespace=>"vanity:#{Vanity::Version::MAJOR}"
-    }
+    DEFAULTS = { :load_path=>"experiments" }
 
     # Created new Playground. Unless you need to, use the global
     # Vanity.playground.
@@ -26,22 +24,23 @@ module Vanity
       options = args.pop if Hash === args.last
       @options = DEFAULTS.merge(options || {})
       if @options.values_at(:host, :port, :db).any?
-        warn "Deprecated: please specify Redis connection as single argument (\"host:port\")"
-        @connection_spec = "#{@options[:host]}:#{@options[:port]}:#{@options[:db]}"
+        warn "Deprecated: please specify Redis connection as URL (\"redis://host:port/db\")"
+        establish_connection :adapter=>"redis", :host=>options[:host], :port=>options[:port], :database=>options[:db]
       elsif @options[:redis]
-        @redis = @options[:redis]
+        @adapter = RedisAdapter.new(:redis=>@options[:redis])
       else
-        @connection_spec = args.shift || @options[:connection]
+        connection_spec = args.shift || @options[:connection]
+        establish_connection "redis://" + connection_spec if connection_spec
       end
 
-      @namespace = @options[:namespace] || DEFAULTS[:namespace]
+      warn "Deprecated: namespace option no longer supported directly" if @options[:namespace]
       @load_path = @options[:load_path] || DEFAULTS[:load_path]
-      @logger = @options[:logger]
-      unless @logger
+      unless @logger = @options[:logger]
         @logger = Logger.new(STDOUT)
         @logger.level = Logger::ERROR
       end
       @loading = []
+      @collecting = true
     end
    
     # Deprecated. Use redis.server instead.
@@ -115,6 +114,22 @@ module Vanity
       metrics[id.to_sym] or raise NameError, "No metric #{id}"
     end
 
+    # True if collection data (metrics and experiments). You only want to
+    # collect data in production environment, everywhere else run with
+    # collection off.
+    #
+    # @since 1.4.0
+    def collecting?
+      @collecting
+    end
+
+    # Turns data collection on and off.
+    #
+    # @since 1.4.0
+    def collecting=(enabled)
+      @collecting = !!enabled
+    end
+
     # Returns hash of metrics (key is metric id).
     #
     # @see Vanity::Metric
@@ -126,6 +141,14 @@ module Vanity
         Dir[File.join(load_path, "metrics/*.rb")].each do |file|
           Metric.load self, @loading, file
         end
+        if File.exist?("config/vanity.yml") && remote = YAML.load_file("config/vanity.yml")["metrics"]
+          remote.each do |id, url|
+            fail "Metric #{id} already defined in playground" if metrics[id.to_sym]
+            metric = Metric.new(self, id)
+            metric.remote url
+            metrics[id.to_sym] = metric
+          end
+        end
       end
       @metrics
     end
@@ -142,61 +165,122 @@ module Vanity
 
 
     # -- Connection management --
-   
-    # Tells the playground where to find Redis.  Accepts one of the following:
-    # - "hostname:port"
-    # - ":port"
-    # - "hostname:port:db"
-    # - Instance of Redis connection. 
-    def redis=(spec_or_connection)
-      case spec_or_connection
+ 
+    # This is the preferred way to programmatically create a new connection (or
+    # switch to a new connection). If no connection was established, the
+    # playground will create a new one by calling this method with no arguments.
+    #
+    # With no argument, uses the connection specified in config/vanity.yml file
+    # for the current environment (RACK_ENV, RAILS_ENV or development). If there
+    # is no config/vanity.yml file, picks the configuration from
+    # config/redis.yml, or defaults to Redis on localhost, port 6379.
+    #
+    # If the argument is a symbol, uses the connection specified in
+    # config/vanity.yml for that environment. For example:
+    #   Vanity.playground.establish_connection :production
+    #
+    # If the argument is a string, it is processed as a URL. For example:
+    #   Vanity.playground.establish_connection "redis://redis.local/5"
+    #
+    # Otherwise, the argument is a hash and specifies the adapter name and any
+    # additional options understood by that adapter (as with config/vanity.yml).
+    # For example:
+    #   Vanity.playground.establish_connection :adapter=>:redis,
+    #                                          :host=>"redis.local"
+    #
+    # @since 1.4.0 
+    def establish_connection(spec = nil)
+      disconnect! if @adapter
+      case spec
+      when nil
+        if File.exists?("config/vanity.yml")
+          env = ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
+          spec = YAML.load_file("config/vanity.yml")[env]
+          fail "No configuration for #{env}" unless spec
+          establish_connection spec
+        elsif File.exists?("config/redis.yml")
+          env = ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
+          redis = YAML.load_file("config/redis.yml")[env]
+          fail "No configuration for #{env}" unless redis
+          establish_connection "redis://" + redis
+        else
+          establish_connection :adapter=>"redis"
+        end
+      when Symbol
+        spec = YAML.load_file("config/vanity.yml")[spec.to_s]
+        establish_connection spec
       when String
-        @connection_spec = spec_or_connection
-        host, port, db = spec_or_connection.split(':').map { |x| x unless x.empty? }
-        @redis = Redis.new(:host=>host, :port=>port, :thread_safe=>true, :db=>db, :thread_safe=>true)
-      when Redis
-        @connection_spec = nil
-        @redis = spec_or_connection
-      when :mock
-        @connection_spec = nil
-        @redis = MockRedis.new
+        uri = URI.parse(spec)
+        params = CGI.parse(uri.query) if uri.query
+        establish_connection :adapter=>uri.scheme, :username=>uri.user, :password=>uri.password,
+          :host=>uri.host, :port=>uri.port, :path=>uri.path, :params=>params
       else
-        raise "I don't know what to do with #{spec_or_connection.inspect}"
+        spec = spec.inject({}) { |hash,(k,v)| hash[k.to_sym] = v ; hash }
+        begin
+          require "vanity/adapters/#{spec[:adapter]}_adapter"
+        rescue LoadError
+          raise "Could not find #{spec[:adapter]} in your load path"
+        end
+        @adapter = Adapters.establish_connection(spec)
       end
     end
 
-    def redis
-      self.redis = @connection_spec unless @redis
-      @redis
+    # Returns the current connection. Establishes new connection is necessary.
+    #
+    # @since 1.4.0
+    def connection
+      @adapter || establish_connection
     end
 
+    # Returns true if connection is open.
+    #
+    # @since 1.4.0
     def connected?
-      !@redis.nil?
+      @adapter && @adapter.active?
     end
 
+    # Closes the current connection.
+    #
+    # @since 1.4.0
     def disconnect!
-      @redis.quit if connected? rescue nil
-      @redis = nil
+      @adapter.disconnect! if @adapter
     end
 
+    # Closes the current connection and establishes a new one.
+    #
+    # @since 1.3.0
     def reconnect!
-      raise "Connect reconnect without connection specification" unless String === @connection_spec
-      disconnect! rescue nil
+      establish_connection
     end
 
-    def mock!
-      warn "Deprecated: use Vanity.playground.test!"
-      test!
-    end
-   
-    # Use this when testing to disable Redis (e.g. if your CI server doesn't
-    # have Redis). 
-    #
-    # @example Put this in config/environments/test.rb
-    #   config.after_initialize { Vanity.playground.test! }
+    # Deprecated. Use Vanity.playground.collecting = true/false instead.  Under
+    # Rails, collecting is true in production environment, false in all other
+    # environments, which is exactly what you want.
     def test!
-      self.redis = :mock
+      warn "Deprecated: use collecting = false instead"
+      self.collecting = false
+    end
+
+    # Deprecated. Use establish_connection or configuration file instead.
+    def redis=(spec_or_connection)
+      warn "Deprecated: use establish_connection method instead"
+      case spec_or_connection
+      when String
+        establish_connection "redis://" + spec_or_connection
+      when ::Redis
+        @connection = Adapters::RedisAdapter.new(spec_or_connection)
+      when :mock
+        establish_connection :adapter=>:mock
+      else
+        raise "I don't know what to do with #{spec_or_connection.inspect}"
+      end
     end
+
+    def redis
+      warn "Deprecated: use connection method instead"
+      connection
+    end
+
   end
 
   @playground = Playground.new

data/test/{ab_test_test.rb → experiment/ab_test.rb}

@@ -359,10 +359,10 @@ class AbTestTest < ActionController::TestCase
     assert experiment(:abcd).score.alts.all? { |alt| alt.z_score.nan? }
     assert experiment(:abcd).score.alts.all? { |alt| alt.probability == 0 }
     assert experiment(:abcd).score.alts.all? { |alt| alt.difference.nil? }
-    assert 1, experiment(:abcd).score.best.id
+    assert_equal 1, experiment(:abcd).score.best.id
     assert_nil experiment(:abcd).score.choice
-    assert 1, experiment(:abcd).score.base.id
-    assert 1, experiment(:abcd).score.least.id
+    assert_equal 2, experiment(:abcd).score.base.id
+    assert_equal 1, experiment(:abcd).score.least.id
   end
 
   def test_scoring_with_some_performers
@@ -647,6 +647,50 @@ This experiment did not run long enough to find a clear winner.
   end
 
 
+  # -- No collection --
+
+  def test_no_collection_does_not_track
+    not_collecting!
+    metric "Coolness"
+    new_ab_test :abcd do
+      metrics :coolness
+    end
+    Vanity.playground.track! :coolness
+    assert_equal 0, experiment(:abcd).alternatives.sum(&:conversions)
+  end
+
+  def test_no_collection_and_completion
+    not_collecting!
+    new_ab_test :quick do
+      outcome_is { alternatives[1] }
+      metrics :coolness
+    end
+    experiment(:quick).complete!
+    assert_nil experiment(:quick).outcome
+  end
+
+  def test_no_collection_and_chooses
+    not_collecting!
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+    end
+    assert !experiment(:simple).showing?(experiment(:simple).alternatives[1])
+    experiment(:simple).chooses(:b)
+    assert experiment(:simple).showing?(experiment(:simple).alternatives[1])
+    assert !experiment(:simple).showing?(experiment(:simple).alternatives[2])
+  end
+
+  def test_no_collection_chooses_without_database
+    not_collecting!
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+    end
+    choice = experiment(:simple).choose
+    assert [:a, :b, :c].include?(choice)
+    assert_equal choice, experiment(:simple).choose
+  end
+
+
   # -- Helper methods --
 
   def fake(name, args)

data/test/{experiment_test.rb → experiment/base_test.rb}

@@ -30,12 +30,6 @@ class ExperimentTest < Test::Unit::TestCase
     end
   end
 
-  def test_uses_playground_namespace_for_experiment
-    new_ab_test(:ice_cream_flavor) { metrics :happiness }
-    assert_equal "vanity:#{Vanity::Version::MAJOR}:ice_cream_flavor", experiment(:ice_cream_flavor).send(:key)
-    assert_equal "vanity:#{Vanity::Version::MAJOR}:ice_cream_flavor:participants", experiment(:ice_cream_flavor).send(:key, "participants")
-  end
-
 
   # -- Loading experiments --
 
@@ -80,7 +74,7 @@ class ExperimentTest < Test::Unit::TestCase
   def test_reloading_experiments
     new_ab_test(:ab) { metrics :happiness }
     new_ab_test(:cd) { metrics :happiness }
-    assert 2, Vanity.playground.experiments.size
+    assert_equal 2, Vanity.playground.experiments.size
     Vanity.playground.reload!
     assert Vanity.playground.experiments.empty?
   end
@@ -114,7 +108,7 @@ class ExperimentTest < Test::Unit::TestCase
  
   def test_experiment_keeps_created_timestamp_across_definitions
     past = Date.today - 1
-    Timecop.travel past do
+    Timecop.freeze past do
       new_ab_test(:ice_cream_flavor) { metrics :happiness }
       assert_equal past.to_time.to_i, experiment(:ice_cream_flavor).created_at.to_i
     end
@@ -133,4 +127,10 @@ class ExperimentTest < Test::Unit::TestCase
     assert_equal "Because 31 is not enough ...", experiment(:ice_cream_flavor).description
   end
 
+  def test_experiment_stores_nothing_when_collection_disabled
+    not_collecting!
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    experiment(:ice_cream_flavor).complete!
+  end
+ 
 end

data/test/metric/active_record_test.rb

@@ -0,0 +1,253 @@
+require "test/test_helper"
+
+class Sky < ActiveRecord::Base
+  connection.drop_table :skies if table_exists?
+  connection.create_table :skies do |t|
+    t.integer :height
+    t.timestamps
+  end
+
+  named_scope :high, lambda { { :conditions=>"height >= 4" } }
+end
+
+
+context "ActiveRecord Metric" do
+
+  test "record count" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create!
+    assert_equal 1, Sky.count
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record sum" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 6, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record average" do
+    Sky.aggregates
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :average=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 3, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record minimum" do
+    Sky.aggregates
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :minimum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 2, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record maximum" do
+    Sky.aggregates
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :maximum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 4, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "with conditions" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height, :conditions=>["height > 4"]
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    high_skies = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, height|
+      assert height > 4
+      high_skies += height
+    end
+    [nil,5,3,6].each do |height|
+      Sky.create! :height=>height
+    end
+    assert_equal 11, Vanity::Metric.data(metric(:sky_is_limit)).sum(&:last)
+    assert_equal 11, high_skies
+  end
+
+  test "with scope" do
+    Sky.aggregates
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky.high
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      total += count
+    end
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    assert_equal 1, total
+  end
+
+  test "hooks" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      assert_equal :sky_is_limit, metric_id
+      assert_in_delta Time.now.to_i, timestamp.to_i, 1
+      total += count
+    end
+    Sky.create! :height=>4
+    assert_equal 4, total
+  end
+
+  test "no hooks when metrics disabled" do
+    not_collecting!
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      total += count
+    end
+    Sky.create! :height=>4
+    assert_equal 0, total
+  end
+
+  test "after_create not after_save" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    once = nil
+    metric(:sky_is_limit).hook do
+      fail "Metric tracked twice" if once
+      once = true
+    end
+    Sky.create!
+    Sky.last.update_attributes :height=>4
+  end
+
+  test "with after_save" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :conditions=>["height > 3"]
+          Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    times = 0
+    metric(:sky_is_limit).hook do
+      times += 1
+    end
+    Sky.create!
+    (1..5).each do |height|
+      Sky.last.update_attributes! :height=>height
+    end
+    assert_equal 2, times
+  end
+
+  test "do it youself" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    (1..5).each do |height|
+      Sky.create! :height=>height
+    end
+    Sky.first.update_attributes! :height=>4
+    assert_equal 3, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "last update for new metric" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    assert_nil metric(:sky_is_limit).last_update_at
+  end
+
+  test "last update with records" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Sky.create! :height=>1
+    Timecop.freeze Time.now + 1.day do
+      Sky.create! :height=>1
+    end
+    assert_in_delta metric(:sky_is_limit).last_update_at.to_i, (Time.now + 1.day).to_i, 1
+  end
+
+  teardown do
+    Sky.delete_all
+    Sky.after_create.clear
+    Sky.after_save.clear
+  end
+end