vanity 1.2.0 → 1.3.0

data/CHANGELOG

@@ -1,3 +1,37 @@
+== 1.3.0 (2010-03-01)
+This release adds support for Google Analytics, AdWords and forking servers (Passenger, Unicorn).
+
+To view Google Analytics metrics from within Vanity, first make sure you are using Garb.  For example, in your Gemfile:
+
+  gem "vanity", "1.3.0"
+  gem "garb",   "0.5.0" 
+
+Next, authenticate using your account credentials.  For example, in your config/environments/production.rb:
+
+  require "garb"
+	Garb::Session.login('..email..', '..password..', account_type: "GOOGLE") rescue nil
+
+Last, define Vanity metrics that tap to Google Analytics metrics.  For example:
+
+  metric "Acquisition: Visitors" do
+    description "Unique visitors on any given page, as tracked by Google Analytics"
+    google_analytics "UA-1828623-6", :visitors
+  end
+
+* Added: Support for Google Analytics metrics, thanks to Tony Pitale's Garb and blog post: http://www.viget.com/extend/user-goal-tracking-in-rails-with-vanity-and-google-analytics/
+* Added: Vanity query parameter that you can use to choose a particular alternative, e.g. to tie an advertisement banner with content of the site.
+* Added: Command line "vanity list" catalogs all ongoing experiments, their alternatives (and fingerprints) and all metrics.
+* Added: Playground.reconnect!, particularly useful when forking (Passenger, Unicorn, etc).
+* Added: Vanity loads Redis configuration from config/redis.yml (if you have such a file).
+* Changed: New way to specify connection configuration: Vanity.playground.redis = "localhost:6379". Use this instead of the separate host/port/db attribute.
+* Changed: Rails integration now separates use_vanity method, filters and helpers.
+* Changed: Explicit vanity_context_filter and vanity_reload_filter so you can skip them, or order filters relative to them.
+* Fixed: If metric cannot be loaded (e.g. offline, no db access) show error message for that metric but don't break dashboard.
+* Fixed: AbTest incorrectly calls identify method instead of identity (issue #2)
+* Fixed: Running vanity command, automatically detects and loads Rails.
+* Fixed: Vanity now picks up on load_path set from within config/environment.rb.
+* Removed: Vanity.playground.define is deprecated. Bad choice for a method name. If you need this feature, make a suggestion and let's create a better API.
+
 == 1.2.0 (2009-12-14)
 This release introduces metrics backed by ActiveRecord.  Use them when your model is already tracking a metric, and you get instant historical data.
 

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org/"
+gem "autotest"
+gem "autotest-fsevent"
+gem "autotest-growl"
+gem "bundler"
+gem "garb"
+gem "jekyll"
+gem "mocha"
+gem "passenger"
+gem "rails", "2.3.5"
+gem "rack", "1.0.1"
+gem "rubygems-update"
+gem "shoulda"
+gem "sqlite3-ruby"
+gem "timecop"
+gem "yard"

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Assaf Arkin
+Copyright (c) 2010 Assaf Arkin
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -11,7 +11,13 @@ http://farm3.static.flickr.com/2540/4099665871_497f274f68_o.jpg
 
 <b>Step 1:</b> Start using Vanity in your Rails application:
 
-  gem.config "vanity"
+  Rails::Initializer.run do |config|
+    gem.config "vanity"
+
+    config.after_initialize do
+      require "vanity"
+    end
+  end
 
 And:
 
@@ -47,21 +53,20 @@ And:
 
 <b>Step 5:</b> Check the report:
 
-  vanity --output vanity.html
+  vanity report --output vanity.html
 
 
 == Building From Source
 
 To run the test suite for the first time:
 
-  $ gem install rails mocha timecop
-  $ rake
+  $ bundle install
+  $ bundle exec rake
 
 You can also +rake test+ if you insist on being explicit.
 
 To build the documentation:
  
-  $ gem install yardoc jekyll
   $ rake docs
   $ open html/index.html
 

data/Rakefile

@@ -0,0 +1,119 @@
+require "rake/testtask"
+
+spec = Gem::Specification.load(File.expand_path("vanity.gemspec", File.dirname(__FILE__)))
+
+desc "Build the Gem"
+task :build=>:test do
+  sh "gem build #{spec.name}.gemspec"
+end
+
+desc "Install #{spec.name} locally"
+task :install=>:build do
+  sudo = "sudo" unless File.writable?( Gem::ConfigMap[:bindir])
+  sh "#{sudo} gem install #{spec.name}-#{spec.version}.gem"
+end
+
+desc "Push new release to gemcutter and git tag"
+task :push=>:build do
+  sh "git push"
+  puts "Tagging version #{spec.version} .."
+  sh "git tag v#{spec.version}"
+  sh "git push --tag"
+  puts "Building and pushing gem .."
+  sh "gem push #{spec.name}-#{spec.version}.gem"
+end
+
+
+task :default=>:test
+desc "Run all tests using Redis mock (also default task)"
+Rake::TestTask.new do |task|
+  task.test_files = FileList['test/*_test.rb']
+  if Rake.application.options.trace
+    #task.warning = true
+    task.verbose = true
+  elsif Rake.application.options.silent
+    task.ruby_opts << "-W0"
+  else
+    task.verbose = true
+  end
+end
+
+desc "Run all tests using live redis server"
+task "test:redis" do
+  ENV["REDIS"] = "true"
+  task(:test).invoke
+end
+
+task(:clobber) { rm_rf "tmp" }
+
+
+begin
+  require "yard"
+  YARD::Rake::YardocTask.new(:yardoc) do |task|
+    task.files  = FileList["lib/**/*.rb"].exclude("lib/vanity/backport.rb")
+    task.options = "--output", "html/api", "--title", "Vanity #{spec.version}", "--main", "README.rdoc", "--files", "CHANGELOG"
+  end
+rescue LoadError
+end
+
+desc "Jekyll generates the main documentation (sans API)"
+task(:jekyll) { sh "jekyll", "doc", "html" }
+
+desc "Create documentation in docs directory (including API)"
+task :docs=>[:jekyll, :yardoc]
+desc "Remove temporary files and directories"
+task(:clobber) { rm_rf "html" }
+
+desc "Publish documentation to vanity.labnotes.org"
+task :publish=>[:clobber, :docs] do
+  sh "rsync -cr --del --progress html/ labnotes.org:/var/www/vanity/"
+end
+
+
+task :report do
+  $LOAD_PATH.unshift "lib"
+  require "vanity"
+  require "timecop"
+  Vanity.playground.load_path = "test/experiments"
+  Vanity.playground.experiments.values.each(&:destroy)
+  Vanity.playground.metrics.values.each(&:destroy!)
+  Vanity.playground.reload!
+
+  # Control	182	35	19.23%	N/A
+  # Treatment A	180	45	25.00%	1.33
+  # Treatment B	189	28	14.81%	-1.13
+  # Treatment C	188	61	32.45%	2.94
+  Vanity.playground.experiment(:null_abc).instance_eval do
+    fake nil=>[182,35], :red=>[180,45], :green=>[189,28], :blue=>[188,61]
+    @created_at = (Date.today - 40).to_time
+    @completed_at = (Date.today - 35).to_time
+  end
+
+  Vanity.playground.experiment(:age_and_zipcode).instance_eval do
+    fake false=>[80,35], true=>[84,32]
+    @created_at = (Date.today - 30).to_time
+    @completed_at = (Date.today - 15).to_time
+  end
+
+  Vanity.context = Object.new
+  Vanity.context.instance_eval { def vanity_identity ; 0 ; end }
+  signups = 50
+  (Date.today - 90..Date.today).each do |date|
+    Timecop.travel date do
+      signups += rand(15) - 5
+      Vanity.playground.track! :signups, signups
+    end
+  end
+
+  cheers, yawns = 0, 0
+  (Date.today - 80..Date.today).each do |date|
+    Timecop.travel date do
+      cheers = cheers - 5 + rand(20)
+      Vanity.playground.track! :yawns, cheers
+      yawns = yawns - 5 + rand(30)
+      Vanity.playground.track! :cheers, yawns
+    end
+  end
+
+  Vanity::Commands.report ENV["OUTPUT"]
+end

data/bin/vanity

@@ -1,8 +1,13 @@
 #!/usr/bin/env ruby
-path = File.expand_path("../lib", File.dirname(__FILE__))
-$LOAD_PATH.unshift path unless $LOAD_PATH.include?(path)
+# Is there a better way to detect Rails?
+if File.exist?("config/boot.rb") && File.exist?("config/environment.rb")
+  require "config/environment"
+else
+  path = File.expand_path("../lib", File.dirname(__FILE__))
+  $LOAD_PATH.unshift path unless $LOAD_PATH.include?(path)
+  require "vanity"
+end
 
-require "vanity"
 require "optparse"
 
 playground = Vanity.playground
@@ -10,24 +15,24 @@ options = Struct.new(:output).new
 opts = OptionParser.new("", 24, "  ") do |opts|
   opts.banner = "Usage: #{File.basename($0)} [options] command\n"
   opts.banner << "Commands:\n"
-  opts.banner << "  report  Report on all running experiments"
-
-  opts.separator ""
-  opts.separator "General options:"
-  opts.on("--path PATH", "Path to experiments directory (default: #{playground.load_path})")  { |v| playground.load_path = v }
-  opts.on("--output FILE", "Write report to this file (default: stdout)")                     { |v| options.output = v }
+  opts.banner << "  list    List all experiments and metrics\n"
+  opts.banner << "  report  Report on all running experiments/metrics\n"
 
   opts.separator ""
-  opts.separator "Redis options:"
-  opts.on("--host HOST", "Redis server host (default: #{playground.host})")       { |v| playground.host = v }
-  opts.on("--port PORT", "Redis server port (default: #{playground.port})")       { |v| playground.port = v }
-  opts.on("--db DB", "Redis database (default: #{playground.db})")                { |v| playground.db = v }
-  opts.on("--password PWD", "Redis database password")                            { |v| playground.password = v }
-  opts.on("--namespace NS", "Redis namespace (default: #{playground.namespace})") { |v| playground.namespace = v }
+  opts.separator "Reporting options:"
+  opts.on "--output FILE", "Write report to this file (default: stdout)" do |path|
+    options.output = path
+  end
 
   opts.separator ""
   opts.separator "Common options:"
-  opts.on_tail "-h", "-H", "--help", "Show this message" do
+  opts.on "--load_path PATH", "Path to experiments directory (default: #{playground.load_path})" do |path|
+    playground.load_path = path
+  end
+  opts.on "--redis HOST:PORT:DB", "Redis server host (default: localhost:6379)" do |redis|
+    playground.redis = redis
+  end
+  opts.on_tail "-h", "--help", "Show this message" do
     puts opts.to_s.gsub(/^.*DEPRECATED.*$/s, '')
     exit
   end
@@ -45,8 +50,8 @@ end
 
 ARGV.each do |cmd|
   case cmd
-  when "report"
-    Vanity::Commands.report options.output
+  when "report";  Vanity::Commands.report options.output
+  when "list";    Vanity::Commands.list
   else fail "No such command: #{cmd}"
   end
 end

data/lib/vanity.rb

@@ -1,9 +1,10 @@
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "../vendor/redis-rb/lib")
-require "redis"
 require "date"
 require "time"
 require "logger"
 
+$LOAD_PATH << File.join(File.dirname(__FILE__), "../vendor/redis-rb/lib")
+autoload :Redis, "redis"
+
 # All the cool stuff happens in other places.
 # @see Vanity::Helper
 # @see Vanity::Rails
@@ -19,14 +20,21 @@ module Vanity
     PATCH = version[2]
     STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
   end
+
 end
 
 require "vanity/backport" if RUBY_VERSION < "1.9"
-require "vanity/metric"
+# Metrics.
+require "vanity/metric/base"
+require "vanity/metric/active_record"
+require "vanity/metric/google_analytics"
+# Experiments.
 require "vanity/experiment/base"
 require "vanity/experiment/ab_test"
+# Playground.
 require "vanity/playground"
 require "vanity/helpers"
 Vanity.autoload :MockRedis, "vanity/mock_redis"
 Vanity.autoload :Commands, "vanity/commands"
-require "vanity/rails" if defined?(Rails)
+# Integration with various frameworks.
+require "vanity/frameworks/rails" if defined?(Rails)

data/lib/vanity/commands.rb

@@ -1 +1,2 @@
 require "vanity/commands/report"
+require "vanity/commands/list"

data/lib/vanity/commands/list.rb

@@ -0,0 +1,21 @@
+module Vanity
+  module Commands
+    class << self
+      # Lists all experiments and metrics.
+      def list
+        Vanity.playground.experiments.each do |id, experiment|
+          puts "experiment :%-.20s (%-.40s)" % [id, experiment.name]
+          if experiment.respond_to?(:alternatives)
+            experiment.alternatives.each do |alt|
+              hash = experiment.fingerprint(alt)
+              puts "  %s: %-40.40s  (%s)" % [alt.name, alt.value, hash]
+            end
+          end
+        end
+        Vanity.playground.metrics.each do |id, metric|
+          puts "metric :%-.20s (%-.40s)" % [id, metric.name]
+        end
+      end
+    end
+  end
+end

data/lib/vanity/experiment/ab_test.rb

@@ -195,11 +195,18 @@ module Vanity
             check_completion!
           end
         else
-          index = redis[key("outcome")] || alternative_for(identify)
+          index = redis[key("outcome")] || alternative_for(identity)
         end
         @alternatives[index.to_i]
       end
 
+      # Returns fingerprint (hash) for given alternative.  Can be used to lookup
+      # alternative for experiment without revealing what values are available
+      # (e.g. choosing alternative from HTTP query parameter).
+      def fingerprint(alternative)
+        Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
+      end
+
       
       # -- Testing --
      

data/lib/vanity/experiment/base.rb

@@ -15,11 +15,16 @@ module Vanity
       # Defines a new experiment, given the experiment's name, type and
       # definition block.
       def define(name, type, options = nil, &block)
-        playground.define(name, type, options || {}, &block)
+        fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
+        klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
+        experiment = klass.new(playground, @experiment_id, name, options)
+        experiment.instance_eval &block
+        experiment.save
+        playground.experiments[@experiment_id] = experiment
       end
 
-      def binding_with(playground)
-        @playground = playground
+      def new_binding(playground, id)
+        @playground, @experiment_id = playground, id
         binding
       end
 
@@ -37,16 +42,16 @@ module Vanity
         end
 
         # Playground uses this to load experiment definitions.
-        def load(playground, stack, path, id)
-          fn = File.join(path, "#{id}.rb")
-          fail "Circular dependency detected: #{stack.join('=>')}=>#{fn}" if stack.include?(fn)
-          source = File.read(fn)
-          stack.push fn
+        def load(playground, stack, file)
+          fail "Circular dependency detected: #{stack.join('=>')}=>#{file}" if stack.include?(file)
+          source = File.read(file)
+          stack.push file
+          id = File.basename(file, ".rb").downcase.gsub(/\W/, "_").to_sym
           context = Object.new
           context.instance_eval do
             extend Definition
-            experiment = eval(source, context.binding_with(playground), fn)
-            fail NameError.new("Expected #{fn} to define experiment #{id}", id) unless experiment.id == id
+            experiment = eval(source, context.new_binding(playground, id), file)
+            fail NameError.new("Expected #{file} to define experiment #{id}", id) unless playground.experiments[id]
             experiment
           end
         rescue
@@ -59,12 +64,12 @@ module Vanity
 
       end
 
-      def initialize(playground, id, name, options, &block)
+      def initialize(playground, id, name, options = nil)
         @playground = playground
         @id, @name = id.to_sym, name
         @options = options || {}
         @namespace = "#{@playground.namespace}:#{@id}"
-        @identify_block = lambda { |context| context.vanity_identity }
+        @identify_block = method(:default_identify)
       end
 
       # Human readable experiment name (first argument you pass when creating a
@@ -101,14 +106,10 @@ module Vanity
       #     end
       #   end
       def identify(&block)
+        fail "Missing block" unless block
         @identify_block = block
       end
 
-      def identity
-        @identify_block.call(Vanity.context)
-      end
-      protected :identity
-
 
       # -- Reporting --
 
@@ -136,19 +137,6 @@ module Vanity
         @complete_block = block
       end
 
-      # Derived classes call this after state changes that may lead to
-      # experiment completing.
-      def check_completion!
-        if @complete_block
-          begin
-            complete! if @complete_block.call
-          rescue
-            # TODO: logging
-          end
-        end
-      end
-      protected :check_completion!
-
       # Force experiment to complete.
       def complete!
         redis.setnx key(:completed_at), Time.now.to_i
@@ -184,6 +172,28 @@ module Vanity
 
     protected
 
+      def identity
+        @identify_block.call(Vanity.context)
+      end
+
+      def default_identify(context)
+        raise "No Vanity.context" unless context
+        raise "Vanity.context does not respond to vanity_identity" unless context.respond_to?(:vanity_identity)
+        context.vanity_identity or raise "Vanity.context.vanity_identity - no identity"
+      end
+
+      # Derived classes call this after state changes that may lead to
+      # experiment completing.
+      def check_completion!
+        if @complete_block
+          begin
+            complete! if @complete_block.call
+          rescue
+            # TODO: logging
+          end
+        end
+      end
+      
       # Returns key for this experiment, or with an argument, return a key
       # using the experiment as the namespace.  Examples:
       #   key => "vanity:experiments:green_button"