vanity 1.0.0 → 1.1.0

data/lib/vanity/experiment/base.rb

@@ -1,6 +1,29 @@
 module Vanity
   module Experiment
 
+    # These methods are available from experiment definitions (files located in
+    # the experiments directory, automatically loaded by Vanity).  Use these
+    # methods to define you experiments, for example:
+    #   ab_test "New Banner" do
+    #     alternatives :red, :green, :blue
+    #     metrics :signup
+    #   end
+    module Definition
+
+      # Defines a new experiment, given the experiment's name, type and
+      # definition block.
+      def define(name, type, options = nil, &block)
+        options ||= {}
+        @playground.define(name, type, options, &block)
+      end
+
+      def binding(playground)
+        @playground = playground
+        Kernel.binding
+      end
+
+    end
+
     # Base class that all experiment types are derived from.
     class Base
 
@@ -12,6 +35,27 @@ module Vanity
           name.split("::").last.gsub(/([a-z])([A-Z])/) { "#{$1}_#{$2}" }.gsub(/([A-Z])([A-Z][a-z])/) { "#{$1}_#{$2}" }.downcase
         end
 
+        # Playground uses this to load experiment definitions.
+        def load(playground, stack, path, id)
+          fn = File.join(path, "#{id}.rb")
+          fail "Circular dependency detected: #{stack.join('=>')}=>#{fn}" if stack.include?(fn)
+          source = File.read(fn)
+          stack.push fn
+          context = Object.new
+          context.instance_eval do
+            extend Definition
+            experiment = eval(source, context.binding(playground), fn)
+            fail NameError.new("Expected #{fn} to define experiment #{id}", id) unless experiment.id == id
+            experiment
+          end
+        rescue
+          error = NameError.exception($!.message, id)
+          error.set_backtrace $!.backtrace
+          raise error
+        ensure
+          stack.pop
+        end
+
       end
 
       def initialize(playground, id, name, options, &block)
@@ -19,7 +63,7 @@ module Vanity
         @id, @name = id.to_sym, name
         @options = options || {}
         @namespace = "#{@playground.namespace}:#{@id}"
-        @identify_block = ->(context){ context.vanity_identity }
+        @identify_block = lambda { |context| context.vanity_identity }
       end
 
       # Human readable experiment name (first argument you pass when creating a
@@ -110,18 +154,19 @@ module Vanity
       # Force experiment to complete.
       def complete!
         redis.setnx key(:completed_at), Time.now.to_i
-        # TODO: logging
+        @completed_at = redis[key(:completed_at)]
+        @playground.logger.info "vanity: completed experiment #{id}"
       end
 
       # Time stamp when experiment was completed.
       def completed_at
-        time = redis[key(:completed_at)]
-        time && Time.at(time.to_i)
+        @completed_at ||= redis[key(:completed_at)]
+        @completed_at && Time.at(@completed_at.to_i)
       end
       
       # Returns true if experiment active, false if completed.
       def active?
-        redis[key(:completed_at)].nil?
+        !redis.exists(key(:completed_at))
       end
 
       # -- Store/validate --
@@ -130,6 +175,7 @@ module Vanity
       def destroy
         redis.del key(:created_at)
         redis.del key(:completed_at)
+        @created_at = @completed_at = nil
       end
 
       # Called by Playground to save the experiment definition.

data/lib/vanity/metric.rb

@@ -0,0 +1,213 @@
+module Vanity
+
+  # A metric is an object that implements two methods: +name+ and +values+.  It
+  # can also respond to addition methods (+track!+, +bounds+, etc), these are
+  # optional.
+  #
+  # This class implements a basic metric that tracks data and stores it in
+  # Redis.  You can use this as the basis for your metric, or as reference for
+  # the methods your metric must and can implement.
+  #
+  # @since 1.1.0
+  class Metric
+
+    # These methods are available when defining a metric in a file loaded
+    # from the +experiments/metrics+ directory.
+    #
+    # For example:
+    #   $ cat experiments/metrics/yawn_sec
+    #   metric "Yawns/sec" do
+    #     description "Most boring metric ever"
+    #   end
+    module Definition
+      
+      # Defines a new metric, using the class Vanity::Metric.
+      def metric(name, &block)
+        metric = Metric.new(@playground, name.to_s, name.to_s.downcase.gsub(/\W/, "_"))
+        metric.instance_eval &block
+        metric
+      end
+
+      def binding(playground)
+        @playground = playground
+        Kernel.binding
+      end
+
+    end
+  
+    # Startup metrics for pirates. AARRR stands for:
+    # * Acquisition
+    # * Activation
+    # * Retention
+    # * Referral
+    # * Revenue
+    # Read more: http://500hats.typepad.com/500blogs/2007/09/startup-metrics.html
+
+    class << self
+
+      # Helper method to return description for a metric.
+      #
+      # A metric object may have a +description+ method that returns a detailed
+      # description.  It may also have no description, or no +description+
+      # method, in which case return +nil+.
+      # 
+      # @example
+      #   puts Vanity::Metric.description(metric)
+      def description(metric)
+        metric.description if metric.respond_to?(:description)
+      end
+
+      # Helper method to return bounds for a metric.
+      #
+      # A metric object may have a +bounds+ method that returns lower and upper
+      # bounds.  It may also have no bounds, or no +bounds+ # method, in which
+      # case we return +[nil, nil]+.
+      # 
+      # @example
+      #   upper = Vanity::Metric.bounds(metric).last
+      def bounds(metric)
+        metric.respond_to?(:bounds) && metric.bounds || [nil, nil]
+      end
+
+      # Returns data set for a given date range.  The data set is an array of
+      # date, value pairs.
+      #
+      # First argument is the metric.  Second argument is the start date, or
+      # number of days to go back in history, defaults to 90 days.  Third
+      # argument is end date, defaults to today.
+      #
+      # @example These are all equivalent:
+      #   Vanity::Metric.data(my_metric) 
+      #   Vanity::Metric.data(my_metric, 90) 
+      #   Vanity::Metric.data(my_metric, Date.today - 90)
+      #   Vanity::Metric.data(my_metric, Date.today - 90, Date.today)
+      def data(metric, *args)
+        first = args.shift || 90
+        to = args.shift || Date.today
+        from = first.respond_to?(:to_date) ? first.to_date : to - first
+        (from..to).zip(metric.values(from, to))
+      end
+
+      # Playground uses this to load metric definitions.
+      def load(playground, stack, path, id)
+        fn = File.join(path, "#{id}.rb")
+        fail "Circular dependency detected: #{stack.join('=>')}=>#{fn}" if stack.include?(fn)
+        source = File.read(fn)
+        stack.push fn
+        context = Object.new
+        context.instance_eval do
+          extend Definition
+          metric = eval(source, context.binding(playground), fn)
+          fail NameError.new("Expected #{fn} to define metric #{id}", id) unless metric.name.downcase.gsub(/\W+/, '_').to_sym == id
+          metric
+        end
+      rescue
+        error = NameError.exception($!.message, id)
+        error.set_backtrace $!.backtrace
+        raise error
+      ensure
+        stack.pop
+      end
+
+    end
+
+
+    # Takes playground (need this to access Redis), friendly name and optional
+    # id (can infer from name).
+    def initialize(playground, name, id = nil)
+      id ||= name.to_s.downcase.gsub(/\W+/, '_')
+      @playground, @name, @id = playground, name.to_s, id.to_sym
+      @hooks = []
+      redis.setnx key(:created_at), Time.now.to_i
+      @created_at = Time.at(redis[key(:created_at)].to_i)
+    end
+
+
+    # -- Tracking --
+
+    # Called to track an action associated with this metric.
+    def track!(count = 1)
+      timestamp = Time.now
+      if count > 0
+        redis.incrby key(timestamp.to_date, "count"), count
+        @playground.logger.info "vanity: #{@id} with count #{count}"
+        @hooks.each do |hook|
+          hook.call @id, timestamp, count
+        end
+      end
+    end
+
+    # Metric definitions use this to introduce tracking hook.  The hook is
+    # called with metric identifier, timestamp, count and possibly additional
+    # arguments.
+    #
+    # For example:
+    #   hook do |metric_id, timestamp, count|
+    #     syslog.info metric_id
+    #   end
+    def hook(&block)
+      @hooks << block
+    end
+
+    # This method returns the acceptable bounds of a metric as an array with
+    # two values: low and high.  Use nil for unbounded.
+    #
+    # Alerts are created when metric values exceed their bounds.  For example,
+    # a metric of user registration can use historical data to calculate
+    # expected range of new registration for the next day.  If actual metric
+    # falls below the expected range, it could indicate registration process is
+    # broken.  Going above higher bound could trigger opening a Champagne
+    # bottle.
+    #
+    # The default implementation returns +nil+.
+    def bounds
+    end
+    
+
+    #  -- Reporting --
+    
+    # Human readable metric name.  All metrics must implement this method.
+    def name
+      @name
+    end
+
+    # Time stamp when metric was created.
+    attr_reader :created_at
+
+    # Human readable description.  Use two newlines to break paragraphs.
+    attr_accessor :description
+
+    # Sets or returns description. For example
+    #   metric "Yawns/sec" do
+    #     description "Most boring metric ever"
+    #   end
+    #
+    #   puts "Just defined: " + metric(:boring).description
+    def description(text = nil)
+      @description = text if text
+      @description
+    end
+
+    # Given two arguments, a start date and an end date (inclusive), returns an
+    # array of measurements.  All metrics must implement this method.
+    def values(from, to)
+      redis.mget((from.to_date..to.to_date).map { |date| key(date, "count") }).map(&:to_i)
+    end
+
+
+    # -- Storage --
+
+    def destroy!
+      redis.del redis.keys(key("*"))
+    end
+
+    def redis
+      @playground.redis
+    end
+
+    def key(*args)
+      "metrics:#{@id}:#{args.join(':')}"
+    end
+
+  end
+end

data/lib/vanity/mock_redis.rb

@@ -0,0 +1,76 @@
+module Vanity
+  # The Redis you should never use in production.
+  class MockRedis
+    @@hash = {}
+
+    def initialize(options = {})
+    end
+
+    def [](key)
+      @@hash[key]
+    end
+
+    def []=(key, value)
+      @@hash[key] = value.to_s
+    end
+
+    def del(*keys)
+      keys.flatten.each do |key|
+        @@hash.delete key
+      end
+    end
+
+    def setnx(key, value)
+      @@hash[key] = value.to_s unless @@hash.has_key?(key)
+    end
+
+    def incr(key)
+      @@hash[key] = (@@hash[key].to_i + 1).to_s
+    end
+
+    def incrby(key, value)
+      @@hash[key] = (@@hash[key].to_i + value).to_s
+    end
+
+    def mget(keys)
+      @@hash.values_at(*keys)
+    end
+
+    def exists(key)
+      @@hash.has_key?(key)
+    end
+
+    def keys(pattern)
+      regexp = Regexp.new(pattern.split("*").map { |r| Regexp.escape(r) }.join(".*"))
+      @@hash.keys.select { |key| key =~ regexp }
+    end
+
+    def flushdb
+      @@hash.clear
+    end
+
+    def sismember(key, value)
+      case set = @@hash[key]
+      when nil ; false
+      when Set ; set.member?(value)
+      else fail "Not a set"
+      end
+    end
+
+    def sadd(key, value)
+      case set = @@hash[key]
+      when nil ; @@hash[key] = Set.new([value])
+      when Set ; set.add value
+      else fail "Not a set"
+      end
+    end
+
+    def scard(key)
+      case set = @@hash[key]
+      when nil ; 0
+      when Set ; set.size
+      else fail "Not a set"
+      end
+    end
+  end
+end

data/lib/vanity/playground.rb

@@ -1,35 +1,23 @@
 module Vanity
 
-  # These methods are available from experiment definitions (files located in
-  # the experiments directory, automatically loaded by Vanity).  Use these
-  # methods to define you experiments, for example:
-  #   ab_test "New Banner" do
-  #     alternatives :red, :green, :blue
-  #   end
-  module Definition
-
-  protected
-    # Defines a new experiment, given the experiment's name, type and
-    # definition block.
-    def define(name, type, options = nil, &block)
-      options ||= {}
-      Vanity.playground.define(name, type, options, &block)
-    end
-
-  end
-
   # Playground catalogs all your experiments, holds the Vanity configuration.
-  # For example:
+  #
+  # @example
   #   Vanity.playground.logger = my_logger
   #   puts Vanity.playground.map(&:name)
   class Playground
 
+    DEFAULTS = { :host=>"127.0.0.1", :port=>6379, :db=>0, :load_path=>"experiments" }
+
     # Created new Playground. Unless you need to, use the global Vanity.playground.
-    def initialize
-      @experiments = {}
-      @host, @port, @db = "127.0.0.1", 6379, 0
+    def initialize(options = {})
+      @host, @port, @db, @load_path = DEFAULTS.merge(options).values_at(:host, :port, :db, :load_path)
       @namespace = "vanity:#{Vanity::Version::MAJOR}"
-      @load_path = "experiments"
+      @logger = options[:logger] || Logger.new(STDOUT)
+      @logger.level = Logger::ERROR
+      @redis = options[:redis]
+      @experiments = {}
+      @loading = []
     end
     
     # Redis host name.  Default is 127.0.0.1
@@ -56,7 +44,7 @@ module Vanity
     # Defines a new experiment. Generally, do not call this directly,
     # use one of the definition methods (ab_test, measure, etc).
     def define(name, type, options = {}, &block)
-      id = name.to_s.downcase.gsub(/\W/, "_")
+      id = name.to_s.downcase.gsub(/\W/, "_").to_sym
       raise "Experiment #{id} already defined once" if @experiments[id]
       klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
       experiment = klass.new(self, id, name, options)
@@ -65,41 +53,19 @@ module Vanity
       @experiments[id] = experiment
     end
 
-    # Returns the named experiment. You may not have guessed, but this method
-    # raises an exception if it cannot load the experiment's definition.
-    #
-    # Experiment names are always mapped by downcasing them and replacing
-    # non-word characters with underscores, so "Green call to action" becomes
-    # "green_call_to_action". You can also use a symbol if you feel like it.
+    # Returns the experiment. You may not have guessed, but this method raises
+    # an exception if it cannot load the experiment's definition.
     def experiment(name)
-      id = name.to_s.downcase.gsub(/\W/, "_")
-      unless @experiments.has_key?(id)
-        @loading ||= []
-        fail "Circular dependency detected: #{@loading.join('=>')}=>#{id}" if @loading.include?(id)
-        begin
-          @loading.push id
-          source = File.read(File.expand_path("#{id}.rb", load_path))
-          context = Object.new
-          context.instance_eval do
-            extend Definition
-            eval source
-          end
-        rescue
-          error = LoadError.exception($!.message)
-          error.set_backtrace $!.backtrace
-          raise error
-        ensure
-          @loading.pop
-        end
-      end
-      @experiments[id] or fail LoadError, "Expected experiments/#{id}.rb to define experiment #{name}"
+      id = name.to_s.downcase.gsub(/\W/, "_").to_sym
+      warn "Deprecated: pleae call experiment method with experiment identifier (a Ruby symbol)" unless id == name
+      @experiments[id] ||= Experiment::Base.load(self, @loading, File.expand_path(load_path), id)
     end
 
     # Returns list of all loaded experiments.
     def experiments
       Dir[File.join(load_path, "*.rb")].each do |file|
         id = File.basename(file).gsub(/.rb$/, "")
-        experiment id
+        experiment id.to_sym
       end
       @experiments.values
     end
@@ -107,25 +73,72 @@ module Vanity
     # Reloads all experiments.
     def reload!
       @experiments.clear
+      @metrics = nil
     end
 
     # Use this instance to access the Redis database.
     def redis
-      redis = Redis.new(host: self.host, port: self.port, db: self.db,
-                        password: self.password, logger: self.logger)
-      class << self ; self ; end.send(:define_method, :redis) { redis }
-      redis
+      @redis ||= Redis.new(:host=>self.host, :port=>self.port, :db=>self.db,
+                           :password=>self.password, :logger=>self.logger)
+      class << self ; self ; end.send(:define_method, :redis) { @redis }
+      @redis
+    end
+
+    # Switches playground to use MockRedis instead of a live server.
+    # Particularly useful for testing, e.g. if you can't access Redis on your CI
+    # server.  This method has no affect after playground accesses live Redis
+    # server.
+    #
+    # @example Put this in config/environments/test.rb
+    #   config.after_initialize { Vanity.playground.mock! }
+    def mock!
+      @redis ||= MockRedis.new
     end
 
+    # Returns a metric (creating one if doesn't already exist).
+    #
+    # @since 1.1.0
+    def metric(id)
+      id = id.to_sym
+      metrics[id] ||= Metric.load(self, @loading, File.expand_path("metrics", load_path), id)
+    end
+
+    # Returns hash of metrics (key is metric id).
+    #
+    # @since 1.1.0
+    def metrics
+      unless @metrics
+        @metrics = {}
+        Dir[File.join(load_path, "metrics/*.rb")].each do |file|
+          begin
+            id = File.basename(file).gsub(/.rb$/, "")
+            metric id
+          rescue NameError
+            @logger.error "Could not load metric #{$!.name}: #{$!}"
+          end
+        end
+      end
+      @metrics
+    end
+
+    # Tracks an action associated with a metric.
+    #
+    # @example
+    #   Vanity.playground.track! :uploaded_video
+    #
+    # @since 1.1.0
+    def track!(id, count = 1)
+      metric(id).track! count
+    end
   end
 
   @playground = Playground.new
   class << self
 
-    # Returns the playground instance.
-    def playground
-      @playground
-    end
+    # The playground instance.
+    #
+    # @see Vanity::Playground
+    attr_accessor :playground
 
     # Returns the Vanity context.  For example, when using Rails this would be
     # the current controller, which can be used to get/set the vanity identity.
@@ -152,8 +165,12 @@ end
 
 class Object
 
-  # Use this method to access an experiment by name.  For example:
+  # Use this method to access an experiment by name.
+  #
+  # @example
   #   puts experiment(:text_size).alternatives
+  #
+  # @see Vanity::Playground#experiment
   def experiment(name)
     Vanity.playground.experiment(name)
   end