mikeg-vanity 1.3.0

data/lib/vanity/experiment/base.rb

@@ -0,0 +1,212 @@
+module Vanity
+  module Experiment
+
+    # These methods are available from experiment definitions (files located in
+    # the experiments directory, automatically loaded by Vanity).  Use these
+    # methods to define you experiments, for example:
+    #   ab_test "New Banner" do
+    #     alternatives :red, :green, :blue
+    #     metrics :signup
+    #   end
+    module Definition
+
+      attr_reader :playground
+
+      # Defines a new experiment, given the experiment's name, type and
+      # definition block.
+      def define(name, type, options = nil, &block)
+        fail "Experiment #{@experiment_id} already defined in playground" if playground.experiments[@experiment_id]
+        klass = Experiment.const_get(type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase })
+        experiment = klass.new(playground, @experiment_id, name, options)
+        experiment.instance_eval &block
+        experiment.save
+        playground.experiments[@experiment_id] = experiment
+      end
+
+      def new_binding(playground, id)
+        @playground, @experiment_id = playground, id
+        binding
+      end
+
+    end
+
+    # Base class that all experiment types are derived from.
+    class Base
+
+      class << self
+        
+        # Returns the type of this class as a symbol (e.g. AbTest becomes
+        # ab_test).
+        def type
+          name.split("::").last.gsub(/([a-z])([A-Z])/) { "#{$1}_#{$2}" }.gsub(/([A-Z])([A-Z][a-z])/) { "#{$1}_#{$2}" }.downcase
+        end
+
+        # Playground uses this to load experiment definitions.
+        def load(playground, stack, file)
+          fail "Circular dependency detected: #{stack.join('=>')}=>#{file}" if stack.include?(file)
+          source = File.read(file)
+          stack.push file
+          id = File.basename(file, ".rb").downcase.gsub(/\W/, "_").to_sym
+          context = Object.new
+          context.instance_eval do
+            extend Definition
+            experiment = eval(source, context.new_binding(playground, id), file)
+            fail NameError.new("Expected #{file} to define experiment #{id}", id) unless playground.experiments[id]
+            experiment
+          end
+        rescue
+          error = NameError.exception($!.message, id)
+          error.set_backtrace $!.backtrace
+          raise error
+        ensure
+          stack.pop
+        end
+
+      end
+
+      def initialize(playground, id, name, options = nil)
+        @playground = playground
+        @id, @name = id.to_sym, name
+        @options = options || {}
+        @namespace = "#{@playground.namespace}:#{@id}"
+        @identify_block = method(:default_identify)
+      end
+
+      # Human readable experiment name (first argument you pass when creating a
+      # new experiment).
+      attr_reader :name
+      alias :to_s :name
+
+      # Unique identifier, derived from name experiment name, e.g. "Green
+      # Button" becomes :green_button.
+      attr_reader :id
+
+      # Time stamp when experiment was created.
+      attr_reader :created_at
+
+      # Time stamp when experiment was completed.
+      attr_reader :completed_at
+
+      # Returns the type of this experiment as a symbol (e.g. :ab_test).
+      def type
+        self.class.type
+      end
+     
+      # Defines how we obtain an identity for the current experiment.  Usually
+      # Vanity gets the identity form a session object (see use_vanity), but
+      # there are cases where you want a particular experiment to use a
+      # different identity.
+      #
+      # For example, if all your experiments use current_user and you need one
+      # experiment to use the current project:
+      #   ab_test "Project widget" do
+      #     alternatives :small, :medium, :large
+      #     identify do |controller|
+      #       controller.project.id
+      #     end
+      #   end
+      def identify(&block)
+        @identify_block = block
+      end
+
+
+      # -- Reporting --
+
+      # Sets or returns description. For example
+      #   ab_test "Simple" do
+      #     description "A simple A/B experiment"
+      #   end
+      #
+      #   puts "Just defined: " + experiment(:simple).description
+      def description(text = nil)
+        @description = text if text
+        @description
+      end
+
+
+      # -- Experiment completion --
+
+      # Define experiment completion condition.  For example:
+      #   complete_if do
+      #     !score(95).chosen.nil?
+      #   end
+      def complete_if(&block)
+        raise ArgumentError, "Missing block" unless block
+        raise "complete_if already called on this experiment" if @complete_block
+        @complete_block = block
+      end
+
+      # Force experiment to complete.
+      def complete!
+        redis.setnx key(:completed_at), Time.now.to_i
+        @completed_at = redis[key(:completed_at)]
+        @playground.logger.info "vanity: completed experiment #{id}"
+      end
+
+      # Time stamp when experiment was completed.
+      def completed_at
+        @completed_at ||= redis[key(:completed_at)]
+        @completed_at && Time.at(@completed_at.to_i)
+      end
+      
+      # Returns true if experiment active, false if completed.
+      def active?
+        !redis.exists(key(:completed_at))
+      end
+
+      # -- Store/validate --
+
+      # Get rid of all experiment data.
+      def destroy
+        redis.del key(:created_at)
+        redis.del key(:completed_at)
+        @created_at = @completed_at = nil
+      end
+
+      # Called by Playground to save the experiment definition.
+      def save
+        redis.setnx key(:created_at), Time.now.to_i
+        @created_at = Time.at(redis[key(:created_at)].to_i)
+      end
+
+    protected
+
+      def identity
+        @identify_block.call(Vanity.context)
+      end
+
+      def default_identify(context)
+        raise "No Vanity.context" unless context
+        raise "Vanity.context does not respond to vanity_identity" unless context.respond_to?(:vanity_identity)
+        context.vanity_identity or raise "Vanity.context.vanity_identity - no identity"
+      end
+
+      # Derived classes call this after state changes that may lead to
+      # experiment completing.
+      def check_completion!
+        if @complete_block
+          begin
+            complete! if @complete_block.call
+          rescue
+            # TODO: logging
+          end
+        end
+      end
+      
+      # Returns key for this experiment, or with an argument, return a key
+      # using the experiment as the namespace.  Examples:
+      #   key => "vanity:experiments:green_button"
+      #   key("participants") => "vanity:experiments:green_button:participants"
+      def key(name = nil)
+        name ? "#{@namespace}:#{name}" : @namespace
+      end
+
+      # Shortcut for Vanity.playground.redis
+      def redis
+        @playground.redis
+      end
+      
+    end
+  end
+end
+

data/lib/vanity/helpers.rb

@@ -0,0 +1,59 @@
+module Vanity  
+  # Helper methods available on Object.
+  #
+  # @example From ERB template
+  #   <%= ab_test(:greeting) %> <%= current_user.name %>
+  # @example From Rails controller
+  #   class AccountController < ApplicationController
+  #     def create
+  #       track! :signup
+  #       Acccount.create! params[:account]
+  #     end
+  #   end
+  # @example From ActiveRecord
+  #   class Posts < ActiveRecord::Base
+  #     after_create do |post|
+  #       track! :images if post.type == :image
+  #     end
+  #   end
+  module Helpers
+    
+    # This method returns one of the alternative values in the named A/B test.
+    #
+    # @example A/B two alternatives for a page
+    #   def index
+    #     if ab_test(:new_page) # true/false test
+    #       render action: "new_page"
+    #     else
+    #       render action: "index"
+    #     end
+    #   end
+    # @example Similar, alternative value is page name
+    #   def index
+    #     render action: ab_test(:new_page)
+    #   end
+    # @since 1.2.0
+    def ab_test(name, &block)
+      value = Vanity.playground.experiment(name).choose
+      if block
+        content = capture(value, &block)
+        block_called_from_erb?(block) ? concat(content) : content
+      else
+        value
+      end
+    end
+
+    # Tracks an action associated with a metric.
+    #
+    # @example
+    #   track! :invitation
+    # @since 1.2.0
+    def track!(name, count = 1)
+      Vanity.playground.track! name, count
+    end
+  end
+end
+
+Object.class_eval do
+  include Vanity::Helpers
+end

data/lib/vanity/metric/active_record.rb

@@ -0,0 +1,77 @@
+module Vanity
+  class Metric
+
+    AGGREGATES = [:average, :minimum, :maximum, :sum]
+
+    # Use an ActiveRecord model to get metric data from database table.  Also
+    # forwards +after_create+ callbacks to hooks (updating experiments).
+    #
+    # Supported options:
+    # :conditions -- Only select records that match this condition
+    # :average -- Metric value is average of this column
+    # :minimum -- Metric value is minimum of this column
+    # :maximum -- Metric value is maximum of this column
+    # :sum -- Metric value is sum of this column
+    # :timestamp -- Use this column to filter/group records (defaults to
+    # +created_at+)
+    #
+    # @example Track sign ups using User model
+    #   metric "Signups" do
+    #     model Account
+    #   end
+    # @example Track satisfaction using Survey model
+    #   metric "Satisfaction" do
+    #     model Survey, :average=>:rating
+    #   end
+    # @example Track only high ratings
+    #   metric "High ratings" do
+    #     model Rating, :conditions=>["stars >= 4"]
+    #   end
+    # @example Track only high ratings (using scope)
+    #   metric "High ratings" do
+    #     model Rating.high
+    #   end
+    #
+    # @since 1.2.0
+    # @see Vanity::Metric::ActiveRecord
+    def model(class_or_scope, options = nil)
+      options = options || {}
+      conditions = options.delete(:conditions)
+      @ar_scoped = conditions ? class_or_scope.scoped(:conditions=>conditions) : class_or_scope
+      @ar_aggregate = AGGREGATES.find { |key| options.has_key?(key) }
+      @ar_column = options.delete(@ar_aggregate)
+      fail "Cannot use multiple aggregates in a single metric" if AGGREGATES.find { |key| options.has_key?(key) }
+      @ar_timestamp = options.delete(:timestamp) || :created_at
+      fail "Unrecognized options: #{options.keys * ", "}" unless options.empty?
+      @ar_scoped.after_create self
+      extend ActiveRecord
+    end
+
+    # Calling model method on a metric extends it with these modules, redefining
+    # the values and track! methods.
+    #
+    # @since 1.3.0
+    module ActiveRecord
+
+      # This values method queries the database.
+      def values(sdate, edate)
+        query = { :conditions=>{ @ar_timestamp=>(sdate.to_time...(edate + 1).to_time) },
+                  :group=>"date(#{@ar_scoped.connection.quote_column_name @ar_timestamp})" }
+        grouped = @ar_column ? @ar_scoped.calculate(@ar_aggregate, @ar_column, query) : @ar_scoped.count(query)
+        (sdate..edate).inject([]) { |ordered, date| ordered << (grouped[date.to_s] || 0) }
+      end
+
+      # This track! method stores nothing, but calls the hooks.
+      def track!(*args)
+        count = args.first || 1
+        call_hooks Time.now, count if count > 0
+      end
+
+      # AR model after_create callback notifies all the hooks.
+      def after_create(record)
+        count = @ar_column ? (record.send(@ar_column) || 0) : 1
+        call_hooks record.send(@ar_timestamp), count if count > 0 && @ar_scoped.exists?(record)
+      end
+    end
+  end
+end

data/lib/vanity/metric/base.rb

@@ -0,0 +1,221 @@
+module Vanity
+
+  # A metric is an object that implements two methods: +name+ and +values+.  It
+  # can also respond to addition methods (+track!+, +bounds+, etc), these are
+  # optional.
+  #
+  # This class implements a basic metric that tracks data and stores it in
+  # Redis.  You can use this as the basis for your metric, or as reference for
+  # the methods your metric must and can implement.
+  #
+  # @since 1.1.0
+  class Metric
+
+    # These methods are available when defining a metric in a file loaded
+    # from the +experiments/metrics+ directory.
+    #
+    # For example:
+    #   $ cat experiments/metrics/yawn_sec
+    #   metric "Yawns/sec" do
+    #     description "Most boring metric ever"
+    #   end
+    module Definition
+      
+      attr_reader :playground
+
+      # Defines a new metric, using the class Vanity::Metric.
+      def metric(name, &block)
+        fail "Metric #{@metric_id} already defined in playground" if playground.metrics[@metric_id]
+        metric = Metric.new(playground, name.to_s, @metric_id)
+        metric.instance_eval &block
+        playground.metrics[@metric_id] = metric
+      end
+
+      def new_binding(playground, id)
+        @playground, @metric_id = playground, id
+        binding
+      end
+
+    end
+  
+    # Startup metrics for pirates. AARRR stands for:
+    # * Acquisition
+    # * Activation
+    # * Retention
+    # * Referral
+    # * Revenue
+    # Read more: http://500hats.typepad.com/500blogs/2007/09/startup-metrics.html
+
+    class << self
+
+      # Helper method to return description for a metric.
+      #
+      # A metric object may have a +description+ method that returns a detailed
+      # description.  It may also have no description, or no +description+
+      # method, in which case return +nil+.
+      # 
+      # @example
+      #   puts Vanity::Metric.description(metric)
+      def description(metric)
+        metric.description if metric.respond_to?(:description)
+      end
+
+      # Helper method to return bounds for a metric.
+      #
+      # A metric object may have a +bounds+ method that returns lower and upper
+      # bounds.  It may also have no bounds, or no +bounds+ # method, in which
+      # case we return +[nil, nil]+.
+      # 
+      # @example
+      #   upper = Vanity::Metric.bounds(metric).last
+      def bounds(metric)
+        metric.respond_to?(:bounds) && metric.bounds || [nil, nil]
+      end
+
+      # Returns data set for a given date range.  The data set is an array of
+      # date, value pairs.
+      #
+      # First argument is the metric.  Second argument is the start date, or
+      # number of days to go back in history, defaults to 90 days.  Third
+      # argument is end date, defaults to today.
+      #
+      # @example These are all equivalent:
+      #   Vanity::Metric.data(my_metric) 
+      #   Vanity::Metric.data(my_metric, 90) 
+      #   Vanity::Metric.data(my_metric, Date.today - 89)
+      #   Vanity::Metric.data(my_metric, Date.today - 89, Date.today)
+      def data(metric, *args)
+        first = args.shift || 90
+        to = args.shift || Date.today
+        from = first.respond_to?(:to_date) ? first.to_date : to - (first - 1)
+        (from..to).zip(metric.values(from, to))
+      end
+
+      # Playground uses this to load metric definitions.
+      def load(playground, stack, file)
+        fail "Circular dependency detected: #{stack.join('=>')}=>#{file}" if stack.include?(file)
+        source = File.read(file)
+        stack.push file
+        id = File.basename(file, ".rb").downcase.gsub(/\W/, "_").to_sym
+        context = Object.new
+        context.instance_eval do
+          extend Definition
+          metric = eval(source, context.new_binding(playground, id), file)
+          fail NameError.new("Expected #{file} to define metric #{id}", id) unless playground.metrics[id]
+          metric
+        end
+      rescue
+        error = NameError.exception($!.message, id)
+        error.set_backtrace $!.backtrace
+        raise error
+      ensure
+        stack.pop
+      end
+
+    end
+
+
+    # Takes playground (need this to access Redis), friendly name and optional
+    # id (can infer from name).
+    def initialize(playground, name, id = nil)
+      @playground, @name = playground, name.to_s
+      @id = (id || name.to_s.downcase.gsub(/\W+/, '_')).to_sym
+      @hooks = []
+      redis.setnx key(:created_at), Time.now.to_i
+      @created_at = Time.at(redis[key(:created_at)].to_i)
+    end
+
+
+    # -- Tracking --
+
+    # Called to track an action associated with this metric.
+    def track!(count = 1)
+      count ||= 1
+      if count > 0
+        timestamp = Time.now
+        redis.incrby key(timestamp.to_date, "count"), count
+        @playground.logger.info "vanity: #{@id} with count #{count}"
+        call_hooks timestamp, count
+      end
+    end
+
+    # Metric definitions use this to introduce tracking hook.  The hook is
+    # called with metric identifier, timestamp, count and possibly additional
+    # arguments.
+    #
+    # For example:
+    #   hook do |metric_id, timestamp, count|
+    #     syslog.info metric_id
+    #   end
+    def hook(&block)
+      @hooks << block
+    end
+
+    # This method returns the acceptable bounds of a metric as an array with
+    # two values: low and high.  Use nil for unbounded.
+    #
+    # Alerts are created when metric values exceed their bounds.  For example,
+    # a metric of user registration can use historical data to calculate
+    # expected range of new registration for the next day.  If actual metric
+    # falls below the expected range, it could indicate registration process is
+    # broken.  Going above higher bound could trigger opening a Champagne
+    # bottle.
+    #
+    # The default implementation returns +nil+.
+    def bounds
+    end
+    
+
+    #  -- Reporting --
+    
+    # Human readable metric name.  All metrics must implement this method.
+    attr_reader :name
+    alias :to_s :name
+
+    # Time stamp when metric was created.
+    attr_reader :created_at
+
+    # Human readable description.  Use two newlines to break paragraphs.
+    attr_accessor :description
+
+    # Sets or returns description. For example
+    #   metric "Yawns/sec" do
+    #     description "Most boring metric ever"
+    #   end
+    #
+    #   puts "Just defined: " + metric(:boring).description
+    def description(text = nil)
+      @description = text if text
+      @description
+    end
+
+    # Given two arguments, a start date and an end date (inclusive), returns an
+    # array of measurements.  All metrics must implement this method.
+    def values(from, to)
+      redis.mget((from.to_date..to.to_date).map { |date| key(date, "count") }).map(&:to_i)
+    end
+
+
+    # -- Storage --
+
+    def destroy!
+      redis.del redis.keys(key("*"))
+    end
+
+    def redis
+      @playground.redis
+    end
+
+    def key(*args)
+      "metrics:#{@id}:#{args.join(':')}"
+    end
+
+    def call_hooks(timestamp, count)
+      count ||= 1
+      @hooks.each do |hook|
+        hook.call @id, timestamp, count
+      end
+    end
+
+  end
+end