vanity 2.0.1 → 2.1.0

data/lib/vanity/adapters/abstract_adapter.rb

@@ -67,6 +67,17 @@ module Vanity
         fail "Not implemented"
       end
 
+      # Store whether an experiment is enabled or not
+      def set_experiment_enabled(experiment, enabled)
+        fail "Not implemented"
+      end
+      
+      # Returns true if experiment is enabled, the default (Vanity.configuration.experiments_start_enabled) is true.
+      # (*except for mock_adapter, where default is true for testing)
+      def is_experiment_enabled?(experiment)
+        fail "Not implemented"
+      end
+
       # Returns counts for given A/B experiment and alternative (by index).
       # Returns hash with values for the keys :participants, :converted and
       # :conversions.

data/lib/vanity/adapters/active_record_adapter.rb

@@ -71,7 +71,7 @@ module Vanity
 
         def increment_conversion(alternative, count = 1)
           record = vanity_conversions.rails_agnostic_find_or_create_by(:alternative, alternative)
-          record.increment!(:conversions, count)
+          record.class.update_counters(record.id, conversions: count)
         end
       end
 
@@ -79,6 +79,7 @@ module Vanity
       class VanityConversion < VanityRecord
         self.table_name = :vanity_conversions
         belongs_to :vanity_experiment
+        attr_accessible :alternative if needs_attr_accessible?
       end
 
       # Participant model
@@ -201,6 +202,19 @@ module Vanity
         !!VanityExperiment.retrieve(experiment).completed_at
       end
 
+      def set_experiment_enabled(experiment, enabled)
+        VanityExperiment.retrieve(experiment).update_attribute(:enabled, enabled)
+      end
+
+      def is_experiment_enabled?(experiment)
+        record = VanityExperiment.retrieve(experiment)
+        if Vanity.configuration.experiments_start_enabled
+          record.enabled != false
+        else
+          record.enabled == true
+        end
+      end
+
       # Returns counts for given A/B experiment and alternative (by index).
       # Returns hash with values for the keys :participants, :converted and
       # :conversions.

data/lib/vanity/adapters/mock_adapter.rb

@@ -95,6 +95,20 @@ module Vanity
       def is_experiment_completed?(experiment)
         @experiments[experiment] && @experiments[experiment][:completed_at]
       end
+      
+      def set_experiment_enabled(experiment, enabled)
+        @experiments[experiment] ||= {}
+        @experiments[experiment][:enabled] = enabled
+      end
+
+      def is_experiment_enabled?(experiment)
+        record = @experiments[experiment]
+        if Vanity.configuration.experiments_start_enabled
+          record == nil || record[:enabled] != false
+        else
+          record && record[:enabled] == true
+        end
+      end
 
       def ab_counts(experiment, alternative)
         @experiments[experiment] ||= {}

data/lib/vanity/adapters/mongodb_adapter.rb

@@ -114,6 +114,7 @@ module Vanity
       def get_experiment_created_at(experiment)
         record = @experiments.find_one({ :_id=>experiment }, { :fields=>[:created_at] })
         record && record["created_at"]
+        #Returns nil if either the record or the field doesn't exist
       end
 
       def set_experiment_completed_at(experiment, time)
@@ -129,6 +130,19 @@ module Vanity
         !!@experiments.find_one(:_id=>experiment, :completed_at=>{ "$exists"=>true })
       end
 
+      def set_experiment_enabled(experiment, enabled)
+        @experiments.update({ :_id=>experiment }, { "$set"=>{ :enabled=>enabled } }, :upsert=>true)
+      end
+
+      def is_experiment_enabled?(experiment)
+        record = @experiments.find_one({ :_id=>experiment}, { :fields=>[:enabled] })
+        if Vanity.configuration.experiments_start_enabled
+          record == nil || record["enabled"] != false
+        else
+          record && record["enabled"] == true
+        end
+      end
+
       def ab_counts(experiment, alternative)
         record = @experiments.find_one({ :_id=>experiment }, { :fields=>[:conversions] })
         conversions = record && record["conversions"]

data/lib/vanity/adapters/redis_adapter.rb

@@ -127,6 +127,21 @@ module Vanity
         end
       end
 
+      def set_experiment_enabled(experiment, enabled)
+        call_redis_with_failover do
+          @experiments.set "#{experiment}:enabled", enabled
+        end
+      end
+
+      def is_experiment_enabled?(experiment)
+        value = @experiments["#{experiment}:enabled"]
+        if Vanity.configuration.experiments_start_enabled
+          value != 'false'
+        else
+          value == 'true'
+        end
+      end
+
       def ab_counts(experiment, alternative)
         {
           :participants => @experiments.scard("#{experiment}:alts:#{alternative}:participants").to_i,

data/lib/vanity/configuration.rb

@@ -25,30 +25,40 @@ module Vanity
         nil
       end
 
+      # 
+      # Filter all User-Agents that have 'bot', 'crawler', 'spider', URL. 
+      # 
       def default_request_filter(request) # :nodoc:
         request &&
-          request.env &&
-          request.env["HTTP_USER_AGENT"] &&
-          request.env["HTTP_USER_AGENT"].match(/\(.*https?:\/\/.*\)/)
+        request.env &&
+        request.env["HTTP_USER_AGENT"] &&
+        request.env["HTTP_USER_AGENT"].match( /(?:https?:\/\/)|(?:bot|spider|crawler)/i )
       end
     end
 
     DEFAULTS = {
+      add_participant_route: "/vanity/add_participant",
       collecting: true,
+      config_file: "vanity.yml",
+      config_path: File.join(Pathname.new("."), "config"),
+      environment: ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development",
       experiments_path: File.join(Pathname.new("."), "experiments"),
-      add_participant_route: "/vanity/add_participant",
-      logger: default_logger,
       failover_on_datastore_error: false,
+      locales_path: File.expand_path(File.join(File.dirname(__FILE__), 'locales')),
+      logger: default_logger,
       on_datastore_error: ->(error, klass, method, arguments) {
         default_on_datastore_error(error, klass, method, arguments)
       },
       request_filter: ->(request) { default_request_filter(request) },
       templates_path: File.expand_path(File.join(File.dirname(__FILE__), 'templates')),
-      locales_path: File.expand_path(File.join(File.dirname(__FILE__), 'locales')),
       use_js: false,
-      config_path: File.join(Pathname.new("."), "config"),
-      config_file: "vanity.yml",
-      environment: ENV["RACK_ENV"] || ENV["RAILS_ENV"] || "development"
+      experiments_start_enabled: true,
+      cookie_name: 'vanity_id',
+      cookie_expires: 20 * 365 * 24 * 60 * 60, # 20 years, give or take.
+      cookie_domain: nil,
+      cookie_path: nil,
+      cookie_secure: false,
+      cookie_httponly: false,
     }.freeze
 
     # True if saving results to the datastore (participants and conversions).
@@ -101,8 +111,9 @@ module Vanity
     #    end
     #
     # The default implementation does a simple test of whether the request's
-    # HTTP_USER_AGENT header contains a URI, since well behaved bots typically
-    # include a reference URI in their user agent strings. (Original idea:
+    # HTTP_USER_AGENT header contains a URI, or the words 'bot', 'crawler', or
+    # 'spider' since well behaved bots typically include a reference URI in
+    # their user agent strings. (Original idea:
     # http://stackoverflow.com/a/9285889.)
     #
     # Alternatively, one could filter an explicit list of IPs, add additional
@@ -147,7 +158,28 @@ module Vanity
     attr_writer :config_file
     # In order of precedence, RACK_ENV, RAILS_ENV or `development`.
     attr_writer :environment
+    # By default experiments start enabled. If you want experiments to be
+    # explicitly enabled after a production release, then set to false.
+    attr_writer :experiments_start_enabled
+
+    # Cookie name. By default 'vanity_id'
+    attr_writer :cookie_name
+
+    # Cookie duration. By default 20 years.
+    attr_writer :cookie_expires
+
+    # Cookie domain.  By default nil.  If domain is nil then the domain from
+    # Rails.application.config.session_options[:domain] will be substituted.
+    attr_writer :cookie_domain
+
+    # Cookie path. By default nil.
+    attr_writer :cookie_path
+
+    # Cookie secure. If true, cookie will only be transmitted to SSL pages. By default false.
+    attr_writer :cookie_secure
 
+    # Cookie path. If true, cookie will not be available to JS. By default false.
+    attr_writer :cookie_httponly
 
     # We independently list each attr_accessor to includes docs, otherwise
     # something like DEFAULTS.each { |key, value| attr_accessor key } would be

data/lib/vanity/experiment/ab_test.rb

@@ -25,6 +25,56 @@ module Vanity
         super
         @score_method = DEFAULT_SCORE_METHOD
         @use_probabilities = nil
+        @is_default_set = false
+      end
+        
+      # -- Default --
+
+      # Call this method once to set a default alternative. Call without 
+      # arguments to obtain the current default. If default is not specified,
+      # the first alternative is used.
+      #
+      # @example Set the default alternative
+      #   ab_test "Background color" do
+      #     alternatives "red", "blue", "orange"
+      #     default "red"
+      #   end
+      # @example Get the default alternative
+      #   assert experiment(:background_color).default == "red"
+      #
+      def default(value)
+        @default = value
+        @is_default_set = true
+        class << self
+          define_method :default do |*args|
+            raise ArgumentError, "default has already been set to #{@default.inspect}" unless args.empty?
+            alternative(@default)
+          end
+        end
+        nil
+      end
+
+      # -- Enabled --
+      
+      # Returns true if experiment is enabled, false if disabled.
+      def enabled?
+        !@playground.collecting? || ( active? && connection.is_experiment_enabled?(@id) )
+      end
+      
+      # Enable or disable the experiment. Only works if the playground is collecting
+      # and this experiment is enabled.
+      #
+      # **Note** You should *not* set the enabled/disabled status of an
+      # experiment until it exists in the database. Ensure that your experiment
+      # has had #save invoked previous to any enabled= calls.
+      def enabled=(bool)
+        return unless @playground.collecting? && active?
+        if created_at.nil?
+          warn 'DB has no created_at for this experiment! This most likely means' + 
+               'you didn\'t call #save before calling enabled=, which you should.'
+        else
+          connection.set_experiment_enabled(@id, bool)
+        end
       end
 
       # -- Metric --
@@ -47,7 +97,9 @@ module Vanity
 
       # Call this method once to set alternative values for this experiment
       # (requires at least two values). Call without arguments to obtain
-      # current list of alternatives.
+      # current list of alternatives. Call with a hash to set custom
+      # probabilities.  If providing a hash of alternates, you may need to
+      # specify a default unless your hashes are ordered. (Ruby >= 1.9)
       #
       # @example Define A/B test with three alternatives
       #   ab_test "Background color" do
@@ -55,13 +107,21 @@ module Vanity
       #     alternatives "red", "blue", "orange"
       #   end
       #
+      # @example Define A/B test with custom probabilities
+      #   ab_test "Background color" do
+      #     metrics :coolness
+      #     alternatives "red" => 10, "blue" => 5, "orange => 1
+      #     default "red"
+      #   end
+      #
       # @example Find out which alternatives this test uses
       #   alts = experiment(:background_color).alternatives
       #   puts "#{alts.count} alternatives, with the colors: #{alts.map(&:value).join(", ")}"
       def alternatives(*args)
-        @alternatives ||= args.empty? ? [true, false] : args.clone
-        @alternatives.each_with_index.map do |value, i|
-          Alternative.new(self, i, value)
+        if has_alternative_weights?(args)
+          build_alternatives_with_weights(args)
+        else
+          build_alternatives(args)
         end
       end
 
@@ -126,16 +186,18 @@ module Vanity
       def choose(request=nil)
         if @playground.collecting?
           if active?
-            identity = identity()
-            index = connection.ab_showing(@id, identity) || connection.ab_assigned(@id, identity)
-            unless index
-              index = alternative_for(identity).to_i
-              save_assignment_if_valid_visitor(identity, index, request) unless @playground.using_js?
+            if enabled?
+              index = alternative_index_for_identity(request)    
+            else
+              # Show the default if experiment is disabled. 
+              index = alternatives.index(default)
             end
           else
+            # If inactive, always show the outcome. Fallback to generation if one can't be found.
             index = connection.ab_get_outcome(@id) || alternative_for(identity)
           end
         else
+          # If collecting=false, show the alternative, but don't track anything.
           identity = identity()
           @showing ||= {}
           @showing[identity] ||= alternative_for(identity)
@@ -145,7 +207,6 @@ module Vanity
         alternatives[index.to_i]
       end
 
-
       # -- Testing and JS Callback --
 
       # Forces this experiment to use a particular alternative. This may be
@@ -409,7 +470,9 @@ module Vanity
       end
 
       def complete!(outcome = nil)
+        # This statement is equivalent to: return unless collecting?
         return unless @playground.collecting? && active?
+        self.enabled = false
         super
 
         unless outcome
@@ -426,28 +489,56 @@ module Vanity
           end
         end
         # TODO: logging
-        connection.ab_set_outcome @id, outcome || 0
+        connection.ab_set_outcome(@id, outcome || 0)
       end
 
 
       # -- Store/validate --
 
       def destroy
-        connection.destroy_experiment @id
+        connection.destroy_experiment(@id)
         super
       end
+      
+      # clears all collected data for the experiment
+      def reset
+        return unless @playground.collecting?
+        connection.destroy_experiment(@id)
+        connection.set_experiment_created_at(@id, Time.now)
+        @outcome = @completed_at = nil
+        self
+      end
 
       # clears all collected data for the experiment
       def reset
-        connection.destroy_experiment @id
-        connection.set_experiment_created_at @id, Time.now
+        connection.destroy_experiment(@id)
+        connection.set_experiment_created_at(@id, Time.now)
         @outcome = @completed_at = nil
         self
       end
 
+      # Set up tracking for metrics and ensure that the attributes of the ab_test
+      # are valid (e.g. has alternatives, has a default, has metrics).
+      # If collecting, this method will also store this experiment into the db.
+      # In most cases, you call this method right after the experiment's been instantiated
+      # and declared.
       def save
+        if @saved
+          warn "Experiment #{name} has already been saved"
+          return
+        end
+        @saved = true
         true_false unless @alternatives
         fail "Experiment #{name} needs at least two alternatives" unless @alternatives.size >= 2
+        if !@is_default_set
+          default(@alternatives.first)
+          warn "No default alternative specified; choosing #{@default} as default."
+        elsif alternative(@default).nil?
+          #Specified a default that wasn't listed as an alternative; warn and override.
+          warn "Attempted to set unknown alternative #{@default} as default! Using #{@alternatives.first} instead."
+          #Set the instance variable directly since default(value) is no longer defined
+          @default = @alternatives.first
+        end
         super
         if @metrics.nil? || @metrics.empty?
           warn "Please use metrics method to explicitly state which metric you are measuring against."
@@ -461,7 +552,7 @@ module Vanity
 
       # Called via a hook by the associated metric.
       def track!(metric_id, timestamp, count, *args)
-        return unless active?
+        return unless active? && enabled?
         identity = args.last[:identity] if args.last.is_a?(Hash)
         identity ||= identity() rescue nil
         if identity
@@ -495,6 +586,18 @@ module Vanity
         end
       end
 
+      # Returns the assigned alternative, previously chosen alternative, or
+      # alternative_for for a given identity.  
+      def alternative_index_for_identity(request)
+        identity = identity()
+        index = connection.ab_showing(@id, identity) || connection.ab_assigned(@id, identity)
+        unless index
+          index = alternative_for(identity).to_i
+          save_assignment_if_valid_visitor(identity, index, request) unless @playground.using_js?
+        end
+        index
+      end
+
       # Chooses an alternative for the identity and returns its index. This
       # method always returns the same alternative for a given experiment and
       # identity, and randomly distributed alternatives for each identity (in the
@@ -550,6 +653,33 @@ module Vanity
         end
       end
 
+      def has_alternative_weights?(args)
+        @alternatives.nil? && args.size == 1 && args[0].is_a?(Hash)
+      end
+
+      def build_alternatives_with_weights(args)
+        @alternatives = args[0]
+        sum_of_probability = @alternatives.values.reduce(0) { |a,b| a+b }
+        cumulative_probability = 0.0
+        @use_probabilities = []
+        result = []
+        @alternatives = @alternatives.each_with_index.map do |(value, probability), i|
+          result << alternative = Alternative.new( self, i, value )
+          probability = probability.to_f / sum_of_probability
+          @use_probabilities << [ alternative, cumulative_probability += probability ]
+          value
+        end
+
+        result
+      end
+
+      def build_alternatives(args)
+        @alternatives ||= args.empty? ? [true, false] : args.clone
+        @alternatives.each_with_index.map do |value, i|
+          Alternative.new(self, i, value)
+        end
+      end
+
       begin
         a = 50.0
         # Returns array of [z-score, percentage]

data/lib/vanity/experiment/alternative.rb

@@ -88,6 +88,10 @@ module Vanity
           @participants = @converted = @conversions = 0
         end
       end
+
+      def default?
+        @experiment.default == self
+      end
     end
   end
 end

data/lib/vanity/frameworks/rails.rb

@@ -53,6 +53,19 @@ module Vanity
           define_method :vanity_identity do
             return @vanity_identity if @vanity_identity
 
+            cookie = lambda do |value|
+              result = {
+                value: value,
+                expires: Time.now + Vanity.configuration.cookie_expires,
+                path: Vanity.configuration.cookie_path,
+                domain: Vanity.configuration.cookie_domain,
+                secure: Vanity.configuration.cookie_secure,
+                httponly: Vanity.configuration.cookie_httponly
+              }
+              result[:domain] ||= ::Rails.application.config.session_options[:domain]
+              result
+            end
+
             # With user sign in, it's possible to visit not-logged in, get
             # cookied and shown alternative A, then sign in and based on
             # user.id, get shown alternative B.
@@ -60,18 +73,15 @@ module Vanity
             # new user.id to avoid the flash of alternative B (FOAB).
             if request.get? && params[:_identity]
               @vanity_identity = params[:_identity]
-              cookies["vanity_id"] = { :value=>@vanity_identity, :expires=>1.month.from_now }
+              cookies[Vanity.configuration.cookie_name] = cookie.call(@vanity_identity)
               @vanity_identity
-            elsif cookies["vanity_id"]
-              @vanity_identity = cookies["vanity_id"]
+            elsif cookies[Vanity.configuration.cookie_name]
+              @vanity_identity = cookies[Vanity.configuration.cookie_name]
             elsif symbol && object = send(symbol)
               @vanity_identity = object.id
             elsif response # everyday use
-              @vanity_identity = cookies["vanity_id"] || SecureRandom.hex(16)
-              cookie = { :value=>@vanity_identity, :expires=>1.month.from_now }
-              # Useful if application and admin console are on separate domains.
-              cookie[:domain] ||= ::Rails.application.config.session_options[:domain]
-              cookies["vanity_id"] = cookie
+              @vanity_identity = cookies[Vanity.configuration.cookie_name] || SecureRandom.hex(16)
+              cookies[Vanity.configuration.cookie_name] = cookie.call(@vanity_identity)
               @vanity_identity
             else # during functional testing
               @vanity_identity = "test"
@@ -284,6 +294,44 @@ module Vanity
     end
 
 
+    # When configuring use_js to true, you must set up a route to
+    # add_participant_route.
+    #
+    # Step 1: Add a new resource in config/routes.rb:
+    #   post "/vanity/add_participant" => "vanity#add_participant"
+    #
+    # Step 2: Include Vanity::Rails::AddParticipant (or Vanity::Rails::Dashboard) in VanityController
+    #   class VanityController < ApplicationController
+    #     include Vanity::Rails::AddParticipant
+    #   end
+    #
+    # Step 3: Open your browser to http://localhost:3000/vanity
+    module AddParticipant
+      # JS callback action used by vanity_js
+      def add_participant
+        if params[:v].nil?
+          render :status => 404, :nothing => true
+          return
+        end
+
+        h = {}
+        params[:v].split(',').each do |pair|
+          exp_id, answer = pair.split('=')
+          exp = Vanity.playground.experiment(exp_id.to_s.to_sym) rescue nil
+          answer = answer.to_i
+
+          if !exp || !exp.alternatives[answer]
+            render :status => 404, :nothing => true
+            return
+          end
+          h[exp] = exp.alternatives[answer].value
+        end
+
+        h.each{ |e,a| e.chooses(a, request) }
+        render :status => 200, :nothing => true
+      end
+    end
+
     # Step 1: Add a new resource in config/routes.rb:
     #   map.vanity "/vanity/:action/:id", :controller=>:vanity
     #
@@ -320,6 +368,18 @@ module Vanity
         end
       end
 
+      def disable
+        exp = Vanity.playground.experiment(params[:e].to_sym)
+        exp.enabled = false
+        render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
+      end
+      
+      def enable
+        exp = Vanity.playground.experiment(params[:e].to_sym)
+        exp.enabled = true
+        render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
+      end
+
       def chooses
         exp = Vanity.playground.experiment(params[:e].to_sym)
         exp.chooses(exp.alternatives[params[:a].to_i].value)
@@ -333,29 +393,7 @@ module Vanity
         render :file=>Vanity.template("_experiment"), :locals=>{:experiment=>exp}
       end
 
-      # JS callback action used by vanity_js
-      def add_participant
-        if params[:v].nil?
-          render :status => 404, :nothing => true
-          return
-        end
-
-        h = {}
-        params[:v].split(',').each do |pair|
-          exp_id, answer = pair.split('=')
-          exp = Vanity.playground.experiment(exp_id.to_s.to_sym) rescue nil
-          answer = answer.to_i
-
-          if !exp || !exp.alternatives[answer]
-            render :status => 404, :nothing => true
-            return
-          end
-          h[exp] = exp.alternatives[answer].value
-        end
-
-        h.each{ |e,a| e.chooses(a, request) }
-        render :status => 200, :nothing => true
-      end
+      include AddParticipant
     end
 
     module TrackingImage

data/lib/vanity/locales/vanity.en.yml

@@ -1,5 +1,8 @@
 en:
   vanity:
+    act_on_this_experiment:
+      enable: 'Enable this experiment'
+      disable: 'Disable this experiment'
     best_alternative: '(%{probability}% this is best)'
     best_alternative_is_significant: 'With %{probability}% probability this result is statistically significant.'
     best_alternative_probability: 'With %{probability}% probability this result is the best.'
@@ -14,8 +17,14 @@ en:
     converted: '%{count} converted'
     converted_percentage: '%{alternative} converted at %{percentage}%.'
     currently_shown: 'currently shown'
+    default: '(Default)'
     didnt_convert: '%{alternative} did not convert.'
+    disable: 'Disable'
+    disabled: 'Disabled'
+    enable: 'Enable'
+    enabled: 'Enabled'
     experiment_has_been_reset: '%{name} experiment has been reset.'
+    experiment_has_been_disabled: 'This experiment is currently disabled, and will always choose the default %{name}.'
     experiment_participants:
       one: 'There is one participant in this experiment.'
       other: 'There are %{count} participants in this experiment.'

data/lib/vanity/locales/vanity.pt-BR.yml

@@ -17,6 +17,10 @@ pt-BR:
     converted_percentage: '%{alternative} converteu %{percentage}%'
     currently_shown: 'atualmente exibido'
     didnt_convert: '%{alternative} não converteu.'
+    disable: 'Desativar'
+    disabled: 'Desativado'
+    enable: 'Activar'
+    enabled: 'Activado'
     experiment_participants:
       one: 'Há um participante neste experimento.'
       other: 'Há %{count} participantes neste experimento.'