vanity 1.8.4 → 1.9.0.beta

data/lib/vanity/adapters/redis_adapter.rb

@@ -5,15 +5,29 @@ module Vanity
       #
       # @since 1.4.0
       def redis_connection(spec)
+        require "redis"
+        fail "redis >= 2.1 is required" unless valid_redis_version?
         require "redis/namespace"
+        fail "redis-namespace >= 1.1.0 is required" unless valid_redis_namespace_version?
+
         RedisAdapter.new(spec)
       end
+
+      def valid_redis_version?
+        Gem.loaded_specs['redis'].version >= Gem::Version.create('2.1')
+      end
+
+      def valid_redis_namespace_version?
+        Gem.loaded_specs['redis'].version >= Gem::Version.create('1.1.0')
+      end
     end
 
     # Redis adapter.
     #
     # @since 1.4.0
     class RedisAdapter < AbstractAdapter
+      attr_reader :redis
+
       def initialize(options)
         @options = options.clone
         @options[:db] ||= @options[:database] || (@options[:path] && @options.delete(:path).split("/")[1].to_i)
@@ -43,34 +57,32 @@ module Vanity
 
       def connect!
         @redis = @options[:redis] || Redis.new(@options)
-        @metrics = Redis::Namespace.new("vanity:metrics", :redis=>@redis)
-        @experiments = Redis::Namespace.new("vanity:experiments", :redis=>@redis)
+        @metrics = Redis::Namespace.new("vanity:metrics", :redis=>redis)
+        @experiments = Redis::Namespace.new("vanity:experiments", :redis=>redis)
       end
 
       def to_s
         redis.id
       end
 
-      def redis
-        @redis
-      end
-
       def flushdb
         @redis.flushdb
       end
 
       # -- Metrics --
-      
+
       def get_metric_last_update_at(metric)
         last_update_at = @metrics["#{metric}:last_update_at"]
         last_update_at && Time.at(last_update_at.to_i)
       end
 
       def metric_track(metric, timestamp, identity, values)
-        values.each_with_index do |v,i|
-          @metrics.incrby "#{metric}:#{timestamp.to_date}:value:#{i}", v
+        call_redis_with_failover(metric, timestamp, identity, values) do
+          values.each_with_index do |v,i|
+            @metrics.incrby "#{metric}:#{timestamp.to_date}:value:#{i}", v
+          end
+          @metrics["#{metric}:last_update_at"] = Time.now.to_i
         end
-        @metrics["#{metric}:last_update_at"] = Time.now.to_i
       end
 
       def metric_values(metric, from, to)
@@ -84,9 +96,15 @@ module Vanity
 
 
       # -- Experiments --
-     
+
+      def experiment_persisted?(experiment)
+        !!@experiments["#{experiment}:created_at"]
+      end
+
       def set_experiment_created_at(experiment, time)
-        @experiments.setnx "#{experiment}:created_at", time.to_i
+        call_redis_with_failover do
+          @experiments.setnx "#{experiment}:created_at", time.to_i
+        end
       end
 
       def get_experiment_created_at(experiment)
@@ -104,58 +122,76 @@ module Vanity
       end
 
       def is_experiment_completed?(experiment)
-        @experiments.exists("#{experiment}:completed_at")
+        call_redis_with_failover do
+          @experiments.exists("#{experiment}:completed_at")
+        end
       end
 
       def ab_counts(experiment, alternative)
-        { :participants => @experiments.scard("#{experiment}:alts:#{alternative}:participants").to_i,
-          :converted    => @experiments.scard("#{experiment}:alts:#{alternative}:converted").to_i,
-          :conversions  => @experiments["#{experiment}:alts:#{alternative}:conversions"].to_i }
+        {
+          :participants => @experiments.scard("#{experiment}:alts:#{alternative}:participants").to_i,
+                :converted    => @experiments.scard("#{experiment}:alts:#{alternative}:converted").to_i,
+          :conversions  => @experiments["#{experiment}:alts:#{alternative}:conversions"].to_i
+        }
       end
 
       def ab_show(experiment, identity, alternative)
-        @experiments["#{experiment}:participant:#{identity}:show"] = alternative
+        call_redis_with_failover do
+          @experiments["#{experiment}:participant:#{identity}:show"] = alternative
+        end
       end
 
       def ab_showing(experiment, identity)
-        alternative = @experiments["#{experiment}:participant:#{identity}:show"]
-        alternative && alternative.to_i
+        call_redis_with_failover do
+          alternative = @experiments["#{experiment}:participant:#{identity}:show"]
+          alternative && alternative.to_i
+        end
       end
 
       def ab_not_showing(experiment, identity)
-        @experiments.del "#{experiment}:participant:#{identity}:show"
+        call_redis_with_failover do
+          @experiments.del "#{experiment}:participant:#{identity}:show"
+        end
       end
 
       def ab_add_participant(experiment, alternative, identity)
-        @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
+  call_redis_with_failover(experiment, alternative, identity) do
+    @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
+  end
       end
 
       def ab_seen(experiment, identity, alternative)
-	if @experiments.sismember "#{experiment}:alts:#{alternative.id}:participants", identity
-	  return alternative
-	else
-	  return nil
-	end
+        call_redis_with_failover(experiment, identity, alternative) do
+          if @experiments.sismember "#{experiment}:alts:#{alternative.id}:participants", identity
+            alternative
+          else
+            nil
+          end
+        end
       end
 
       # Returns the participant's seen alternative in this experiment, if it exists
       def ab_assigned(experiment, identity)
-        Vanity.playground.experiments[experiment].alternatives.each do |alternative|
-          if @experiments.sismember "#{experiment}:alts:#{alternative.id}:participants", identity
-            return alternative.id
-          end
+        call_redis_with_failover do
+          Vanity.playground.experiments[experiment].alternatives.each do |alternative|
+            if @experiments.sismember "#{experiment}:alts:#{alternative.id}:participants", identity
+              return alternative.id
+            end
+                end
+          nil
         end
-        return nil
       end
 
       def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
-        if implicit
-          @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
-        else
-          participating = @experiments.sismember("#{experiment}:alts:#{alternative}:participants", identity)
+        call_redis_with_failover(experiment, alternative, identity, count, implicit) do
+          if implicit
+            @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
+          else
+            participating = @experiments.sismember("#{experiment}:alts:#{alternative}:participants", identity)
+          end
+          @experiments.sadd "#{experiment}:alts:#{alternative}:converted", identity if implicit || participating
+          @experiments.incrby "#{experiment}:alts:#{alternative}:conversions", count
         end
-        @experiments.sadd "#{experiment}:alts:#{alternative}:converted", identity if implicit || participating
-        @experiments.incrby "#{experiment}:alts:#{alternative}:conversions", count
       end
 
       def ab_get_outcome(experiment)
@@ -173,6 +209,21 @@ module Vanity
         @experiments.del *alternatives unless alternatives.empty?
       end
 
+      protected
+
+      def call_redis_with_failover(*arguments)
+        calling_method = caller[0][/`.*'/][1..-2]
+        begin
+          yield
+        rescue => e
+          if Vanity.playground.failover_on_datastore_error?
+            Vanity.playground.on_datastore_error.call(e, self.class, calling_method, arguments)
+          else
+            raise e
+          end
+        end
+      end
+
     end
   end
 end

data/lib/vanity/commands/report.rb

@@ -7,15 +7,15 @@ module Vanity
   # outside Rails).
   module Render
 
-    # Render the named template.  Used for reporting and the dashboard.
+    # Render the named template. Used for reporting and the dashboard.
     def render(path_or_options, locals = {})
       if path_or_options.respond_to?(:keys)
-	render_erb(
-	  path_or_options[:file] || path_or_options[:partial],
-	  path_or_options[:locals]
-	)
+        render_erb(
+          path_or_options[:file] || path_or_options[:partial],
+          path_or_options[:locals]
+        )
       else
-	render_erb(path_or_options, locals)
+       render_erb(path_or_options, locals)
       end
     end
 
@@ -54,9 +54,9 @@ module Vanity
 
     def partialize(template_name)
       if template_name[0] != '_'
-	"_#{template_name}"
+        "_#{template_name}"
       else
-	template_name
+        template_name
       end
     end
   end
@@ -66,7 +66,7 @@ module Vanity
     class << self
       include Render
 
-      # Generate an HTML report.  Outputs to the named file, or stdout with no
+      # Generate an HTML report. Outputs to the named file, or stdout with no
       # arguments.
       def report(output = nil)
         html = render(Vanity.template("report"))

data/lib/vanity/experiment/ab_test.rb

@@ -7,23 +7,23 @@ module Vanity
     # The meat.
     class AbTest < Base
       class << self
-      	# Convert z-score to probability.
-      	def probability(score)
-      	  score = score.abs
-      	  probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
-      	  probability ? probability.last : 0
-      	end
+        # Convert z-score to probability.
+        def probability(score)
+          score = score.abs
+          probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
+          probability ? probability.last : 0
+        end
 
-      	def friendly_name
-      	  "A/B Test"
-      	end
+        def friendly_name
+          "A/B Test"
+        end
       end
 
       DEFAULT_SCORE_METHOD = :z_score
 
       def initialize(*args)
         super
-      	@score_method = DEFAULT_SCORE_METHOD
+        @score_method = DEFAULT_SCORE_METHOD
         @use_probabilities = nil
       end
 
@@ -46,7 +46,7 @@ module Vanity
       # -- Alternatives --
 
       # Call this method once to set alternative values for this experiment
-      # (requires at least two values).  Call without arguments to obtain
+      # (requires at least two values). Call without arguments to obtain
       # current list of alternatives.
       #
       # @example Define A/B test with three alternatives
@@ -84,7 +84,7 @@ module Vanity
         alternatives.find { |alt| alt.value == value }
       end
 
-      # What method to use for calculating score.  Default is :ab_test, but can
+      # What method to use for calculating score. Default is :ab_test, but can
       # also be set to :bayes_bandit_score to calculate probability of each
       # alternative being the best.
       #
@@ -95,13 +95,13 @@ module Vanity
       #   score_method :bayes_bandit_score
       # end
       def score_method(method=nil)
-      	if method
-      	  @score_method = method
-      	end
-      	@score_method
+        if method
+          @score_method = method
+        end
+        @score_method
       end
 
-      # Defines an A/B test with two alternatives: false and true.  This is the
+      # Defines an A/B test with two alternatives: false and true. This is the
       # default pair of alternatives, so just syntactic sugar for those who love
       # being explicit.
       #
@@ -116,42 +116,31 @@ module Vanity
       end
       alias true_false false_true
 
-      # Chooses a value for this experiment.  You probably want to use the
+      # Returns fingerprint (hash) for given alternative. Can be used to lookup
+      # alternative for experiment without revealing what values are available
+      # (e.g. choosing alternative from HTTP query parameter).
+      def fingerprint(alternative)
+        Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
+      end
+
+      # Chooses a value for this experiment. You probably want to use the
       # Rails helper method ab_test instead.
       #
       # This method picks an alternative for the current identity and returns
-      # the alternative's value.  It will consistently choose the same
+      # the alternative's value. It will consistently choose the same
       # alternative for the same identity, and randomly split alternatives
       # between different identities.
       #
       # @example
       #   color = experiment(:which_blue).choose
-      def choose
+      def choose(request=nil)
         if @playground.collecting?
           if active?
             identity = identity()
             index = connection.ab_showing(@id, identity)
             unless index
-              index = alternative_for(identity)
-              if !@playground.using_js?
-                # if we have an on_assignment block, call it on new assignments
-                if @on_assignment_block
-                  assignment = alternatives[index.to_i]
-                  if !connection.ab_seen @id, identity, assignment
-                    @on_assignment_block.call(Vanity.context, identity, assignment, self)
-                  end
-                end
-                # if we are rebalancing probabilities, keep track of how long it has been since we last rebalanced
-                if @rebalance_frequency
-                  @assignments_since_rebalancing += 1
-                  if @assignments_since_rebalancing >= @rebalance_frequency
-                    @assignments_since_rebalancing = 0
-                    rebalance!
-                  end
-                end
-                connection.ab_add_participant @id, index, identity
-                check_completion!
-              end
+              index = alternative_for(identity).to_i
+              save_assignment_if_valid_visitor(identity, index, request) unless @playground.using_js?
             end
           else
             index = connection.ab_get_outcome(@id) || alternative_for(identity)
@@ -165,22 +154,17 @@ module Vanity
         alternatives[index.to_i]
       end
 
-      # Returns fingerprint (hash) for given alternative.  Can be used to lookup
-      # alternative for experiment without revealing what values are available
-      # (e.g. choosing alternative from HTTP query parameter).
-      def fingerprint(alternative)
-        Digest::MD5.hexdigest("#{id}:#{alternative.id}")[-10,10]
-      end
-
 
-      # -- Testing --
+      # -- Testing and JS Callback --
 
-      # Forces this experiment to use a particular alternative.  You'll want to
-      # use this from your test cases to test for the different alternatives.
+      # Forces this experiment to use a particular alternative. This may be
+      # used in test cases to force a specific alternative to obtain a
+      # deterministic test. This method also is used in the add_participant
+      # callback action when adding participants via vanity_js.
       #
       # @example Setup test to red button
       #   setup do
-      #     experiment(:button_color).select(:red)
+      #     experiment(:button_color).chooses(:red)
       #   end
       #
       #   def test_shows_red_button
@@ -189,23 +173,20 @@ module Vanity
       #
       # @example Use nil to clear selection
       #   teardown do
-      #     experiment(:green_button).select(nil)
+      #     experiment(:green_button).chooses(nil)
       #   end
-      def chooses(value)
+      def chooses(value, request=nil)
         if @playground.collecting?
           if value.nil?
             connection.ab_not_showing @id, identity
           else
             index = @alternatives.index(value)
-            #add them to the experiment unless they are already in it
-            unless index == connection.ab_showing(@id, identity)
-              connection.ab_add_participant @id, index, identity
-              check_completion!
-            end
+            save_assignment_if_valid_visitor(identity, index, request)
+
             raise ArgumentError, "No alternative #{value.inspect} for #{name}" unless index
             if (connection.ab_showing(@id, identity) && connection.ab_showing(@id, identity) != index) ||
               alternative_for(identity) != index
-              connection.ab_show @id, identity, index
+              connection.ab_show(@id, identity, index)
             end
           end
         else
@@ -230,14 +211,14 @@ module Vanity
       # -- Reporting --
 
       def calculate_score
-      	if respond_to?(score_method)
-      	  self.send(score_method)
-      	else
-      	  score
-      	end
+        if respond_to?(score_method)
+          self.send(score_method)
+        else
+          score
+        end
       end
 
-      # Scores alternatives based on the current tracking data.  This method
+      # Scores alternatives based on the current tracking data. This method
       # returns a structure with the following attributes:
       # [:alts]   Ordered list of alternatives, populated with scoring info.
       # [:base]   Second best performing alternative.
@@ -278,7 +259,7 @@ module Vanity
         # choice alternative can only pick best if we have high probability (>90%).
         best = sorted.last if sorted.last.measure > 0.0
         choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
-      	Struct.new(:alts, :best, :base, :least, :choice, :method).new(alts, best, base, least, choice, :score)
+        Struct.new(:alts, :best, :base, :least, :choice, :method).new(alts, best, base, least, choice, :score)
       end
 
       # Scores alternatives based on the current tracking data, using Bayesian
@@ -302,24 +283,24 @@ module Vanity
       # The choice alternative is set only if its probability is higher or
       # equal to the specified probability (default is 90%).
       def bayes_bandit_score(probability = 90)
-      	begin
-      	  require "backports/1.9.1/kernel/define_singleton_method" if RUBY_VERSION < "1.9"
-      	  require "integration"
-      	  require "rubystats"
-      	rescue LoadError
-      	  fail "to use bayes_bandit_score, install integration and rubystats gems"
-      	end
+        begin
+          require "backports/1.9.1/kernel/define_singleton_method" if RUBY_VERSION < "1.9"
+          require "integration"
+          require "rubystats"
+        rescue LoadError
+          fail "to use bayes_bandit_score, install integration and rubystats gems"
+        end
 
-      	begin
-      	  require "gsl"
-      	rescue LoadError
-      	  warn "for better integration performance, install gsl gem"
-      	end
+        begin
+          require "gsl"
+        rescue LoadError
+          warn "for better integration performance, install gsl gem"
+        end
 
-      	BayesianBanditScore.new(alternatives, outcome).calculate!
+        BayesianBanditScore.new(alternatives, outcome).calculate!
       end
 
-      # Use the result of #score or #bayes_bandit_score to derive a conclusion.  Returns an
+      # Use the result of #score or #bayes_bandit_score to derive a conclusion. Returns an
       # array of claims.
       def conclusion(score = score)
         claims = []
@@ -338,24 +319,24 @@ module Vanity
           # we want a result that's clearly better than 2nd best.
           best, second = sorted[0], sorted[1]
           if best.measure > second.measure
-      	    diff = ((best.measure - second.measure) / second.measure * 100).round
-      	    better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
-      	    claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
-      	    if score.method == :bayes_bandit_score
-      	      if best.probability >= 90
-      		      claims << "With %d%% probability this result is the best." % score.best.probability
-      	      else
-      		      claims << "This result does not have strong confidence behind it, suggest you continue this experiment."
-      	      end
-      	    else
-      	      if best.probability >= 90
-      		      claims << "With %d%% probability this result is statistically significant." % score.best.probability
-      	      else
-      		      claims << "This result is not statistically significant, suggest you continue this experiment."
-      	      end
-      	    end
-      	    sorted.delete best
-      	  end
+            diff = ((best.measure - second.measure) / second.measure * 100).round
+            better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
+            claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
+            if score.method == :bayes_bandit_score
+              if best.probability >= 90
+                claims << "With %d%% probability this result is the best." % score.best.probability
+              else
+                claims << "This result does not have strong confidence behind it, suggest you continue this experiment."
+              end
+            else
+              if best.probability >= 90
+                claims << "With %d%% probability this result is statistically significant." % score.best.probability
+              else
+                claims << "This result is not statistically significant, suggest you continue this experiment."
+              end
+            end
+            sorted.delete best
+          end
           sorted.each do |alt|
             if alt.measure > 0.0
               claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
@@ -413,7 +394,7 @@ module Vanity
       # Defines how the experiment can choose the optimal outcome on completion.
       #
       # By default, Vanity will take the best alternative (highest conversion
-      # rate) and use that as the outcome.  You experiment may have different
+      # rate) and use that as the outcome. You experiment may have different
       # needs, maybe you want the least performing alternative, or factor cost
       # in the equation?
       #
@@ -440,7 +421,7 @@ module Vanity
         return unless @playground.collecting? && active?
         super
 
-      	unless outcome
+        unless outcome
           if @outcome_is
             begin
               result = @outcome_is.call
@@ -530,6 +511,44 @@ module Vanity
         return Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
       end
 
+      # Saves the assignment of an alternative to a person and performs the
+      # necessary housekeeping. Ignores repeat identities and filters using
+      # Playground#request_filter.
+      def save_assignment_if_valid_visitor(identity, index, request)
+        return if index == connection.ab_showing(@id, identity) || filter_visitor?(request)
+
+        call_on_assignment_if_available(identity, index)
+        rebalance_if_necessary!
+
+        connection.ab_add_participant(@id, index, identity)
+        check_completion!
+      end
+
+      def filter_visitor?(request)
+        @playground.request_filter.call(request)
+      end
+
+      def call_on_assignment_if_available(identity, index)
+        # if we have an on_assignment block, call it on new assignments
+        if @on_assignment_block
+          assignment = alternatives[index]
+          if !connection.ab_seen @id, identity, assignment
+            @on_assignment_block.call(Vanity.context, identity, assignment, self)
+          end
+        end
+      end
+
+      def rebalance_if_necessary!
+        # if we are rebalancing probabilities, keep track of how long it has been since we last rebalanced
+        if @rebalance_frequency
+          @assignments_since_rebalancing += 1
+          if @assignments_since_rebalancing >= @rebalance_frequency
+            @assignments_since_rebalancing = 0
+            rebalance!
+          end
+        end
+      end
+
       begin
         a = 50.0
         # Returns array of [z-score, percentage]
@@ -543,7 +562,7 @@ module Vanity
 
 
     module Definition
-      # Define an A/B test with the given name.  For example:
+      # Define an A/B test with the given name. For example:
       #   ab_test "New Banner" do
       #     alternatives :red, :green, :blue
       #   end