vanity 1.8.2 → 1.8.3.beta

data/.gitignore

@@ -1,10 +1,19 @@
-.bundle
-.yardoc
-*.gem
-html
-test/myapp/log/
 vendor
 log/*
+test/myapp/log/
 test/dummy/db/*.sqlite3
 test/dummy/log/*.log
 test/dummy/tmp/
+
+# Documentation cache and generated files
+.yardoc
+html
+
+# Generated files
+*.gem
+
+# Environment normalisation
+.bundle
+.rvmrc
+.ruby-version
+.ruby-gemset

data/Gemfile

@@ -14,6 +14,11 @@ gem "mongo"
 gem "mysql"
 gem "pg"
 
+# Math libraries
+gem "backports", :platforms => :mri_18
+gem "integration"
+gem "rubystats"
+
 # APIs
 gem "garb"
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    vanity (1.8.2)
-      redis (~> 2.0)
-      redis-namespace (~> 1.0.0)
+    vanity (1.8.3.beta)
+      redis (>= 2.1)
+      redis-namespace (>= 1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -26,6 +26,7 @@ GEM
     appraisal (0.5.2)
       bundler
       rake
+    backports (3.3.5)
     bson (1.6.0)
     bson_ext (1.6.0)
       bson (= 1.6.0)
@@ -38,6 +39,7 @@ GEM
     garb (0.9.1)
       activesupport (>= 2.2.0)
       crack (>= 0.1.6)
+    integration (0.1.0)
     jekyll (0.11.2)
       albino (~> 1.3)
       classifier (~> 1.3)
@@ -70,9 +72,10 @@ GEM
       activesupport (= 2.3.14)
       rake (>= 0.8.3)
     rake (10.1.0)
-    redis (2.2.2)
-    redis-namespace (1.0.4)
-      redis (< 3.0.0)
+    redis (3.0.6)
+    redis-namespace (1.3.2)
+      redis (~> 3.0.4)
+    rubystats (0.2.3)
     shoulda (3.0.1)
       shoulda-context (~> 1.0.0)
       shoulda-matchers (~> 1.0.0)
@@ -92,8 +95,10 @@ DEPENDENCIES
   RedCloth
   SystemTimer (= 1.2.3)
   appraisal
+  backports
   bson_ext
   garb
+  integration
   jekyll
   mocha
   mongo
@@ -103,6 +108,7 @@ DEPENDENCIES
   rack
   rails (~> 2.3.8)
   rake
+  rubystats
   shoulda
   timecop
   vanity!

data/gemfiles/rails3.gemfile

@@ -7,6 +7,9 @@ gem "bson_ext"
 gem "mongo"
 gem "mysql"
 gem "pg"
+gem "backports", :platforms=>:mri_18
+gem "integration"
+gem "rubystats"
 gem "garb"
 gem "SystemTimer", "1.2.3", :platforms=>:mri_18
 gem "appraisal"

data/gemfiles/rails3.gemfile.lock

@@ -8,8 +8,8 @@ PATH
   remote: /Users/phill/Development/ruby/vanity
   specs:
     vanity (1.8.2)
-      redis (~> 2.0)
-      redis-namespace (~> 1.0.0)
+      redis (>= 2.1)
+      redis-namespace (>= 1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -47,6 +47,7 @@ GEM
       bundler
       rake
     arel (2.0.10)
+    backports (3.3.5)
     bson (1.6.0)
     bson_ext (1.6.0)
       bson (= 1.6.0)
@@ -59,6 +60,7 @@ GEM
       activesupport (>= 2.2.0)
       crack (>= 0.1.6)
     i18n (0.5.0)
+    integration (0.1.0)
     json (1.6.5)
     mail (2.2.19)
       activesupport (>= 2.3.6)
@@ -101,9 +103,10 @@ GEM
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
-    redis (2.2.2)
-    redis-namespace (1.0.4)
-      redis (< 3.0.0)
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
+    rubystats (0.2.3)
     shoulda (3.0.1)
       shoulda-context (~> 1.0.0)
       shoulda-matchers (~> 1.0.0)
@@ -125,9 +128,11 @@ PLATFORMS
 DEPENDENCIES
   SystemTimer (= 1.2.3)
   appraisal
+  backports
   bson_ext
   fastthread!
   garb
+  integration
   mocha
   mongo
   mysql
@@ -135,6 +140,7 @@ DEPENDENCIES
   pg
   rack
   rails (= 3.0.11)
+  rubystats
   shoulda
   timecop
   vanity!

data/gemfiles/rails31.gemfile

@@ -7,6 +7,9 @@ gem "bson_ext"
 gem "mongo"
 gem "mysql"
 gem "pg"
+gem "backports", :platforms=>:mri_18
+gem "integration"
+gem "rubystats"
 gem "garb"
 gem "SystemTimer", "1.2.3", :platforms=>:mri_18
 gem "appraisal"

data/gemfiles/rails31.gemfile.lock

@@ -8,8 +8,8 @@ PATH
   remote: /Users/phill/Development/ruby/vanity
   specs:
     vanity (1.8.2)
-      redis (~> 2.0)
-      redis-namespace (~> 1.0.0)
+      redis (>= 2.1)
+      redis-namespace (>= 1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -48,6 +48,7 @@ GEM
       bundler
       rake
     arel (2.2.3)
+    backports (3.3.5)
     bson (1.6.0)
     bson_ext (1.6.0)
       bson (= 1.6.0)
@@ -60,6 +61,7 @@ GEM
       crack (>= 0.1.6)
     hike (1.2.1)
     i18n (0.6.0)
+    integration (0.1.0)
     json (1.6.5)
     mail (2.3.0)
       i18n (>= 0.4.0)
@@ -107,16 +109,16 @@ GEM
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
-    redis (2.2.2)
-    redis-namespace (1.0.4)
-      redis (< 3.0.0)
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
+    rubystats (0.2.3)
     shoulda (3.0.1)
       shoulda-context (~> 1.0.0)
       shoulda-matchers (~> 1.0.0)
     shoulda-context (1.0.0)
     shoulda-matchers (1.0.0)
     sprockets (2.0.3)
-      hike (~> 1.2)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
     thor (0.14.6)
@@ -136,9 +138,11 @@ PLATFORMS
 DEPENDENCIES
   SystemTimer (= 1.2.3)
   appraisal
+  backports
   bson_ext
   fastthread!
   garb
+  integration
   mocha
   mongo
   mysql
@@ -146,6 +150,7 @@ DEPENDENCIES
   pg
   rack
   rails (= 3.1.3)
+  rubystats
   shoulda
   timecop
   vanity!

data/gemfiles/rails32.gemfile

@@ -7,6 +7,9 @@ gem "bson_ext"
 gem "mongo"
 gem "mysql"
 gem "pg"
+gem "backports", :platforms=>:mri_18
+gem "integration"
+gem "rubystats"
 gem "garb"
 gem "SystemTimer", "1.2.3", :platforms=>:mri_18
 gem "appraisal"

data/gemfiles/rails32.gemfile.lock

@@ -8,8 +8,8 @@ PATH
   remote: /Users/phill/Development/ruby/vanity
   specs:
     vanity (1.8.2)
-      redis (~> 2.0)
-      redis-namespace (~> 1.0.0)
+      redis (>= 2.1)
+      redis-namespace (>= 1.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -47,6 +47,7 @@ GEM
       bundler
       rake
     arel (3.0.2)
+    backports (3.3.5)
     bson (1.6.0)
     bson_ext (1.6.0)
       bson (= 1.6.0)
@@ -59,6 +60,7 @@ GEM
       crack (>= 0.1.6)
     hike (1.2.1)
     i18n (0.6.0)
+    integration (0.1.0)
     journey (1.0.3)
     json (1.6.5)
     mail (2.4.1)
@@ -105,9 +107,10 @@ GEM
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
-    redis (2.2.2)
-    redis-namespace (1.0.4)
-      redis (< 3.0.0)
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
+    rubystats (0.2.3)
     shoulda (3.0.1)
       shoulda-context (~> 1.0.0)
       shoulda-matchers (~> 1.0.0)
@@ -134,9 +137,11 @@ PLATFORMS
 DEPENDENCIES
   SystemTimer (= 1.2.3)
   appraisal
+  backports
   bson_ext
   fastthread!
   garb
+  integration
   mocha
   mongo
   mysql
@@ -144,6 +149,7 @@ DEPENDENCIES
   pg
   rack
   rails (= 3.2.1)
+  rubystats
   shoulda
   timecop
   vanity!

data/lib/vanity/adapters/abstract_adapter.rb

@@ -38,15 +38,15 @@ module Vanity
       # Empty the database. This is used during tests.
       def flushdb
       end
-      
+
 
       # -- Metrics --
-     
+
       # Return when metric was last updated.
       def get_metric_last_update_at(metric)
         fail "Not implemented"
       end
-  
+
       # Track metric data.
       def metric_track(metric, timestamp, identity, values)
         fail "Not implemented"
@@ -66,7 +66,7 @@ module Vanity
 
       # -- Experiments --
 
-      # Store when experiment was created (do not write over existing value). 
+      # Store when experiment was created (do not write over existing value).
       def set_experiment_created_at(experiment, time)
         fail "Not implemented"
       end
@@ -75,8 +75,8 @@ module Vanity
       def get_experiment_created_at(experiment)
         fail "Not implemented"
       end
-     
-      # Returns true if experiment completed. 
+
+      # Returns true if experiment completed.
       def is_experiment_completed?(experiment)
         fail "Not implemented"
       end
@@ -86,7 +86,7 @@ module Vanity
       # :conversions.
       def ab_counts(experiment, alternative)
         fail "Not implemented"
-      end 
+      end
 
       # Pick particular alternative (by index) to show to this particular
       # participant (by identity).
@@ -112,7 +112,12 @@ module Vanity
 
       # Determines if a participant already has seen this alternative in this experiment.
       def ab_seen(experiment, identity, assignment)
-	fail "Not implemented"
+        fail "Not implemented"
+      end
+
+      # Determines what alternative a participant has already been given, if any
+      def ab_assigned(experiment, identity)
+        fail "Not implemented"
       end
 
       # Records a conversion in this experiment for the given alternative.

data/lib/vanity/adapters/active_record_adapter.rb

@@ -222,6 +222,12 @@ module Vanity
       	participant && participant.seen == alternative.id
       end
 
+      # Returns the participant's seen alternative in this experiment, if it exists
+      def ab_assigned(experiment, identity)
+        participant = VanityParticipant.retrieve(experiment, identity, false)
+        participant && participant.seen
+      end
+
       # Records a conversion in this experiment for the given alternative.
       # Associates a value with the conversion (default to 1). If implicit is
       # true, add participant if not already recorded for this experiment. If

data/lib/vanity/adapters/mongodb_adapter.rb

@@ -156,6 +156,12 @@ module Vanity
 	participant && participant["seen"].first == alternative.id
       end
 
+      # Returns the participant's seen alternative in this experiment, if it exists
+      def ab_assigned(experiment, identity)
+        participant = @participants.find_one({ :experiment=>experiment, :identity=>identity }, { :fields=>[:seen] })
+        participant && participant["seen"].first
+      end
+
       def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
         if implicit
           @participants.update({ :experiment=>experiment, :identity=>identity }, { "$push"=>{ :seen=>alternative } }, :upsert=>true)

data/lib/vanity/adapters/redis_adapter.rb

@@ -138,6 +138,16 @@ module Vanity
 	end
       end
 
+      # Returns the participant's seen alternative in this experiment, if it exists
+      def ab_assigned(experiment, identity)
+        Vanity.playground.experiments[experiment].alternatives.each do |alternative|
+          if @experiments.sismember "#{experiment}:alts:#{alternative.id}:participants", identity
+            return alternative.id
+          end
+        end
+        return nil
+      end
+
       def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
         if implicit
           @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity

data/lib/vanity/backport.rb

@@ -19,8 +19,7 @@ class Date
   unless method_defined?(:to_time)
     # Backported from Ruby 1.9.
     def to_time
-      Time.local(year, mon, mday) 
+      Time.local(year, mon, mday)
     end
   end
 end
-

data/lib/vanity/experiment/ab_test.rb

@@ -1,116 +1,32 @@
 require "digest/md5"
+require "vanity/experiment/alternative"
+require "vanity/experiment/bayesian_bandit_score"
 
 module Vanity
   module Experiment
+    # The meat.
+    class AbTest < Base
+      class << self
+      	# Convert z-score to probability.
+      	def probability(score)
+      	  score = score.abs
+      	  probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
+      	  probability ? probability.last : 0
+      	end
 
-    # One of several alternatives in an A/B test (see AbTest#alternatives).
-    class Alternative
-
-      def initialize(experiment, id, value) #, participants, converted, conversions)
-        @experiment = experiment
-        @id = id
-        @name = "option #{(@id + 65).chr}"
-        @value = value
-      end
-
-      # Alternative id, only unique for this experiment.
-      attr_reader :id
-
-      # Alternative name (option A, option B, etc).
-      attr_reader :name
-
-      # Alternative value.
-      attr_reader :value
-
-      # Experiment this alternative belongs to.
-      attr_reader :experiment
-
-      # Number of participants who viewed this alternative.
-      def participants
-        load_counts unless @participants
-        @participants
-      end
-
-      # Number of participants who converted on this alternative (a participant is counted only once).
-      def converted
-        load_counts unless @converted
-        @converted
-      end
-
-      # Number of conversions for this alternative (same participant may be counted more than once).
-      def conversions
-        load_counts unless @conversions
-        @conversions
+      	def friendly_name
+      	  "A/B Test"
+      	end
       end
 
-      # Z-score for this alternative, related to 2nd-best performing alternative. Populated by AbTest#score.
-      attr_accessor :z_score
-
-      # Probability derived from z-score. Populated by AbTest#score.
-      attr_accessor :probability
-
-      # Difference from least performing alternative. Populated by AbTest#score.
-      attr_accessor :difference
-
-      # Conversion rate calculated as converted/participants
-      def conversion_rate
-        @conversion_rate ||= (participants > 0 ? converted.to_f/participants.to_f  : 0.0)
-      end
-
-      # The measure we use to order (sort) alternatives and decide which one is better (by calculating z-score).
-      # Defaults to conversion rate.
-      def measure
-        conversion_rate
-      end
-
-      def <=>(other)
-        measure <=> other.measure
-      end
-
-      def ==(other)
-        other && id == other.id && experiment == other.experiment
-      end
-
-      def to_s
-        name
-      end
-
-      def inspect
-        "#{name}: #{value} #{converted}/#{participants}"
-      end
-
-      def load_counts
-        if @experiment.playground.collecting?
-          @participants, @converted, @conversions = @experiment.playground.connection.ab_counts(@experiment.id, id).values_at(:participants, :converted, :conversions)
-        else
-          @participants = @converted = @conversions = 0
-        end
-      end
-    end
-
-
-      # The meat.
-      class AbTest < Base
-        class << self
-
-          # Convert z-score to probability.
-          def probability(score)
-            score = score.abs
-            probability = AbTest::Z_TO_PROBABILITY.find { |z,p| score >= z }
-            probability ? probability.last : 0
-          end
-
-          def friendly_name
-            "A/B Test"
-          end
-
-        end
+      DEFAULT_SCORE_METHOD = :z_score
 
       def initialize(*args)
         super
+      	@score_method = DEFAULT_SCORE_METHOD
+        @use_probabilities = nil
       end
 
-
       # -- Metric --
 
       # Tells A/B test which metric we're measuring, or returns metric in use.
@@ -168,6 +84,23 @@ module Vanity
         alternatives.find { |alt| alt.value == value }
       end
 
+      # What method to use for calculating score.  Default is :ab_test, but can
+      # also be set to :bandit_score to calculate probability of each
+      # alternative being the best.
+      #
+      # @example Define A/B test which uses bayes_bandit_score in reporting
+      # ab_test "noodle_test" do
+      #   alternatives "spaghetti", "linguine"
+      #   metrics :signup
+      #   score_method :bayes_bandit_score
+      # end
+      def score_method(method=nil)
+      	if method
+      	  @score_method = method
+      	end
+      	@score_method
+      end
+
       # Defines an A/B test with two alternatives: false and true.  This is the
       # default pair of alternatives, so just syntactic sugar for those who love
       # being explicit.
@@ -197,20 +130,28 @@ module Vanity
         if @playground.collecting?
           if active?
             identity = identity()
-      	    index = connection.ab_showing(@id, identity)
-      	    unless index
-      	      index = alternative_for(identity)
-      	      if !@playground.using_js?
-            		# if we have an on_assignment block, call it on new assignments
-            		if @on_assignment_block
-            		  assignment = alternatives[index.to_i]
-            		  if !connection.ab_seen @id, identity, assignment
-            		    @on_assignment_block.call(Vanity.context, identity, assignment, self)
-            		  end
-            		end
-            		connection.ab_add_participant @id, index, identity
-            		check_completion!
-      	      end
+            index = connection.ab_showing(@id, identity)
+            unless index
+              index = alternative_for(identity)
+              if !@playground.using_js?
+                # if we have an on_assignment block, call it on new assignments
+                if @on_assignment_block
+                  assignment = alternatives[index.to_i]
+                  if !connection.ab_seen @id, identity, assignment
+                    @on_assignment_block.call(Vanity.context, identity, assignment, self)
+                  end
+                end
+                # if we are rebalancing probabilities, keep track of how long it has been since we last rebalanced
+                if @rebalance_frequency
+                  @assignments_since_rebalancing += 1
+                  if @assignments_since_rebalancing >= @rebalance_frequency
+                    @assignments_since_rebalancing = 0
+                    rebalance!
+                  end
+                end
+                connection.ab_add_participant @id, index, identity
+                check_completion!
+              end
             end
           else
             index = connection.ab_get_outcome(@id) || alternative_for(identity)
@@ -288,12 +229,20 @@ module Vanity
 
       # -- Reporting --
 
+      def calculate_score
+      	if respond_to?(score_method)
+      	  self.send(score_method)
+      	else
+      	  score
+      	end
+      end
+
       # Scores alternatives based on the current tracking data.  This method
       # returns a structure with the following attributes:
       # [:alts]   Ordered list of alternatives, populated with scoring info.
       # [:base]   Second best performing alternative.
       # [:least]  Least performing alternative (but more than zero conversion).
-      # [:choice] Choice alterntive, either the outcome or best alternative.
+      # [:choice] Choice alternative, either the outcome or best alternative.
       #
       # Alternatives returned by this method are populated with the following
       # attributes:
@@ -329,10 +278,48 @@ module Vanity
         # choice alternative can only pick best if we have high probability (>90%).
         best = sorted.last if sorted.last.measure > 0.0
         choice = outcome ? alts[outcome.id] : (best && best.probability >= probability ? best : nil)
-        Struct.new(:alts, :best, :base, :least, :choice).new(alts, best, base, least, choice)
+      	Struct.new(:alts, :best, :base, :least, :choice, :method).new(alts, best, base, least, choice, :score)
       end
 
-      # Use the result of #score to derive a conclusion.  Returns an
+      # Scores alternatives based on the current tracking data, using Bayesian
+      # estimates of the best binomial bandit. Based on the R bandit package,
+      # http://cran.r-project.org/web/packages/bandit, which is based on
+      # Steven L. Scott, A modern Bayesian look at the multi-armed bandit,
+      # Appl. Stochastic Models Bus. Ind. 2010; 26:639-658.
+      # (http://www.economics.uci.edu/~ivan/asmb.874.pdf)
+      #
+      # This method returns a structure with the following attributes:
+      # [:alts]   Ordered list of alternatives, populated with scoring info.
+      # [:base]   Second best performing alternative.
+      # [:least]  Least performing alternative (but more than zero conversion).
+      # [:choice] Choice alternative, either the outcome or best alternative.
+      #
+      # Alternatives returned by this method are populated with the following
+      # attributes:
+      # [:probability]  Probability (probability this is the best alternative).
+      # [:difference]   Difference from the least performant altenative.
+      #
+      # The choice alternative is set only if its probability is higher or
+      # equal to the specified probability (default is 90%).
+      def bayes_bandit_score(probability = 90)
+      	begin
+      	  require "backports/1.9.1/kernel/define_singleton_method" if RUBY_VERSION < "1.9"
+      	  require "integration"
+      	  require "rubystats"
+      	rescue LoadError
+      	  fail "to use bayes_bandit_score, install integration and rubystats gems"
+      	end
+
+      	begin
+      	  require "gsl"
+      	rescue LoadError
+      	  warn "for better integration performance, install gsl gem"
+      	end
+
+      	BayesianBanditScore.new(alternatives, outcome).calculate!
+      end
+
+      # Use the result of #score or #bayes_bandit_score to derive a conclusion.  Returns an
       # array of claims.
       def conclusion(score = score)
         claims = []
@@ -351,16 +338,24 @@ module Vanity
           # we want a result that's clearly better than 2nd best.
           best, second = sorted[0], sorted[1]
           if best.measure > second.measure
-            diff = ((best.measure - second.measure) / second.measure * 100).round
-            better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
-            claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
-            if best.probability >= 90
-              claims << "With %d%% probability this result is statistically significant." % score.best.probability
-            else
-              claims << "This result is not statistically significant, suggest you continue this experiment."
-            end
-            sorted.delete best
-          end
+      	    diff = ((best.measure - second.measure) / second.measure * 100).round
+      	    better = " (%d%% better than %s)" % [diff, second.name] if diff > 0
+      	    claims << "The best choice is %s: it converted at %.1f%%%s." % [best.name, best.measure * 100, better]
+      	    if score.method == :bayes_bandit_score
+      	      if best.probability >= 90
+      		      claims << "With %d%% probability this result is the best." % score.best.probability
+      	      else
+      		      claims << "This result does not have strong confidence behind it, suggest you continue this experiment."
+      	      end
+      	    else
+      	      if best.probability >= 90
+      		      claims << "With %d%% probability this result is statistically significant." % score.best.probability
+      	      else
+      		      claims << "This result is not statistically significant, suggest you continue this experiment."
+      	      end
+      	    end
+      	    sorted.delete best
+      	  end
           sorted.each do |alt|
             if alt.measure > 0.0
               claims << "%s converted at %.1f%%." % [alt.name.gsub(/^o/, "O"), alt.measure * 100]
@@ -375,6 +370,43 @@ module Vanity
         claims
       end
 
+      # -- Unequal probability assignments --
+
+      def set_alternative_probabilities(alternative_probabilities)
+        # create @use_probabilities as a function to go from [0,1] to outcome
+        cumulative_probability = 0.0
+        new_probabilities = alternative_probabilities.map {|am| [am, (cumulative_probability += am.probability)/100.0]}
+        @use_probabilities = new_probabilities
+      end
+
+      # -- Experiment rebalancing --
+
+      # Experiment rebalancing allows the app to automatically adjust the probabilities for each alternative; when one is performing better, it will increase its probability
+      #  according to Bayesian one-armed bandit theory, in order to (eventually) maximize your overall conversions.
+
+      # Sets or returns how often (as a function of number of people assigned) to rebalance. For example:
+      #   ab_test "Simple" do
+      #     rebalance_frequency 100
+      #   end
+      #
+      #  puts "The experiment will automatically rebalance after every " + experiment(:simple).description + " users are assigned."
+      def rebalance_frequency(rf = nil)
+        if rf
+          @assignments_since_rebalancing = 0
+          @rebalance_frequency = rf
+          rebalance!
+        end
+        @rebalance_frequency
+      end
+
+      # Force experiment to rebalance.
+      def rebalance!
+        return unless @playground.collecting?
+        score_results = bayes_bandit_score
+        if score_results.method == :bayes_bandit_score
+          set_alternative_probabilities score_results.alts
+        end
+      end
 
       # -- Completion --
 
@@ -404,19 +436,22 @@ module Vanity
         outcome && _alternatives[outcome]
       end
 
-      def complete!
+      def complete!(outcome = nil)
         return unless @playground.collecting? && active?
         super
-        if @outcome_is
-          begin
-            result = @outcome_is.call
-            outcome = result.id if Alternative === result && result.experiment == self
-          rescue
-            warn "Error in AbTest#complete!: #{$!}"
+
+      	unless outcome
+          if @outcome_is
+            begin
+              result = @outcome_is.call
+              outcome = result.id if Alternative === result && result.experiment == self
+            rescue
+              warn "Error in AbTest#complete!: #{$!}"
+            end
+          else
+            best = score.best
+            outcome = best.id if best
           end
-        else
-          best = score.best
-          outcome = best.id if best
         end
         # TODO: logging
         connection.ab_set_outcome @id, outcome || 0
@@ -484,7 +519,15 @@ module Vanity
       # identity, and randomly distributed alternatives for each identity (in the
       # same experiment).
       def alternative_for(identity)
-        Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
+        if @use_probabilities
+          existing_assignment = connection.ab_assigned @id, identity
+          return existing_assignment if existing_assignment
+          random_outcome = rand()
+          @use_probabilities.each do |alternative, max_prob|
+            return alternative.id if random_outcome < max_prob
+          end
+        end
+        return Digest::MD5.hexdigest("#{name}/#{identity}").to_i(17) % @alternatives.size
       end
 
       begin