split 0.4.6 → 0.5.0

data/lib/split.rb

@@ -1,8 +1,7 @@
-require 'split/experiment'
-require 'split/alternative'
-require 'split/helper'
-require 'split/version'
-require 'split/configuration'
+%w[algorithms extensions metric trial experiment alternative helper version configuration persistence exceptions].each do |f|
+  require "split/#{f}"
+end
+
 require 'split/engine' if defined?(Rails)
 require 'redis/namespace'
 
@@ -52,9 +51,9 @@ module Split
   #     config.ignore_ips = '192.168.2.1'
   #   end
   def configure
-     self.configuration ||= Configuration.new
-     yield(configuration)
-   end
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
 end
 
-Split.configure {}
+Split.configure {}

data/lib/split/algorithms.rb

@@ -0,0 +1,3 @@
+%w[weighted_sample whiplash].each do |f|
+  require "split/algorithms/#{f}"
+end

data/lib/split/algorithms/weighted_sample.rb

@@ -0,0 +1,17 @@
+module Split
+  module Algorithms
+    module WeightedSample
+      def self.choose_alternative(experiment)
+        weights = experiment.alternatives.map(&:weight)
+
+        total = weights.inject(:+)
+        point = rand * total
+
+        experiment.alternatives.zip(weights).each do |n,w|
+          return n if w >= point
+          point -= w
+        end
+      end
+    end
+  end
+end

data/lib/split/algorithms/whiplash.rb

@@ -0,0 +1,35 @@
+# A multi-armed bandit implementation inspired by 
+# @aaronsw and victorykit/whiplash
+require 'simple-random'
+
+module Split
+  module Algorithms
+    module Whiplash
+      def self.choose_alternative(experiment)
+        experiment[best_guess(experiment.alternatives)]
+      end
+
+      private
+      
+      def self.arm_guess(participants, completions)
+        a = [participants, 0].max
+        b = [participants-completions, 0].max
+        s = SimpleRandom.new; s.set_seed; s.beta(a+fairness_constant, b+fairness_constant)
+      end
+      
+      def self.best_guess(alternatives)
+        guesses = {}
+        alternatives.each do |alternative| 
+          guesses[alternative.name] = arm_guess(alternative.participant_count, alternative.completed_count) 
+        end
+        gmax = guesses.values.max
+        best = guesses.keys.select {|name| guesses[name] ==  gmax }
+        return best.sample
+      end
+      
+      def self.fairness_constant
+        7
+      end
+    end
+  end
+end

data/lib/split/alternative.rb

@@ -20,27 +20,33 @@ module Split
     end
 
     def participant_count
-      Split.redis.hget(key, 'participant_count').to_i
+      @participant_count ||= Split.redis.hget(key, 'participant_count').to_i
     end
 
     def participant_count=(count)
+      @participant_count = count
       Split.redis.hset(key, 'participant_count', count.to_i)
     end
 
     def completed_count
-      Split.redis.hget(key, 'completed_count').to_i
+      @completed_count ||= Split.redis.hget(key, 'completed_count').to_i
+    end
+    
+    def unfinished_count
+      participant_count - completed_count
     end
 
     def completed_count=(count)
+      @completed_count = count
       Split.redis.hset(key, 'completed_count', count.to_i)
     end
 
     def increment_participation
-      Split.redis.hincrby key, 'participant_count', 1
+      @participant_count = Split.redis.hincrby key, 'participant_count', 1
     end
 
     def increment_completion
-      Split.redis.hincrby key, 'completed_count', 1
+       @completed_count = Split.redis.hincrby key, 'completed_count', 1
     end
 
     def control?
@@ -87,6 +93,8 @@ module Split
     end
 
     def reset
+      @participant_count = nil
+      @completed_count = nil
       Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0
     end
 

data/lib/split/configuration.rb

@@ -1,5 +1,18 @@
 module Split
   class Configuration
+    BOTS = {
+      'Baidu' => 'Chinese spider',
+      'Gigabot' => 'Gigabot spider',
+      'Googlebot' => 'Google spider',
+      'libwww-perl' => 'Perl client-server library loved by script kids',
+      'lwp-trivial' => 'Another Perl library loved by script kids',
+      'msnbot' => 'Microsoft bot',
+      'SiteUptime' => 'Site monitoring services',
+      'Slurp' => 'Yahoo spider',
+      'WordPress' => 'WordPress spider',
+      'ZIBB' => 'ZIBB spider',
+      'ZyBorg' => 'Zyborg? Hmmm....'
+    }
     attr_accessor :robot_regex
     attr_accessor :ignore_ip_addresses
     attr_accessor :db_failover
@@ -7,15 +20,92 @@ module Split
     attr_accessor :db_failover_allow_parameter_override
     attr_accessor :allow_multiple_experiments
     attr_accessor :enabled
+    attr_accessor :experiments
+    attr_accessor :persistence
+    attr_accessor :algorithm
+
+    def disabled?
+      !enabled
+    end
+
+    def experiment_for(name)
+      if normalized_experiments
+        normalized_experiments[name]
+      end
+    end
+
+    def metrics
+      return @metrics if defined?(@metrics)
+      @metrics = {}
+      if self.experiments
+        self.experiments.each do |key, value|
+          metric_name = value[:metric]
+          if metric_name
+            @metrics[metric_name] ||= []
+            @metrics[metric_name] << Split::Experiment.load_from_configuration(key)
+          end
+        end
+      end
+      @metrics
+    end
+    
+    def normalized_experiments
+      if @experiments.nil?
+        nil
+      else
+        experiment_config = {}
+        @experiments.keys.each do | name |
+          experiment_config[name] = {}
+        end
+        @experiments.each do | experiment_name, settings|
+          experiment_config[experiment_name][:alternatives] = normalize_alternatives(settings[:alternatives]) if settings[:alternatives] 
+        end
+        experiment_config
+      end
+    end
+    
+    
+    def normalize_alternatives(alternatives)
+      given_probability, num_with_probability = alternatives.inject([0,0]) do |a,v|
+        p, n = a
+        if v.kind_of?(Hash) && v[:percent]
+          [p + v[:percent], n + 1]
+        else
+          a
+        end
+      end
+
+      num_without_probability = alternatives.length - num_with_probability
+      unassigned_probability = ((100.0 - given_probability) / num_without_probability / 100.0)
+
+      if num_with_probability.nonzero?
+        alternatives = alternatives.map do |v|
+          if v.kind_of?(Hash) && v[:name] && v[:percent]
+            { v[:name] => v[:percent] / 100.0 }
+          elsif v.kind_of?(Hash) && v[:name]
+            { v[:name] => unassigned_probability }
+          else
+            { v => unassigned_probability }
+          end
+        end
+        [alternatives.shift, alternatives]
+      else
+        alternatives = alternatives.dup
+        [alternatives.shift, alternatives]
+      end
+    end
 
     def initialize
-      @robot_regex = /\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i
+      @robot_regex = /\b(#{BOTS.keys.join('|')})\b/i
       @ignore_ip_addresses = []
       @db_failover = false
       @db_failover_on_db_error = proc{|error|} # e.g. use Rails logger here
       @db_failover_allow_parameter_override = false
       @allow_multiple_experiments = false
       @enabled = true
+      @experiments = {}
+      @persistence = Split::Persistence::SessionAdapter
+      @algorithm = Split::Algorithms::WeightedSample
     end
   end
 end

data/lib/split/dashboard/helpers.rb

@@ -23,11 +23,11 @@ module Split
 
       if z == 0.0
         'No Change'
-      elsif z < 1.96
+      elsif z < 1.645
         'no confidence'
-      elsif z < 2.57
+      elsif z < 1.96
         '95% confidence'
-      elsif z < 3.29
+      elsif z < 2.57
         '99% confidence'
       else
         '99.9% confidence'

data/lib/split/dashboard/views/_experiment.erb

@@ -37,7 +37,7 @@
           <% end %>
         </td>
         <td><%= alternative.participant_count %></td>
-        <td><%= alternative.participant_count - alternative.completed_count %></td>
+        <td><%= alternative.unfinished_count %></td>
         <td><%= alternative.completed_count %></td>
         <td>
           <%= number_to_percentage(alternative.conversion_rate) %>%

data/lib/split/exceptions.rb

@@ -0,0 +1,4 @@
+module Split
+  class InvalidPersistenceAdapterError < StandardError; end
+  class ExperimentNotFound < StandardError; end
+end

data/lib/split/experiment.rb

@@ -1,12 +1,40 @@
 module Split
   class Experiment
     attr_accessor :name
+    attr_writer :algorithm
+    attr_accessor :resettable
 
-    def initialize(name, *alternative_names)
-      @name = name.to_s
-      @alternatives = alternative_names.map do |alternative|
-                        Split::Alternative.new(alternative, name)
-                      end
+    def initialize(name, options = {})      
+      options = {
+          :resettable => true,
+        }.merge(options)
+       
+      @name = name.to_s 
+      @alternatives = options[:alternatives]  if !options[:alternatives].nil?
+      
+      if !options[:algorithm].nil?    
+        @algorithm = options[:algorithm].is_a?(String) ? options[:algorithm].constantize : options[:algorithm]
+      end
+      
+      if !options[:resettable].nil?
+        @resettable = options[:resettable].is_a?(String) ? options[:resettable] == 'true' : options[:resettable]
+      end
+      
+      if !options[:alternative_names].nil? 
+        @alternatives = options[:alternative_names].map do |alternative|
+                          Split::Alternative.new(alternative, name)
+                        end
+      end
+      
+      
+    end
+    
+    def algorithm
+      @algorithm ||= Split.configuration.algorithm
+    end
+
+    def ==(obj)
+      self.name == obj.name
     end
 
     def winner
@@ -16,6 +44,10 @@ module Split
         nil
       end
     end
+    
+    def participant_count
+      alternatives.inject(0){|sum,a| sum + a.participant_count}
+    end
 
     def control
       alternatives.first
@@ -33,6 +65,10 @@ module Split
       t = Split.redis.hget(:experiment_start_times, @name)
       Time.parse(t) if t
     end
+    
+    def [](name)
+      alternatives.find{|a| a.name == name} 
+    end
 
     def alternatives
       @alternatives.dup
@@ -47,14 +83,10 @@ module Split
     end
 
     def random_alternative
-      weights = alternatives.map(&:weight)
-
-      total = weights.inject(:+)
-      point = rand * total
-
-      alternatives.zip(weights).each do |n,w|
-        return n if w >= point
-        point -= w
+      if alternatives.length > 1
+        Split.configuration.algorithm.choose_alternative(self)
+      else
+        alternatives.first
       end
     end
 
@@ -78,6 +110,10 @@ module Split
       "#{key}:finished"
     end
 
+    def resettable?
+      resettable
+    end
+
     def reset
       alternatives.each(&:reset)
       reset_winner
@@ -105,9 +141,31 @@ module Split
         Split.redis.del(name)
         @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) }
       end
+      config_key = Split::Experiment.experiment_config_key(name)
+      Split.redis.hset(config_key, :resettable, resettable)
+      Split.redis.hset(config_key, :algorithm, algorithm.to_s)
+      self
     end
 
     def self.load_alternatives_for(name)
+      if Split.configuration.experiment_for(name)
+        load_alternatives_from_configuration_for(name)
+      else
+        load_alternatives_from_redis_for(name)
+      end
+    end
+
+    def self.load_alternatives_from_configuration_for(name)
+      alts = Split.configuration.experiment_for(name)[:alternatives]
+      raise ArgumentError, "Experiment configuration is missing :alternatives array" if alts.nil?
+      if alts.is_a?(Hash)
+        alts.keys
+      else
+        alts.flatten
+      end
+    end
+
+    def self.load_alternatives_from_redis_for(name)
       case Split.redis.type(name)
       when 'set' # convert legacy sets to lists
         alts = Split.redis.smembers(name)
@@ -119,14 +177,46 @@ module Split
       end
     end
 
+    def self.load_from_configuration(name)
+      exp_config = Split.configuration.experiment_for(name) || {}
+      self.new(name, :alternative_names => load_alternatives_for(name), 
+                           :resettable => exp_config[:resettable], 
+                           :algorithm => exp_config[:algorithm])
+    end
+
+    def self.load_from_redis(name)
+      exp_config = Split.redis.hgetall(experiment_config_key(name))
+      self.new(name, :alternative_names => load_alternatives_for(name),
+                      :resettable => exp_config['resettable'], 
+                      :algorithm => exp_config['algorithm'])     
+    end
+    
+    def self.experiment_config_key(name)
+      "experiment_configurations/#{name}"
+    end
+
     def self.all
-      Array(Split.redis.smembers(:experiments)).map {|e| find(e)}
+      Array(all_experiment_names_from_redis + all_experiment_names_from_configuration).map {|e| find(e)}
     end
 
+    def self.all_experiment_names_from_redis
+      Split.redis.smembers(:experiments)
+    end
+
+    def self.all_experiment_names_from_configuration
+      Split.configuration.experiments ? Split.configuration.experiments.keys : []
+    end
+
+
     def self.find(name)
-      if Split.redis.exists(name)
-        self.new(name, *load_alternatives_for(name))
+      if Split.configuration.experiment_for(name)
+        obj = load_from_configuration(name)
+      elsif Split.redis.exists(name)
+        obj = load_from_redis(name)
+      else
+        obj = nil
       end
+      obj
     end
 
     def self.find_or_create(key, *alternatives)
@@ -135,8 +225,6 @@ module Split
       if alternatives.length == 1
         if alternatives[0].is_a? Hash
           alternatives = alternatives[0].map{|k,v| {k => v} }
-        else
-          raise InvalidArgument, 'You must declare at least 2 alternatives'
         end
       end
 
@@ -145,16 +233,16 @@ module Split
       if Split.redis.exists(name)
         existing_alternatives = load_alternatives_for(name)
         if existing_alternatives == alts.map(&:name)
-          experiment = self.new(name, *alternatives)
+          experiment = self.new(name, :alternative_names => alternatives)
         else
-          exp = self.new(name, *existing_alternatives)
+          exp = self.new(name, :alternative_names => existing_alternatives)
           exp.reset
           exp.alternatives.each(&:delete)
-          experiment = self.new(name, *alternatives)
+          experiment = self.new(name, :alternative_names =>alternatives)
           experiment.save
         end
       else
-        experiment = self.new(name, *alternatives)
+        experiment = self.new(name, :alternative_names => alternatives)
         experiment.save
       end
       return experiment
@@ -164,7 +252,7 @@ module Split
     def self.initialize_alternatives(alternatives, name)
 
       unless alternatives.all? { |a| Split::Alternative.valid?(a) }
-        raise InvalidArgument, 'Alternatives must be strings'
+        raise ArgumentError, 'Alternatives must be strings'
       end
 
       alternatives.map do |alternative|
@@ -172,4 +260,4 @@ module Split
       end
     end
   end
-end
+end