skab 0.1.0

data/README.rdoc

@@ -0,0 +1,104 @@
+= Skab
+
+This is a tool to help run statistical analyses of A/B testing experiments
+we run here at Songkick.
+
+We use this util mainly to generate CSV files that we can plot using Google
+Docs in order to determine if an A/B test is a success or a failure.
+
+== Getting started
+
+* Install skab by running `gem install skab`
+* You can run the util by using the `skab` command line
+
+== Command line arguments
+
+    skab [output] [model] [model_args]
+
+The command line accepts a variable number of arguments:
+
+* `output` is the name of the output module to use to print data
+* `model` is the name of the model used to model the process to analyse
+* All other arguments are model dependent and are passed to the model
+
+== Outputs
+
+Skab is able to output different statistics, all based on the model used to
+generate the distribution.
+
+We currently support two main outputs:
+
+* Distribution: the discrete probability distribution for each group, 
+  based on the model used to represent the process
+* Differential: the discrete probability distribution for Xb - Xa
+
+== Models
+
+Skab currently supports two models to generate a distribution of the mean
+depending on the actual observed values:
+
+* Poisson model, working with rate of events on a specific period of time
+* Binomial model, working with success rates
+
+=== The poisson model
+
+The poisson model accepts two integer parameters: A and B. Each parameter
+corresponds to the measured number of events occuring in group A or B,
+respectively.
+
+The distribution outputs a list of probability for each mean depending on the
+A or B group, according to the poisson law of small numbers.
+
+Here is an example, with 1450 events observed for group A and 1430 for group B:
+
+    skab distribution poisson 1450 1430
+
+It is worth noting that the Poisson distribution is expensive to compute for
+large numbers (> 100), so this model uses an approximation using a normal
+distribution (using a variance of delta).
+
+=== The binomial model
+
+The binomial model is used to generate a distribution of success rates
+depending on a number of trials and successes for each group A and B.
+
+The distribution outputs a list of probable success rates and their respective
+probability for groups A and B.
+
+For example, this command generate the binomial distribution with:
+
+* 200 successes out of 450 trials for group A
+* 220 successes out of 470 trials for group B
+
+    skab distribution binomial 450 200 470 220
+
+== Known issues
+
+This software relies on Hash ordering to display values in the correct order.
+On Ruby versions older than 1.9, hash ordering wasn't guaranteed, and this
+will cause some output to be inconsistent (mainly differential CSV and
+summary outputs).
+
+== LICENSE
+
+The MIT License
+
+Copyright © 2012 Songkick
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the “Software”), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/bin/skab

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+version = RUBY_VERSION.split('.').map { |s| s.to_i }
+if version[0] <= 1 && version [1] < 9
+  STDERR.puts <<-WARN18
+WARNING: This software relies on feature available in Ruby 1.9 only. Using it
+under Ruby 1.8 may yield unexpected results. Read the README for more
+information.
+  WARN18
+end
+
+usage = <<-USAGE
+Usage: skab [output] [model] [extra_params]
+\tTry `skab help model` for more info about available models
+\tor  `skab help output` for more info about outputs
+USAGE
+
+require File.expand_path('../../lib/skab', __FILE__)
+
+if ARGV.empty?
+  puts usage
+  exit
+end
+
+if ARGV[0] == 'help'
+  if ARGV[1] == 'model'
+    if ARGV[2]
+      model = Skab::Models.from_name(ARGV[2])
+      if model
+        puts model.help
+      else
+        puts <<-UNKNOWN_MODEL
+This model doesn't exist. List of known models:
+#{Skab::Models.model_names.join(', ')}
+        UNKNOWN_MODEL
+      end
+    else
+      puts Skab::Models.help
+    end
+  elsif ARGV[1] == 'output'
+    if ARGV[2]
+      output = Skab::Output.from_name(ARGV[2])
+      if output
+        puts output.help
+      else
+        puts <<-UNKNOWN_OUTPUT
+This output doesn't exist. List of known output:
+#{Skab::Output.output_names.join(', ')}
+        UNKNOWN_OUTPUT
+      end
+    else
+      puts Skab::Output.help
+    end
+  else
+    puts usage
+  end
+  exit
+end
+
+model_args = ARGV.slice(2, ARGV.length)
+model_class = Skab::Models.from_name(ARGV[1])
+unless model_class
+  puts <<-UNKNOWN_MODEL
+This model doesn't exist. List of known models:
+#{Skab::Models.model_names.join(', ')}
+  UNKNOWN_MODEL
+  exit
+end
+
+output_class = Skab::Output.from_name(ARGV[0])
+unless output_class
+  puts <<-UNKNOWN_OUTPUT
+This output doesn't exist. List of known outputs:
+#{Skab::Output.output_names.join(', ')}
+  UNKNOWN_OUTPUT
+  exit
+end
+
+model = model_class.new(model_args)
+output = output_class.new(STDOUT)
+output.output(model)

data/lib/skab.rb

@@ -0,0 +1,6 @@
+module Skab
+  ROOT = File.expand_path('..', __FILE__)
+  require ROOT + '/skab/models'
+  require ROOT + '/skab/output'
+end
+

data/lib/skab/models.rb

@@ -0,0 +1,26 @@
+module Skab
+  require ROOT + '/skab/models/poisson'
+  require ROOT + '/skab/models/binomial'
+  
+  module Models
+    def self.from_name(name)
+      case name
+        when 'poisson'
+          Poisson
+        when 'binomial'
+          Binomial
+      end
+    end
+
+    def self.model_names
+      ['poisson', 'binomial']
+    end
+
+    def self.help
+      <<-HELP
+The following models are available: #{model_names.join(', ')}
+\tTry `skab help model [model] to find out more about a model
+      HELP
+    end
+  end
+end

data/lib/skab/models/binomial.rb

@@ -0,0 +1,83 @@
+module Skab
+  module Models
+    class Binomial
+
+      def initialize(args)
+        @a_trials = args.shift.to_i
+        @a_success = args.shift.to_i
+        @b_trials = args.shift.to_i
+        @b_success = args.shift.to_i
+      end
+
+      def distribution
+        return @distribution if @distribution
+        @distribution = []
+        sums = [0, 0, 0]
+        i = 0.0
+        while i <= 1000
+          @distribution[i] = []
+          @distribution[i][0] = i / 1000
+          @distribution[i][1] = binomial(@a_trials, @a_success, i / 1000)
+          @distribution[i][2] = binomial(@b_trials, @b_success, i / 1000)
+          sums[1] += binomial(@a_trials, @a_success, i / 1000)
+          sums[2] += binomial(@b_trials, @b_success, i / 1000)
+          i += 1
+        end
+        i = 0.0
+        while i <= 1000
+          @distribution[i][1] /= sums[1]
+          @distribution[i][2] /= sums[2]
+          i += 1
+        end
+        @distribution
+      end
+
+      def differential
+        return @differential if @differential
+        @differential = Hash.new(0)
+        i = 0.0
+        while i <= 1000
+          j = 0.0
+          while j <= 1000
+            @differential[(j - i) / 1000] += distribution[j][2] * distribution[i][1]
+            j += 1
+          end
+          i += 1
+        end
+        @differential
+      end
+
+      def self.help
+        <<-HELP
+skab [output] binomial [trials_a] [successes_a] [trials_b] [successes_b]
+\tWhere: all parameters are integers
+\tPlots the binomial distribution for A and B, given their respective
+\tnumber of successes and trials
+        HELP
+      end
+
+      private
+
+      attr_reader :a, :b
+
+      def binomial(trials, success, rate)
+        binomial_coef(trials, success) * 
+            (rate ** success) * 
+            ((1 - rate) ** (trials - success))
+      end
+
+      def binomial_coef(n, k)
+        fact(n) / (fact(k) * fact(n - k))
+      end
+
+      def fact(n)
+        f = 1
+        (1..n).each do |i|
+          f *= i
+        end
+        f
+      end
+
+    end
+  end
+end

data/lib/skab/models/poisson.rb

@@ -0,0 +1,67 @@
+module Skab
+  module Models
+    class Poisson
+
+      def initialize(args)
+        @a = args.shift.to_i
+        @b = args.shift.to_i
+      end
+
+      def distribution
+        return @distribution if @distribution
+        @distribution = []
+        (0..limit).each do |n|
+          @distribution[n] = []
+          @distribution[n][0] = n
+          @distribution[n][1] = normal_approximation(n, a)
+          @distribution[n][2] = normal_approximation(n, b)
+        end
+        @distribution
+      end
+
+      def differential
+        return @differential if @differential
+        @differential = Hash.new(0)
+        (0..limit).each do |a|
+          (0..limit).each do |b|
+            @differential[b - a] += distribution[b][2] * distribution[a][1]
+          end
+        end
+        @differential
+      end
+
+      def self.help
+        <<-USAGE
+skab [output] poisson [a] [b]
+\tWhere: [a] and [b] are integers
+\tPlots the poisson distribution for [a] and [b]
+        USAGE
+      end
+      
+      private
+
+      attr_reader :a, :b
+
+      def limit
+        limit = [a, b].max * 2
+      end
+
+      def normal_approximation(k, delta)
+        (1 / (Math.sqrt(delta) * Math.sqrt(2 * Math::PI))) * Math.exp(-0.5 * (((k - delta) / Math.sqrt(delta))**2))
+      end
+
+      def poisson(k, delta)
+        ((delta ** k) * Math.exp(-delta)) / factorial(k)
+      end
+
+      def factorial(n)
+        f = 1
+        (1..n).each do |i|
+          f *= i
+        end
+        f
+      end
+
+    end
+  end
+end

data/lib/skab/output.rb

@@ -0,0 +1,29 @@
+module Skab
+  require ROOT + '/skab/output/distribution'
+  require ROOT + '/skab/output/differential'
+  require ROOT + '/skab/output/summary'
+
+  module Output
+    def self.from_name(name)
+      case name
+        when 'differential'
+          Differential
+        when 'distribution'
+          Distribution
+        when 'summary'
+          Summary
+      end
+    end
+
+    def self.output_names
+      ['distribution', 'differential', 'summary']
+    end
+
+    def self.help
+      <<-HELP
+The following outputs are available: #{output_names.join(', ')}
+\tTry `skab help output [output] to find out more about a given output
+      HELP
+    end
+  end
+end

data/lib/skab/output/differential.rb

@@ -0,0 +1,44 @@
+module Skab
+  module Output
+    class Differential
+      def initialize(out)
+        @out = out
+      end
+
+      def output(model)
+        data = model.differential
+        
+        range = 0
+        data.each do |k, v|
+          if v != 0 && abs(k) > range
+            range = abs(k)
+          end
+        end
+
+        range += range / 10
+
+        Hash[data.sort].each do |k, v|
+          if abs(k) <= range
+            @out.puts "#{k},#{v}"
+          end
+        end
+      end
+
+      def self.help
+        <<-HELP
+Usage: skab differential [model] [parameters]
+\tOutputs the discrete probability distribution for (B - A) as returned by the
+\tspecified model. The output is a two columns CSV file, where the first
+\tcolumn is the absolute value of (B - A) and the second column the
+\tcorresponding discrete probability
+        HELP
+      end
+
+      private
+
+      def abs(n)
+        n >= 0 ? n : -n
+      end
+    end
+  end
+end

data/lib/skab/output/distribution.rb

@@ -0,0 +1,25 @@
+module Skab
+  module Output
+    class Distribution
+      def initialize(out)
+        @out = out
+      end
+
+      def output(model)
+        model.distribution.each do |d|
+          @out.puts "#{d.join(',')}"
+        end
+      end
+
+      def self.help
+        <<-HELP
+Usage: skab distribution [model] [parameters]
+\tOutputs the discrete probability distribution for both A and B, as
+\treturned by the specified model. The output is a three columns CSV
+\tfile, where the first column is the probable mean and the second and
+\tthird column the corresponding discrete probability for A and B.
+        HELP
+      end
+    end
+  end
+end

data/lib/skab/output/summary.rb

@@ -0,0 +1,34 @@
+module Skab
+  module Output
+    class Summary
+      def initialize(out)
+        @out = out
+      end
+
+      def output(model)
+        sum = 0.0
+        min = 0
+        max = 0
+        Hash[model.differential.sort].each do |k, v|
+          sum += v
+          if min == 0 || sum <= 0.05
+            min = k
+          end
+          if max == 0 && sum >= 0.95
+            max = k
+          end
+        end
+
+        @out.puts "The difference is located between #{min} and #{max} (90% confidence)"
+      end
+
+      def self.help
+        <<-HELP
+Usage: skab summary [model] [parameters]
+\tOutputs a summary of the whole statistical analysis conducted on A and
+\tB, using the specified model.
+        HELP
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+name: skab
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Vivien Barousse
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-10-10 00:00:00 Z
+dependencies: []
+
+description: 
+email: vivien@songkick.com
+executables: 
+- skab
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- bin/skab
+- lib/skab.rb
+- lib/skab/output.rb
+- lib/skab/models/binomial.rb
+- lib/skab/models/poisson.rb
+- lib/skab/output/summary.rb
+- lib/skab/output/distribution.rb
+- lib/skab/output/differential.rb
+- lib/skab/models.rb
+homepage: http://github.com/songkick/skab
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: A/B testing statistical analysis utility
+test_files: []
+