benchmark-lab 0.0.1

data/README.md

@@ -0,0 +1,103 @@
+[![PullReview stats](https://www.pullreview.com/github/toch/benchmark-lab/badges/master.svg?)](https://www.pullreview.com/github/toch/benchmark-lab/reviews/master)
+
+# Benchmark Lab
+
+Run Real Experiment and Calculate Non-Parametric Statistics.
+
+## Installation
+
+Install it yourself as:
+
+    $ gem install benchmark-lab
+
+## Usage
+
+There are two ways to use it:
+1. classic: as Benchmark.bm does
+2. iterative: collects and measures separately, stores into different JSON
+files, then put everything together and rank them
+
+### Classic Usage
+
+```Ruby
+require 'benchmark/lab'
+
+n = 5_000_000
+cases = {
+  'for:'    => proc { for i in 1..n; a = "1"; end },
+  'times:'  => proc { n.times do   ; a = "1"; end },
+  'upto:'   => proc { 1.upto(n) do ; a = "1"; end }
+}
+
+# How many times do you run the function
+# 20 is a good minimum number
+nbr_of_samples = 20
+
+Benchmark.experiment(nbr_of_samples) do |x|
+  cases.each { |label, blk| x.report(label, &blk) }
+end
+```
+
+The output looks like the following:
+```
+          user                system              total               real
+for:      [0.77,0.77,0.78]    [0.00,0.00,0.00]    [0.77,0.77,0.78]    [0.77,0.77,0.78]
+times:    [0.74,0.74,0.74]    [0.00,0.00,0.00]    [0.74,0.74,0.74]    [0.74,0.74,0.74]
+upto:     [0.75,0.75,0.75]    [0.00,0.00,0.00]    [0.75,0.75,0.75]    [0.75,0.75,0.75]
+The best "times:" is significantly (95%) better (total time).
+```
+
+### Iterative Usage
+
+```Ruby
+require 'benchmark/lab'
+
+n = 5_000_000
+
+# How many times do you run the function
+# 20 is a good minimum number
+nbr_of_samples = 20
+
+jsons = []
+
+jsons << Benchmark.observe_and_summarize(nbr_of_samples) do |x|
+  x.report('for') { for i in 1..n; a = "1"; end }
+end
+
+jsons << Benchmark.observe_and_summarize(nbr_of_samples) do |x|
+  x.report('times') { n.times do   ; a = "1"; end }
+end
+
+jsons << Benchmark.observe_and_summarize(nbr_of_samples) do |x|
+  x.report('upto') { 1.upto(n) do ; a = "1"; end }
+end
+
+best, is_h0_rejected = Benchmark.aggregate_and_rank(jsons.map { |json| JSON.parse(json) })
+
+puts best
+puts is_h0_rejected
+```
+
+The output looks like the following:
+```
+{"name"=>"total", "sample"=>[0.6899999999999977, 0.6899999999999977, 0.6899999999999977, 0.6899999999999977, 0.6900000000000013, 0.6900000000000048, 0.6900000000000048, 0.6999999999999957, 0.6999999999999957, 0.6999999999999957, 0.6999999999999957, 0.6999999999999957, 0.6999999999999993, 0.6999999999999993, 0.7000000000000028, 0.7000000000000028, 0.7000000000000028, 0.7000000000000028, 0.7000000000000028, 0.7000000000000028], "sample_size"=>20, "minimum"=>0.6899999999999977, "maximum"=>0.7000000000000028, "first_quartile"=>0.690000000000003, "third_quartile"=>0.7000000000000028, "median"=>0.6999999999999957, "interquartile_range"=>0.009999999999999787, "label"=>"upto"}
+true
+```
+
+## Ideas
+
+* compare two different implementations of a same function
+  1. get the stats, then compare
+  2. use git (commit, branch)
+  3. use tests to check no performance regression at the same time
+  4. annotate the tests you want to check
+* decide the sample size automatically (based on the power you want to reach)
+* explain correctly why we should do that
+
+## Contributing
+
+1. Fork it ( https://github.com/toch/benchmark-lab/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'spec'
+  t.test_files = FileList['spec/*/*_spec.rb']
+end
+
+task default: :test

data/benchmark-lab.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'benchmark/lab/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'benchmark-lab'
+  spec.version       = Benchmark::Experiment::VERSION
+  spec.authors       = ['Christophe Philemotte']
+  spec.email         = ['christophe.philemotte@8thcolor.com']
+  spec.summary       = %q{Run Real Experiment and Calculate Non-Parametric Statistics.}
+  spec.description   = %q{Run Real Experiment and Calculate Non-Parametric Statistics.}
+  spec.homepage      = 'https://github.com/toch/benchmark-lab'
+  spec.license       = 'GPLv3'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'minitest', '4.5.0'
+  spec.add_development_dependency 'turn', '~> 0.9'
+  spec.add_runtime_dependency 'distribution'
+end

data/lib/benchmark/lab.rb

@@ -0,0 +1,147 @@
+require 'benchmark'
+require 'benchmark/lab/descriptive_statistics'
+require 'benchmark/lab/mann_whitney_u_test'
+require 'benchmark/lab/version'
+
+require 'json'
+
+module Benchmark
+  class Sample
+    include Enumerable
+
+    def initialize
+      @observations = []
+    end
+
+    def <<(observation)
+      @observations << observation
+    end
+
+    def each(&block)
+      @observations.each do |observation|
+        if block_given?
+          block.call observation
+        else
+          yield observation
+        end
+      end
+    end
+  end
+
+  class Job
+    alias old_item item
+    def item(label = '', &blk)
+      old_item(label, &blk)
+      @list.last << Sample.new
+      @list.last << []
+      self
+    end
+
+    def observe_and_summarize(sample_size)
+      @list.each do |label, item, sample, stats|
+        sample_size.times.each do
+          sample << Benchmark.measure(label, &item)
+        end
+
+        Experiment::MEASURED_TIMES.keys.each do |time_name|
+          stats << Benchmark::Experiment::DescriptiveStatistics.new(sample.map(&time_name), time_name)
+        end
+      end
+    end
+
+    alias report item
+  end
+
+  module Experiment
+    MEASURED_TIMES =
+      {
+        utime: 'user',
+        stime: 'system',
+        total: 'total',
+        real: 'real'
+      }
+
+    def observe_and_summarize(sample_size, &blk)
+      job = Job.new(0)
+      yield(job)
+      job.observe_and_summarize(sample_size)
+      all_stats = job.list.map{ |label, _, _, stats| [label, stats] }.to_h
+      all_stats.to_json
+    end
+
+    def aggregate_and_rank(jsons)
+      return if jsons.empty?
+      all_stats = jsons.inject({}) { |elem, hsh| hsh.merge(elem) }
+      rank(all_stats)
+    end
+
+    def experiment(sample_size, &blk)
+      all_stats = JSON.parse(observe_and_summarize(sample_size, &blk))
+      print_stats(all_stats)
+
+      best, is_the_best_significative = rank(all_stats)
+
+      puts "The best \"#{best['label']}\" is #{is_the_best_significative ? '' : 'not '}significantly (95%) better (total time)."
+
+      all_stats
+    end
+
+    def rank(all_stats, alpha = 0.05)
+      ranked = all_stats.map do |label, stats|
+        total = stats.select{ |stat| stat['name'] == 'total' }.first
+        total['label'] = label
+        total
+      end.sort_by { |stat| stat['median'] }
+      is_h0_rejected = true
+      if all_stats.size > 1
+        z = Benchmark::Experiment::MannWhitneyUTest::calculate_z(ranked.first['sample'], ranked[1]['sample'])
+        p_value = Benchmark::Experiment::MannWhitneyUTest::calculate_probability_z(z)
+        is_h0_rejected = Benchmark::Experiment::MannWhitneyUTest::is_null_hypothesis_rejected?(p_value, alpha)
+      end
+
+      return ranked.first, is_h0_rejected
+    end
+
+    def iterative_experiment
+    end
+
+    private
+
+    def print_stats(all_stats)
+      width = label_width(all_stats)
+
+      lines = []
+      spacing = [0] * MEASURED_TIMES.size
+      tab = ' ' * 4
+
+      all_stats.each do |label, stats|
+        line = ''
+        line << label.ljust(width)
+
+        stats.each_with_index do |stat, index|
+          value = "#{tab}[#{'%.2f' % stat['first_quartile']},#{'%.2f' % stat['median']},#{'%.2f' % stat['third_quartile']}]"
+          spacing[index] = [spacing[index], value.length].minmax.last
+          line << value
+        end
+        line << "\n"
+        lines << line
+      end
+
+      print ''.ljust(width)
+      MEASURED_TIMES.values.each_with_index do |head, index|
+        print "#{tab}#{head}".ljust(spacing[index])
+      end
+      print "\n"
+
+      lines.each { |line| print line }
+    end
+
+    def label_width(all_stats)
+      label_widths = all_stats.map { |label, _| label.to_s.length }
+      label_widths.minmax.last
+    end
+
+  end
+
+  extend Benchmark::Experiment
+end

data/lib/benchmark/lab/descriptive_statistics.rb

@@ -0,0 +1,63 @@
+module Benchmark
+  module Experiment
+    class DescriptiveStatistics
+      def initialize(sample, name = '')
+        # raise exception if empty sample
+        @name = name
+        @sample = sample.sort
+        @minimum, @maximum = @sample.minmax
+        @median = calculate_median_of(@sample)
+        @first_quartile = calculate_first_quartile_of(@sample)
+        @third_quartile = calculate_third_quartile_of(@sample)
+      end
+
+      attr_reader :name, :sample, :minimum, :maximum, :first_quartile, :third_quartile, :median
+
+      def sample_size
+        sample.size
+      end
+
+      def interquartile_range
+        @third_quartile - @first_quartile
+      end
+
+      def to_json(options = {})
+        {
+          'name' => name,
+          'sample' => sample,
+          'sample_size' => sample_size,
+          'minimum' => minimum,
+          'maximum' => maximum,
+          'first_quartile' => first_quartile,
+          'third_quartile' => third_quartile,
+          'median' => median,
+          'interquartile_range' => interquartile_range
+        }.to_json
+      end
+
+      private
+
+      # https://en.wikipedia.org/wiki/Median
+      def calculate_median_of(data)
+        return data[data.size / 2] if data.size.odd?
+
+        (data[(data.size - 1) / 2] + data[data.size / 2]) / 2.0
+      end
+
+      # http://mathworld.wolfram.com/Quartile.html
+      # https://en.wikipedia.org/wiki/Quartile
+      def calculate_first_quartile_of(data)
+        return calculate_median_of(data[0..(data.size / 2)]) if data.size.odd?
+
+        calculate_median_of(data[0..((data.size - 1) / 2)])
+      end
+
+      def calculate_third_quartile_of(data)
+        return calculate_median_of(data[(data.size / 2)..-1]) if data.size.odd?
+
+        calculate_median_of(data[(data.size / 2)..-1])
+      end
+
+    end
+  end
+end

data/lib/benchmark/lab/mann_whitney_u_test.rb

@@ -0,0 +1,114 @@
+require 'distribution'
+
+module Benchmark
+  module Experiment
+    module MannWhitneyUTest
+      def self.calculate_U(x, y)
+        ranked = concatenate_and_label(x, y)
+
+        rank!(ranked)
+
+        adjust_ties!(ranked)
+
+        r_x = rank_sum(ranked, :x)
+        r_y = rank_sum(ranked, :y)
+
+        n_x = x.size
+        n_y = y.size
+
+        u_x = calculate_u_sample(n_x, n_y, r_x)
+        u_y = calculate_u_sample(n_y, n_x, r_y)
+
+        [u_x, u_y]
+      end
+
+      def self.calculate_z(x, y)
+        n_x = x.size.to_f
+        n_y = y.size.to_f
+        n = n_x + n_y
+        n_xy = n_x * n_y
+
+        u = calculate_U(x, y).minmax.first.to_f
+
+        t = ties?(x, y)
+
+        mu_u = n_xy / 2.0
+
+        if !t.first
+          sigma_u = Math::sqrt(n_xy * (n + 1.0) / 12.0)
+        else
+          sigma_u = Math::sqrt(n_xy / (n * (n + 1)) * ((n**3 - n) / 12.0 - t.last))
+        end
+
+        (u - mu_u) / sigma_u
+      end
+
+      def self.calculate_probability_z(z, two_sided = true)
+        prob = (1.0 - Distribution::Normal.cdf(z.abs()))
+        prob *= 2.0 if two_sided
+        prob
+      end
+
+      def self.is_null_hypothesis_rejected?(pvalue, significance_level)
+        pvalue < significance_level
+      end
+
+      private
+
+      def self.ties?(x, y)
+        all = x + y
+        ties = all.group_by { |e| e }.reject { |_, v| v.size < 2 }
+
+        found_ties = ties.size > 0
+        [
+          found_ties,
+          ties.inject(0) { |a, v| a + (v.size**3 - v.size) / 12.0 }
+        ]
+      end
+
+      def self.concatenate_and_label(x, y)
+        ranked = []
+        
+        ranked += x.map { |e| [e, :x] }
+        ranked += y.map { |e| [e, :y] }
+      end
+
+      def self.rank!(ranked)
+        ranked.sort!
+
+        ranked.inject(1) do |rank, elem|
+          elem << rank
+          rank + 1
+        end
+      end
+
+      def self.rank_sum(ranked, label)
+        ranked
+          .select { |elem| elem[1] == label }
+          .inject(0) { |rank_sum, elem| rank_sum + elem.last }
+      end
+
+      def self.adjust_ties!(ranked)
+        ties = {}
+
+        ranked
+          .group_by { |e| e.first }
+          .reject { |_, v| v.size < 2 }
+          .each do |score, data|
+            ties[score] = data.inject(0) do |sum, elem|
+              sum + elem.last
+            end / data.size.to_f
+          end 
+
+        ranked.map! do |elem|
+          elem[-1] = ties[elem.first] if ties.keys.include? elem.first
+          elem
+        end if ties.keys.size > 0
+      end
+
+      def self.calculate_u_sample(n, n_other, r)
+        n * n_other + n * (n + 1) / 2.0 - r
+      end
+    end
+  end
+end