benchmark-lab 0.0.1

data/lib/benchmark/lab/version.rb

@@ -0,0 +1,5 @@
+module Benchmark
+  module Experiment
+    VERSION = '0.0.1'
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+gem 'minitest' # demand gem version
+require 'minitest/autorun'
+require 'turn/autorun'
+
+$LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
+require_relative '../lib/benchmark/lab'

data/spec/unit/descriptive_statistics_spec.rb

@@ -0,0 +1,45 @@
+require_relative '../spec_helper'
+
+describe Benchmark::Experiment::DescriptiveStatistics do
+  let(:sample) {
+    [49, 7, 6, 15, 43, 39, 47, 41, 42, 36, 40]
+  }
+
+  let(:data_name) { 'Test' }
+
+  subject {
+    Benchmark::Experiment::DescriptiveStatistics.new(sample, data_name)
+  }
+
+  it 'returns the name of the data collected' do
+    subject.name.must_equal data_name
+  end
+
+  it 'returns the sample size' do
+    subject.sample_size.must_equal 11
+  end
+
+  it 'returns the minimum of the sample' do
+    subject.minimum.must_equal 6
+  end
+
+  it 'returns the maximum of the sample' do
+    subject.maximum.must_equal 49
+  end
+
+  it 'returns the median of the sample' do
+    subject.median.must_equal 40
+  end
+
+  it 'returns the first quartile of the sample' do
+    subject.first_quartile.must_equal 25.5
+  end
+
+  it 'returns the first quartile of the sample' do
+    subject.third_quartile.must_equal 42.5
+  end
+
+  it 'returns the interquartile range of the sample' do
+    subject.interquartile_range.must_equal 17
+  end
+end

data/spec/unit/experiment_spec.rb

@@ -0,0 +1,101 @@
+require_relative '../spec_helper'
+
+describe Benchmark::Experiment do
+  let(:cases) do
+    n = 5_000_000
+    {
+      'for:'    => proc { for i in 1..n; a = "1"; end },
+      'times:'  => proc { n.times do   ; a = "1"; end },
+      'upto:'   => proc { 1.upto(n) do ; a = "1"; end }
+    }
+  end
+
+  it 'prints a report for each run case.' do
+    output_arr = []
+    output_arr << %r{\s+user\s+system\s+total\s+real\s+\n}
+
+    cases.keys.each do |label|
+      output_arr << %r{#{label}}
+      4.times.each do
+        output_arr << %r{\s+\[\d+\.\d+,\d+\.\d+,\d+\.\d+\]}
+      end
+      output_arr << %r{\n}
+    end
+
+    output_regexp = output_arr.inject(//) { |o, r| Regexp.new(o.source + r.source) }
+
+    times = 2
+    proc do
+      Benchmark.experiment(times) do |x|
+        cases.each { |label, blk| x.report(label, &blk) }
+      end
+    end.must_output output_regexp
+  end
+
+  it 'returns an array with stats for each run case.' do
+    times = 20
+    results = Benchmark.experiment(times) do |x|
+      cases.each { |label, blk| x.report(label, &blk) }
+    end
+
+    results.size.must_equal 3
+    results.map{ |label, _| label}.must_equal cases.keys
+  end
+
+  it 'collects and stores the descriptive statistics into a JSON' do
+    times = 2
+    result = Benchmark.observe_and_summarize(times) do |x|
+      cases.each { |label, blk| x.report(label, &blk) }
+    end
+
+    items = JSON.parse(result)
+    items.size.must_equal 3
+    items['for:'].first['name'].must_equal 'utime'
+  end
+
+  it 'aggregates several benchmark results and ranks them' do
+    stat_one = {
+      'first' => [
+        {
+          'name' => 'total',
+          'median' => 10,
+          'sample' => [10] * 20
+        }
+      ]
+    }
+    stat_two = {
+      'second' => [
+        {
+          'name' => 'total',
+          'median' => 20,
+          'sample' => [20] * 20
+        }
+      ]
+    }
+    best, is_h0_rejected = Benchmark.aggregate_and_rank([stat_one, stat_two])
+    assert is_h0_rejected
+    best['label'].must_equal 'first'
+  end
+
+  it 'ranks given stats' do
+    stats = {
+      'first' => [
+        {
+          'name' => 'total',
+          'median' => 10,
+          'sample' => [10] * 20
+        }
+      ],
+      'second' => [
+        {
+          'name' => 'total',
+          'median' => 20,
+          'sample' => [20] * 20
+        }
+      ]
+    }
+    best, is_h0_rejected = Benchmark.rank(stats)
+    assert is_h0_rejected
+    best['label'].must_equal 'first'
+  end
+end

data/spec/unit/mann_whitney_u_test_spec.rb

@@ -0,0 +1,32 @@
+require_relative '../spec_helper'
+
+describe Benchmark::Experiment::MannWhitneyUTest do
+  let(:x)         { [19, 22, 16, 29, 24] }
+  let(:y_no_ties) { [20, 11, 17, 12] }
+  let(:y_ties)    { [19, 20, 11, 17, 12] }
+
+  it 'calculates U' do
+    Benchmark::Experiment::MannWhitneyUTest::calculate_U(x, y_no_ties).must_equal [3.0, 17.0]
+  end
+
+  it 'calculates U with ties' do
+    Benchmark::Experiment::MannWhitneyUTest::calculate_U(x, y_ties).must_equal [4.5, 20.5]
+  end
+
+  it 'calculates the standardized value z' do
+    Benchmark::Experiment::MannWhitneyUTest::calculate_z(x, y_no_ties).must_be_close_to -1.715
+  end
+
+  it 'calculates the standardized value z with ties' do
+    Benchmark::Experiment::MannWhitneyUTest::calculate_z(x, y_ties).must_be_close_to -1.853
+  end
+
+  it 'calculates p-value' do
+    z = Benchmark::Experiment::MannWhitneyUTest::calculate_z(x, y_no_ties)
+    Benchmark::Experiment::MannWhitneyUTest::calculate_probability_z(z).must_be_close_to 0.086
+  end
+
+  it 'rejects null hypothesis' do
+    refute Benchmark::Experiment::MannWhitneyUTest::is_null_hypothesis_rejected?(0.9, 0.05)
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: benchmark-lab
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Christophe Philemotte
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: distribution
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Run Real Experiment and Calculate Non-Parametric Statistics.
+email:
+- christophe.philemotte@8thcolor.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".pullreview.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark-lab.gemspec
+- lib/benchmark/lab.rb
+- lib/benchmark/lab/descriptive_statistics.rb
+- lib/benchmark/lab/mann_whitney_u_test.rb
+- lib/benchmark/lab/version.rb
+- spec/spec_helper.rb
+- spec/unit/descriptive_statistics_spec.rb
+- spec/unit/experiment_spec.rb
+- spec/unit/mann_whitney_u_test_spec.rb
+homepage: https://github.com/toch/benchmark-lab
+licenses:
+- GPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Run Real Experiment and Calculate Non-Parametric Statistics.
+test_files:
+- spec/spec_helper.rb
+- spec/unit/descriptive_statistics_spec.rb
+- spec/unit/experiment_spec.rb
+- spec/unit/mann_whitney_u_test_spec.rb
+has_rdoc: