benchmark-perf 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b05020c28d534d0348e045bb0778e18ecabd5236
+  data.tar.gz: bcf5fdf5efbca164cce07a5718c01000d32f4671
+SHA512:
+  metadata.gz: f8392ae93fa4f2e48de3c41fb987a188b781410b690f8bcf8c47df9ee019ffdced40de4f424c587c6ada25237909ca018593523321da2077c03e619c49db2d5f
+  data.tar.gz: 5fa5b6298532e8936b85e615a2e0b1ce748e1cedb209e22621ec7313611f017b80627aa522977f65736668889caeb3380eca65cdb45a814fe011d7528e1121ad

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,28 @@
+---
+language: ruby
+sudo: false
+cache: bundler
+bundler_args: --without yard benchmarks
+script: "bundle exec rake ci"
+rvm:
+  - 1.9.3
+  - 2.0
+  - 2.1
+  - 2.2
+  - ruby-head
+  - jruby-19mode
+  - jruby
+  - jruby-head
+  - rbx-2
+env:
+  global:
+    - JRUBY_OPTS="-Xcli.debug=true --debug"
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-head
+  fast_finish: true
+branches:
+  only: master
+notifications:
+  email: false

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Change log
+
+## [v0.1.0] - 2016-01-25
+
+Initial release
+
+[v0.1.0]: https://github.com/peter-murach/benchmark-perf/compare/v0.1.0

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'rspec', '~> 3.4.0'
+  gem 'yard',  '~> 0.8.7'
+end
+
+group :metrics do
+  gem 'coveralls', '~> 0.8.9'
+  gem 'simplecov', '~> 0.10.0'
+  gem 'yardstick', '~> 0.9.9'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Piotr Murach
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+# Benchmark::Perf
+
+[![Gem Version](https://badge.fury.io/rb/benchmark-perf.svg)][gem]
+[![Build Status](https://secure.travis-ci.org/peter-murach/benchmark-perf.svg?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/peter-murach/benchmark-perf/badges/gpa.svg)][codeclimate]
+[![Coverage Status](https://coveralls.io/repos/github/peter-murach/benchmark-perf/badge.svg?branch=master)][coverage]
+[![Inline docs](http://inch-ci.org/github/peter-murach/benchmark-perf.svg?branch=master)][inchpages]
+
+[gem]: http://badge.fury.io/rb/benchmark-perf
+[travis]: http://travis-ci.org/peter-murach/benchmark-perf
+[codeclimate]: https://codeclimate.com/github/peter-murach/benchmark-perf
+[coverage]: https://coveralls.io/github/peter-murach/benchmark-perf?branch=master
+[inchpages]: http://inch-ci.org/github/peter-murach/benchmark-perf
+
+> Measure execution time and iterations per second.
+
+The **Benchmark::Perf** is used by [rspec-benchmark](https://github.com/peter-murach/rspec-benchmark)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'benchmark-perf'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install benchmark-perf
+
+## Usage
+
+To see how long it takes to execute code do:
+
+```ruby
+bench = Benchmark::Perf::ExecutionTime.new
+mean, stddev = bench.run { ... }
+```
+
+By default `30` samples are taken, and `1` sample for the warmup phase. If you need to change number of measurement samples do:
+
+```ruby
+bench = Benchmark::Perf::ExecutionTime.new samples: 10
+```
+
+And to change number of warmup cycles do:
+
+```ruby
+bench = Benchmark::Perf::ExecutionTime.new warmup: 2
+```
+
+In order to check how many iterations per second a given code takes do:
+
+```ruby
+bench = Benchmark::Perf::Iteration.new
+mean, stddev, iter, elapsed_time = bench.run { ... }
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/benchmark-perf/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Copyright
+
+Copyright (c) 2016 Piotr Murach. See LICENSE for further details.

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+require "bundler/gem_tasks"
+
+FileList['tasks/**/*.rake'].each(&method(:import))
+
+desc 'Run all specs'
+task ci: %w[ spec ]
+
+task default: :spec
+

data/benchmark-perf.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'benchmark/perf/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "benchmark-perf"
+  spec.version       = Benchmark::Perf::VERSION
+  spec.authors       = ["Piotr Murach"]
+  spec.email         = [""]
+  spec.summary       = %q{Execution time and iteration performance benchmarking}
+  spec.description   = %q{Execution time and iteration performance benchmarking}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', '>= 1.5.0', '< 2.0'
+  spec.add_development_dependency 'rake'
+end

data/lib/benchmark-perf.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+
+require 'benchmark/perf'

data/lib/benchmark/perf.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+require 'benchmark'
+
+require 'benchmark/perf/execution_time'
+require 'benchmark/perf/iteration'
+require 'benchmark/perf/version'
+
+module Benchmark
+  module Perf
+    # Calculate arithemtic average of measurements
+    #
+    # @param [Array[Float]] measurements
+    #
+    # @return [Float]
+    #   the average of given measurements
+    #
+    # @api public
+    def average(measurements)
+      return 0 if measurements.empty?
+      measurements.reduce(&:+).to_f / measurements.size
+    end
+
+    # Calculate standard deviation
+    #
+    # @param [Array[Float]] measurements
+    #
+    # @api public
+    def std_dev(measurements)
+      return 0 if measurements.empty?
+      average = average(measurements)
+      Math.sqrt(
+        measurements.reduce(0) do |sum, x|
+          sum + (x - average)**2
+        end.to_f / (measurements.size - 1)
+      )
+    end
+
+    # Run given work and gather time statistics
+    #
+    # @param [Float] threshold
+    #
+    # @return [Boolean]
+    #
+    # @api public
+    def assert_perform_under(threshold, options = {}, &work)
+      bench = ExecutionTime.new(options)
+      actual, _ = bench.run(&work)
+      actual <= threshold
+    end
+
+    # Assert work is performed within expected iterations per second
+    #
+    # @param [Integer] iterations
+    #
+    # @return [Boolean]
+    #
+    # @api public
+    def assert_perform_ips(iterations, options = {}, &work)
+      bench = Iteration.new(options)
+      mean, stddev, _ = bench.run(&work)
+      iterations <= (mean + 3 * stddev)
+    end
+
+    extend Benchmark::Perf
+  end # Perf
+end # Benchmark

data/lib/benchmark/perf/execution_time.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+module Benchmark
+  module Perf
+    # Measure length of time the work could take on average
+    #
+    # @api public
+    class ExecutionTime
+      attr_reader :io
+
+      # Initialize execution time
+      #
+      # @param [Hash] options
+      #
+      # @param options :warmup
+      #   the number of cycles for warmup, default 1
+      #
+      # @api public
+      def initialize(options = {})
+        @io      = options.fetch(:io) { nil }
+        @samples = options.fetch(:samples) { 30 }
+        @warmup  = options.fetch(:warmup) { 1 }
+      end
+
+      # Set of ranges in linear progression
+      #
+      # @api private
+      def linear_range(min, max, step = 1)
+        (min..max).step(step).to_a
+      end
+
+      def bench_range
+        linear_range(1, @samples)
+      end
+
+      # Isolate run in subprocess
+      #
+      # @example
+      #   iteration.run_in_subproces { ... }
+      #
+      # @return [Float]
+      #   the elapsed time of the measurement
+      #
+      # @api private
+      def run_in_subprocess
+        return yield unless Process.respond_to?(:fork)
+
+        reader, writer = IO.pipe
+        pid = Process.fork do
+          GC.start
+          GC.disable if ENV['BENCH_DISABLE_GC']
+          reader.close
+
+          time = yield
+
+          io.print "%9.6f" % time if io
+          writer.write(Marshal.dump(time))
+          GC.enable if ENV['BENCH_DISABLE_GC']
+          exit!(0) # run without hooks
+        end
+
+        writer.close
+        Process.waitpid(pid)
+        Marshal.load(reader.read)
+      end
+
+      # Run warmup measurement
+      #
+      # @api private
+      def run_warmup(&work)
+        GC.start
+        @warmup.times do
+          run_in_subprocess { ::Benchmark.realtime(&work) }
+        end
+      end
+
+      # Perform work x times
+      #
+      # @param [Integer] times
+      #   how many times sample the code measuremenets
+      #
+      # @example
+      #   iteration = Iteration.new
+      #   iteration.run(10) { ... }
+      #
+      # @return [Array[Float, Float]]
+      #   average and standard deviation
+      #
+      # @api public
+      def run(times = (not_set = true), &work)
+        range = not_set ? bench_range : (0..times)
+        measurements = []
+        run_warmup(&work)
+
+        range.each do
+          GC.start
+
+          measurements << run_in_subprocess do
+            ::Benchmark.realtime(&work)
+          end
+        end
+        io.puts if io
+
+        [Perf.average(measurements), Perf.std_dev(measurements)]
+      end
+    end # ExecutionTime
+  end # Perf
+end # Benchmark

data/lib/benchmark/perf/iteration.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+
+module Benchmark
+  module Perf
+    # Measure number of iterations a work could take in a second
+    #
+    # @api private
+    class Iteration
+      MICROSECONDS_PER_SECOND = 1_000_000
+      MICROSECONDS_PER_100MS = 100_000
+
+      def initialize(options = {})
+        @time   = options.fetch(:time) { 2 } # Default 2 seconds for measurement
+        @warmup = options.fetch(:warmup) { 1 }
+      end
+
+      # Call work by given times
+      #
+      # @param [Integer] times
+      #   the times to call
+      #
+      # @return [Integer]
+      #   the number of times worke has been called
+      #
+      # @api private
+      def call_times(times)
+        i = 0
+        while i < times
+          yield
+          i += 1
+        end
+      end
+
+      # Calcualte the number of cycles needed for 100ms
+      #
+      # @param [Integer] iterations
+      # @param [Float] elapsed_time
+      #   the total time for all iterations
+      #
+      # @return [Integer]
+      #   the cycles per 100ms
+      #
+      # @api private
+      def cycles_per_100ms(iterations, elapsed_time)
+        cycles = (iterations * (MICROSECONDS_PER_100MS / elapsed_time)).to_i
+        cycles <= 0 ? 1 : cycles
+      end
+
+      # Warmup run
+      #
+      # @api private
+      def run_warmup(&work)
+        GC.start
+        target = Time.now + @warmup
+        iter = 0
+
+        elapsed_time = ::Benchmark.realtime do
+          while Time.now < target
+            call_times(1, &work)
+            iter += 1
+          end
+        end
+
+        elapsed_time *= MICROSECONDS_PER_SECOND
+        cycles_per_100ms(iter, elapsed_time)
+      end
+
+      # Run measurements
+      #
+      # @api public
+      def run(&work)
+        target = Time.now + @time
+        iter = 0
+        measurements = []
+        cycles = run_warmup(&work)
+
+        GC.start
+
+        while Time.now < target
+          time = ::Benchmark.realtime { call_times(cycles, &work) }
+          next if time <= 0.0 # Iteration took no time
+          iter += cycles
+          measurements << time * MICROSECONDS_PER_SECOND
+        end
+
+        ips = measurements.map do |time_ms|
+          (cycles / time_ms) * MICROSECONDS_PER_SECOND
+        end
+
+        final_time = Time.now
+        elapsed_time = (final_time - target).abs
+
+        [Perf.average(ips).round, Perf.std_dev(ips).round, iter, elapsed_time]
+      end
+    end # Iteration
+  end # Perf
+end # Benchmark

data/lib/benchmark/perf/version.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+module Benchmark
+  module Perf
+    VERSION = "0.1.0"
+  end # Perf
+end # Benchmark

data/spec/spec_helper.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+if RUBY_VERSION > '1.9' and (ENV['COVERAGE'] || ENV['TRAVIS'])
+  require 'simplecov'
+  require 'coveralls'
+
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+  ]
+
+  SimpleCov.start do
+    command_name 'spec'
+    add_filter 'spec'
+  end
+end
+
+require 'benchmark-perf'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  config.disable_monkey_patching!
+
+  config.warnings = true
+
+  if config.files_to_run.one?
+    config.default_formatter = 'doc'
+  end
+
+  config.profile_examples = 2
+
+  config.order = :random
+
+  Kernel.srand config.seed
+end

data/spec/unit/assertions_spec.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+RSpec.describe Benchmark::Perf, '#assert_perform' do
+  it "passes asertion by performing under threshold" do
+    bench = Benchmark::Perf
+    assertion = bench.assert_perform_under(0.01, samples: 2) { 'x' * 1_024 }
+    expect(assertion).to eq(true)
+  end
+
+  it "passes asertion by performing 10K ips" do
+    bench = Benchmark::Perf
+    assertion = bench.assert_perform_ips(10_000) { 'x' * 1_024 }
+    expect(assertion).to eq(true)
+  end
+end

data/spec/unit/average_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+RSpec.describe Benchmark::Perf do
+  it "calculates average without measurements" do
+    expect(Benchmark::Perf.average([])).to eq(0)
+  end
+
+  it "calculates average with measurements" do
+    expect(described_class.average([1,2,3])).to eq(2.0)
+  end
+end

data/spec/unit/execution_time_spec.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+RSpec.describe Benchmark::Perf::ExecutionTime do
+  it "provides linear range" do
+    bench = described_class.new
+    expect(bench.linear_range(0, 3)).to eq([0,1,2,3])
+  end
+
+  it "provides default benchmark range" do
+    bench = described_class.new
+    expect(bench.bench_range.size).to eq(30)
+  end
+
+  it "provides measurements for 30 samples by default" do
+    bench = described_class.new
+    sample = bench.run { 'x' * 1024 }
+    expect(sample).to all(be < 0.01)
+  end
+
+  it "executes code to warmup ruby vm" do
+    bench = described_class.new
+    sample = bench.run_warmup { 'x' * 1_000_000 }
+    expect(sample).to eq(1)
+  end
+
+  it "measures work performance for 10 samples" do
+    bench = described_class.new
+    sample = bench.run(10) { 'x' * 1_000_000 }
+    expect(sample.size).to eq(2)
+    expect(sample).to all(be < 0.01)
+  end
+end

data/spec/unit/iteration_spec.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+RSpec.describe Benchmark::Perf::Iteration do
+  it "defines cycles per 100 microseconds" do
+    bench = described_class.new
+    sample = bench.run_warmup { 'x' * 1_000_000 }
+    expect(sample).to be > 25
+  end
+
+  it "measures 10K iterations per second" do
+    bench = described_class.new
+    sample = bench.run { 'x' * 1_000_000 }
+    expect(sample.size).to eq(4)
+    expect(sample[0]).to be > 250
+    expect(sample[1]).to be > 5
+    expect(sample[2]).to be > 250
+  end
+end

data/tasks/coverage.rake

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+desc 'Measure code coverage'
+task :coverage do
+  begin
+    original, ENV['COVERAGE'] = ENV['COVERAGE'], 'true'
+    Rake::Task['spec'].invoke
+  ensure
+    ENV['COVERAGE'] = original
+  end
+end

data/tasks/spec.rake

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+begin
+  require 'rspec/core/rake_task'
+
+  desc 'Run all specs'
+  RSpec::Core::RakeTask.new(:spec) do |task|
+    task.pattern = 'spec/{unit,integration}{,/*/**}/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run unit specs'
+    RSpec::Core::RakeTask.new(:unit) do |task|
+      task.pattern = 'spec/unit{,/*/**}/*_spec.rb'
+    end
+
+    desc 'Run integration specs'
+    RSpec::Core::RakeTask.new(:integration) do |task|
+      task.pattern = 'spec/integration{,/*/**}/*_spec.rb'
+    end
+
+    desc 'Run performance specs'
+    RSpec::Core::RakeTask.new(:perf) do |task|
+      task.pattern = 'spec/performance{,/*/**}/*_spec.rb'
+    end
+  end
+
+rescue LoadError
+  %w[spec spec:unit spec:integration spec:perf].each do |name|
+    task name do
+      $stderr.puts "In order to run #{name}, do `gem install rspec`"
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: benchmark-perf
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Piotr Murach
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Execution time and iteration performance benchmarking
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark-perf.gemspec
+- lib/benchmark-perf.rb
+- lib/benchmark/perf.rb
+- lib/benchmark/perf/execution_time.rb
+- lib/benchmark/perf/iteration.rb
+- lib/benchmark/perf/version.rb
+- spec/spec_helper.rb
+- spec/unit/assertions_spec.rb
+- spec/unit/average_spec.rb
+- spec/unit/execution_time_spec.rb
+- spec/unit/iteration_spec.rb
+- tasks/coverage.rake
+- tasks/spec.rake
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Execution time and iteration performance benchmarking
+test_files:
+- spec/spec_helper.rb
+- spec/unit/assertions_spec.rb
+- spec/unit/average_spec.rb
+- spec/unit/execution_time_spec.rb
+- spec/unit/iteration_spec.rb
+has_rdoc: