benchmark_driver 0.10.16 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 6082e75d0eda1a86f600c95730d10b0dfd1c50ce0015e3620109a975a302ac70
-  data.tar.gz: a864a92790907f9ce4f04187f064d42d99dbb8e0c84a5daf4373da910ef9addf
+SHA1:
+  metadata.gz: 16b3e45ce8476d4acf4e3528d04079f1346b6436
+  data.tar.gz: 0e4a7a94f47ba9bb596ba96904712db4c6935032
 SHA512:
-  metadata.gz: c5cafbd01fdb7192915ac1372f173aadd62f2d19da708d2d224e1cc3044d763cb15b57037dd19efa90bbc3c85c669daa0b9a2f15ea697182888f8c2d255f8238
-  data.tar.gz: 8108b55f9a9de835e5e42237c9cdf661ebcdb2b54ef41ce58ae02c5df8312248fed6aa76e9bc874704fe8befe2093a17a96aae712e09fb73a5d92c09717163d0
+  metadata.gz: 75b834584e0fc996dda837d879af768c6bab9acba07fb53f284103ec73aafd53b8b5cc82001b9067a7d0403420c97fa30f59146bcff9c3341c89cf4c003e915e
+  data.tar.gz: cafd73b0bdaeb3b8635a0cbc847eaf9d3b6969d80369093004db615618cab41b6e004640f81a58bd8d38389243fc4061c230473ed696b7524bc15d23bbe6d87d

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper.rb

data/.travis.yml

@@ -10,4 +10,4 @@ branches:
     - master
 before_install: gem install bundler -v 1.15.4
 script:
-  - bundle exec rake
+  - RSPEC_RETRIES=3 VERBOSE=1 bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# v0.11.0
+
+- [breaking change] Plugin interface is completely changed, so all plugins need migration
+  - YAML/Ruby interface is not changed at all
+  - Now the internal model allows to have multiple metrics in the same job,
+    but having multiple metrics is still not respected by built-in plugins
+  - "executable" is renamed to be "context", but still configuring a context
+    other than an executable is not supported yet
+  - A metric can have a name
+- Add `ruby_stdout` runner
+  - This can parse an arbitrary "environment" for the "context"
+  - Metric can have name
+- Metric name is shwon on some outputs like markdown and simple
+- `--run-duration` accepts floating-point number
+- News: Now benchmark-driver.gem can be used as an alias to install benchmark\_driver.gem
+
 # v0.10.16
 
 - `command_stdout` runner uses YAML's dirname as `working_directory` by default

data/README.md

@@ -1,4 +1,4 @@
-# BenchmarkDriver [![Build Status](https://travis-ci.org/k0kubun/benchmark_driver.svg?branch=master)](https://travis-ci.org/k0kubun/benchmark_driver)
+# BenchmarkDriver [![Build Status](https://travis-ci.org/benchmark-driver/benchmark-driver.svg?branch=master)](https://travis-ci.org/benchmark-driver/benchmark-driver)
 
 Fully-featured accurate benchmark driver for Ruby
 
@@ -168,10 +168,26 @@ Benchmark.driver do |x|
 end
 ```
 
+With following `blank_loop.yml`,
+
+```
+loop_count: 20000000
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?
+```
+
+you can output results in various ways like:
+
 ### compare
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --output compare --rbenv '2.4.2;2.5.0'
+$ benchmark-driver blank_loop.yml --output compare --rbenv '2.4.2;2.5.0'
 Calculating -------------------------------------
                           2.4.2       2.5.0
                empty   195.957M    129.970M i/s -     20.000M times in 0.102063s 0.153882s
@@ -190,7 +206,7 @@ Comparison:
 ### simple
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --output simple --rbenv '2.4.2;2.5.0'
+$ benchmark-driver blank_loop.yml --output simple --rbenv '2.4.2;2.5.0'
 benchmark results (i/s):
           2.4.2     2.5.0
 empty  184.084M  117.319M
@@ -200,7 +216,7 @@ blank   65.843M   62.093M
 ### markdown
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --output markdown --rbenv '2.4.2;2.5.0'
+$ benchmark-driver blank_loop.yml --output markdown --rbenv '2.4.2;2.5.0'
 ```
 
 benchmark results (i/s)
@@ -215,7 +231,7 @@ benchmark results (i/s)
 Measure first, output with various formats later.
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --output record --rbenv '2.4.2;2.5.0'
+$ benchmark-driver blank_loop.yml --output record --rbenv '2.4.2;2.5.0'
 benchmarking....
 
 $ benchmark-driver benchmark_driver.record.yml --output compare
@@ -262,7 +278,7 @@ ips, time, memory, once
 ### ips
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --runner ips --rbenv '2.4.3;2.5.0'
+$ benchmark-driver blank_loop.yml --runner ips --rbenv '2.4.3;2.5.0'
 Calculating -------------------------------------
                           2.4.3       2.5.0
                empty   228.802M    180.125M i/s -     20.000M times in 0.087412s 0.111034s
@@ -281,7 +297,7 @@ Comparison:
 ### time
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --runner time --rbenv '2.4.3;2.5.0'
+$ benchmark-driver blank_loop.yml --runner time --rbenv '2.4.3;2.5.0'
 Calculating -------------------------------------
                           2.4.3       2.5.0
                empty      0.087       0.110 s -     20.000M times
@@ -300,7 +316,7 @@ Comparison:
 ### memory
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --runner memory --rbenv '2.4.3;2.5.0'
+$ benchmark-driver blank_loop.yml --runner memory --rbenv '2.4.3;2.5.0'
 Calculating -------------------------------------
                           2.4.3       2.5.0
                empty     9.192M      9.364M bytes -     20.000M times
@@ -321,7 +337,7 @@ Comparison:
 Only for testing purpose.
 
 ```
-$ benchmark-driver examples/yaml/blank_loop.yml --runner once --rbenv '2.4.3;2.5.0'
+$ benchmark-driver blank_loop.yml --runner once --rbenv '2.4.3;2.5.0'
 Calculating -------------------------------------
                           2.4.3       2.5.0
                empty     1.818M      2.681M i/s -       1.000 times in 0.000001s 0.000000s

data/Rakefile

@@ -1,52 +1,9 @@
 require 'bundler/gem_tasks'
-require 'shellwords'
+require 'rspec/core/rake_task'
 
-task :test do
-  blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
-  blank_hash = File.expand_path('./examples/yaml/blank_hash.yml', __dir__) # needs warmup
-  {
-    'ips' => 'compare',
-    'time' => 'simple',
-    'memory' => 'simple',
-    'once' => 'markdown',
-  }.each do |runner, output|
-    Bundler.with_clean_env do
-      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', runner, '-o', output].shelljoin
-      puts
-      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_hash, '-r', runner, '-o', output, '--run-duration', '1'].shelljoin
-      puts
-    end
-  end
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.ruby_opts = %w[-w]
+  t.rspec_opts = %w[--profile]
 end
 
-task :test_record do
-  blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
-  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', 'ips', '-o', 'record'].shelljoin
-  puts
-  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'compare'].shelljoin
-  puts
-  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'record'].shelljoin
-  puts
-  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'simple'].shelljoin
-  puts
-end
-
-task :test_ruby do
-  Dir.glob(File.expand_path('./examples/*.rb', __dir__)).sort.each do |file|
-    Bundler.with_clean_env do
-      sh ['time', 'bundle', 'exec', 'ruby', file].shelljoin
-    end
-    puts
-  end
-end
-
-task :test_yaml do
-  Dir.glob(File.expand_path('./examples/yaml/*.yml', __dir__)).sort.each do |file|
-    Bundler.with_clean_env do
-      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', file, '--run-duration', '1'].shelljoin
-    end
-    puts
-  end
-end
-
-task default: [:test, :test_record, :test_ruby, :test_yaml]
+task default: :spec

data/benchmark-driver/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/benchmark-driver/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at mamaveb@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/benchmark-driver/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in benchmark-driver.gemspec
+gemspec
+
+gem 'rake'

data/benchmark-driver/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 bmarkons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/benchmark-driver/README.md

@@ -0,0 +1,8 @@
+# benchmark-driver.gem
+
+This is an alias to install `benchmark_driver.gem`.
+It was built because its command name is actually `benchmark-driver`.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/benchmark-driver/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/benchmark-driver/benchmark-driver.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "benchmark-driver"
+  spec.version       = "0.1.0"
+  spec.authors       = ["bmarkons"]
+  spec.email         = ["mamaveb@gmail.com"]
+
+  spec.summary       = %q{Alias gem to install benchmark_driver.gem}
+  spec.homepage      = "https://github.com/benchmark-driver/benchmark-driver"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "benchmark_driver"
+end

data/benchmark-driver/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "benchmark/driver"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/benchmark-driver/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/benchmark-driver/lib/benchmark-driver.rb

@@ -0,0 +1 @@
+require 'benchmark_driver'

data/benchmark-driver/lib/benchmark/driver.rb

@@ -0,0 +1 @@
+require 'benchmark_driver'

data/benchmark_driver.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'Fully-featured accurate benchmark driver for Ruby'
   spec.description   = 'Fully-featured accurate benchmark driver for Ruby'
-  spec.homepage      = 'https://github.com/k0kubun/benchmark_driver'
+  spec.homepage      = 'https://github.com/benchmark-driver/benchmark-driver'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -23,4 +23,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec-retry'
 end

data/exe/benchmark-driver

@@ -35,7 +35,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
         executables << BenchmarkDriver::Rbenv.parse_spec(version)
       end
     end
-    o.on('--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result (TODO)') do |v|
+    o.on('--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result or the worst memory usage') do |v|
       begin
         c.repeat_count = Integer(v)
       rescue ArgumentError
@@ -57,9 +57,9 @@ config = BenchmarkDriver::Config.new.tap do |c|
     end
     o.on('--run-duration [SECONDS]', 'Warmup estimates loop_count to run for this duration (default: 3)') do |v|
       begin
-        c.run_duration = Integer(v)
+        c.run_duration = Float(v)
       rescue ArgumentError
-        abort "--run-duration must take Integer, but got #{v.inspect}"
+        abort "--run-duration must take Float, but got #{v.inspect}"
       end
     end
   end

data/lib/benchmark_driver/config.rb

@@ -9,14 +9,14 @@ module BenchmarkDriver
     :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
     :filters,      # @param [Array<Regexp>]
     :repeat_count, # @param [Integer]
-    :run_duration, # @param [Integer]
+    :run_duration, # @param [Float]
     :verbose,      # @param [Integer]
     defaults: {
       runner_type: 'ips',
       output_type: 'compare',
       filters: [],
       repeat_count: 1,
-      run_duration: 3,
+      run_duration: 3.0,
       verbose: 0,
     },
   )
@@ -25,7 +25,7 @@ module BenchmarkDriver
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
     :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
     :repeat_count, # @param [Integer]
-    :run_duration, # @param [Integer]
+    :run_duration, # @param [Float]
     :verbose,      # @param [Integer]
   )