benchmark_driver 0.8.6 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: a012a5ba29bb3da226c6d809ffe149a368355f1aad15d1604c9ae16ee4a1f718
-  data.tar.gz: 1e1667a847cb9eacfaf0c6095f6fc5755c8904ae96b12e0bd73540799469ba86
+SHA1:
+  metadata.gz: 0cd5a98489bf28a8f6439e66e91350b20d330433
+  data.tar.gz: bd97724eb0f7bb197bd2453b8eccb17e8fa5dcf2
 SHA512:
-  metadata.gz: d9b870fd971c13c6c3f74b951a5da8c9ef9ccbeaf8a415cd3c56d975973d8a35e6287c3850d50bacd023e861de0b5552c866fc73b5c4dfec65ccfa26f47e0934
-  data.tar.gz: c43161e186236f6c82d3e8f9b5577ee935747f89badb6bbf4698f42488d0e12fb3593a6bfc7ab00e3b870652ea1968466c7855d813935019eff23dc6a51dbece
+  metadata.gz: 9debc903a940716911021dafe54dadd4be762259bb8c170c13266b6c15bd56f1762368d4a8982e75d326870b9611be65ba9fe30081a71c91874c489a516aa115
+  data.tar.gz: 72dbf0e5038f7218311364bf5226c4b909a40bb4ab343ccb393a2e76a14c8fe8db4d4f986203fcf6d16a0147afcdd1cf0b8456c19be511b31e49dfbb1b699112

data/.travis.yml

@@ -5,13 +5,11 @@ rvm:
   - 2.3.6
   - 2.4.3
   - 2.5.0
-  - ruby-head
 cache: bundler
 branches:
   only:
     - master
-before_install:
-  - gem update --system
+before_install: gem install bundler -v 1.15.4
 script:
   - bundle exec rake ruby_examples
   - bundle exec rake yaml_examples

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# v0.9.0
+
+- The concept of runner is renewed
+  - Now it's for specifying metrics like ips, time, memory usage
+  - Old runners (:call and :eval) are no longer supported. :exec only.
+     - So Ruby interface can't take Proc
+  - YAML can have arbitrary format depending on the runner
+- `--compare` option is dropped and changed to `--output compare`
+
 # v0.8.6
 
 - Automatically require `benchmark/output/foo` when `-o foo` is specified

data/Gemfile

@@ -2,12 +2,7 @@ source 'https://rubygems.org'
 
 git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
-# Specify your gem's dependencies in benchmark_driver.gemspec
+# Specify your gem's dependencies in benchmark-driver.gemspec
 gemspec
 
-# For my debugging
 gem 'pry'
-
-# For benchmark examples
-gem 'erubi'
-gem 'erubis'

data/README.md

@@ -1,23 +1,24 @@
-# Benchmark::Driver [![Build Status](https://travis-ci.org/k0kubun/benchmark_driver.svg?branch=master)](https://travis-ci.org/k0kubun/benchmark_driver)
+# BenchmarkDriver [![Build Status](https://travis-ci.org/k0kubun/benchmark_driver.svg?branch=master)](https://travis-ci.org/k0kubun/benchmark_driver)
 
 Fully-featured accurate benchmark driver for Ruby
 
 ## Project Status
 
-Beta. Features are almost implemented but interface may change in the future.
+**Under Construction**
 
 ## Features
 ### Accurate Measurement
 
 - Low overhead benchmark by running generated script instead of calling Proc
+- Profiling memory, high-precision real time, user time and system time
 - Running multiple times to minimize measurement errors
-- Profiling memory, high-precision real time
 
 ### Pluggable & Fully Featured
 
 - Flexible and real-time output format in ips, execution time, markdown table, etc.
-- Runner and output are all pluggable
-- Bundler integration for benchmark that requires gems
+- Benchmark with various profiling/running options
+- Integrated benchmark support using external libraries
+- Runner, profiler and output format are all pluggable
 
 ### Flexible Interface
 
@@ -33,32 +34,19 @@ $ gem install benchmark_driver
 
 ## Usage
 
-### Ruby Interface: Compatible Mode
-
-This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's good for migration.
-
-```rb
-require 'benchmark/driver'
-require 'active_support/all'
-
-Benchmark.driver do |x|
-  array = []
-  x.report('blank?') { array.blank? }
-  x.report('empty?') { array.empty? }
-  x.compare!
-end
-```
-
-### Ruby Interface: Low Overhead Mode
+### Ruby Interface
 
 This interface generates code to profile with low overhead and executes it.
 
 ```rb
-require 'benchmark/driver'
-require 'active_support/all'
+require 'benchmark_driver'
+
+Benchmark.drive do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
 
-Benchmark.driver do |x|
-  x.prelude %{ array = [] }
   x.report 'blank?', %{ array.blank? }
   x.report 'empty?', %{ array.empty? }
 end
@@ -67,13 +55,16 @@ end
 or simply:
 
 ```rb
-require 'benchmark/driver'
-require 'active_support/all'
+require 'benchmark_driver'
 
-Benchmark.driver do |x|
-  x.prelude %{ array = [] }
-  x.report %{ array.blank?' }
-  x.report %{ array.empty?' }
+Benchmark.drive do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
+
+  x.report %{ array.blank? }
+  x.report %{ array.empty? }
 end
 ```
 
@@ -84,10 +75,14 @@ With `benchmark-driver` command, you can describe benchmark with YAML input.
 ```
 $ benchmark-driver -h
 Usage: benchmark-driver [options] [YAML]
-    -e, --executables [EXECS]        Ruby executables (e1::path1; e2::path2; e3::path3;...)
-        --rbenv [VERSIONS]           Ruby executables in rbenv (2.3.5;2.4.2;...)
-    -c, --compare                    Compare results (currently only supported in ips output)
-    -r, --repeat-count [NUM]         Try benchmark NUM times and use the fastest result
+    -r, --runner [TYPE]              Specify runner type: ips, time, memory (default: ips)
+    -o, --output [TYPE]              Specify output type: compare, simple, markdown (default: compare)
+    -e, --executables [EXECS]        Ruby executables (e1::path1,arg1,...; e2::path2,arg2;...)
+        --rbenv [VERSIONS]           Ruby executables in rbenv (x.x.x,arg1,...;y.y.y,arg2,...;...)
+        --repeat-count [NUM]         Try benchmark NUM times and use the fastest result (TODO)
+        --bundler                    Install and use gems specified in Gemfile
+        --filter [REGEXP]            Filter out benchmarks with given regexp
+        --run-duration [SECONDS]     Warmup esitmates loop_count to run for this duration (default: 3)
 ```
 
 #### Running single script
@@ -104,16 +99,17 @@ benchmark: erb.result
 you can benchmark the script with multiple ruby executables.
 
 ```
-$ exe/benchmark-driver examples/yaml/example_single.yml --rbenv '2.4.2;trunk' --compare
+$ benchmark-driver example_single.yml --rbenv '2.4.1;2.5.0'
 Warming up --------------------------------------
-          erb.result    10.973k i/100ms
+          erb.result    71.683k i/s
 Calculating -------------------------------------
-                          2.4.2       trunk
-          erb.result   109.268k    123.611k i/s -    548.675k in 4.017080s 4.438720s
+                          2.4.1       2.5.0
+          erb.result    72.387k     75.046k i/s -    215.049k times in 2.970833s 2.865581s
 
 Comparison:
-  erb.result (trunk):    123611.1 i/s
-  erb.result (2.4.2):    109268.4 i/s - 1.13x  slower
+                       erb.result
+               2.5.0:     75045.5 i/s
+               2.4.1:     72386.8 i/s - 1.04x  slower
 ```
 
 #### Running multiple scripts
@@ -133,20 +129,23 @@ benchmark:
 you can benchmark the scripts with multiple ruby executables.
 
 ```
-$ exe/benchmark-driver examples/yaml/example_multi.yml --rbenv '2.4.2;trunk' --compare
+$ benchmark-driver example_multi.yml --rbenv '2.4.1;2.5.0'
 Warming up --------------------------------------
-                join   515.787k i/100ms
-          str-interp   438.646k i/100ms
+                join     2.509M i/s
+          str-interp     1.772M i/s
 Calculating -------------------------------------
-                          2.4.2       trunk
-                join     5.200M      4.740M i/s -     20.631M in 3.967750s 4.352565s
-          str-interp     4.306M      6.034M i/s -     21.932M in 4.075159s 3.634986s
+                          2.4.1       2.5.0
+                join     2.661M      2.863M i/s -      7.527M times in 2.828771s 2.629191s
+          str-interp     1.890M      3.258M i/s -      5.315M times in 2.812240s 1.630997s
 
 Comparison:
-  str-interp (trunk):   6033674.6 i/s
-        join (2.4.2):   5199794.6 i/s - 1.16x  slower
-        join (trunk):   4740075.1 i/s - 1.27x  slower
-  str-interp (2.4.2):   4305563.1 i/s - 1.40x  slower
+                             join
+               2.5.0:   2862755.1 i/s
+               2.4.1:   2660777.4 i/s - 1.08x  slower
+
+                       str-interp
+               2.5.0:   3258489.7 i/s
+               2.4.1:   1889805.6 i/s - 1.72x  slower
 ```
 
 ## Contributing

data/benchmark_driver.gemspec

@@ -1,10 +1,10 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'benchmark/driver/version'
+require 'benchmark_driver/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'benchmark_driver'
-  spec.version       = Benchmark::Driver::VERSION
+  spec.version       = BenchmarkDriver::VERSION
   spec.authors       = ['Takashi Kokubun']
   spec.email         = ['takashikkbn@gmail.com']
 
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.1.0'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/bin/console

@@ -1,14 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "benchmark/driver"
+require 'bundler/setup'
+require 'benchmark_driver'
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)
+require 'pry'
+Pry.start

data/examples/exec_blank.rb

@@ -1,6 +1,6 @@
-require 'benchmark/driver'
+require 'benchmark_driver'
 
-Benchmark.driver(runner: :exec) do |x|
+Benchmark.driver do |x|
   x.prelude <<-EOS
     class Array
       alias_method :blank?, :empty?

data/examples/exec_blank_simple.rb

@@ -1,6 +1,6 @@
-require 'benchmark/driver'
+require 'benchmark_driver'
 
-Benchmark.driver(runner: :exec) do |x|
+Benchmark.driver(output: :simple) do |x|
   x.prelude <<-EOS
     class Array
       alias_method :blank?, :empty?
@@ -9,5 +9,4 @@ Benchmark.driver(runner: :exec) do |x|
   EOS
   x.report %{ array.empty? }
   x.report %{ array.blank? }
-  x.compare!
 end

data/exe/benchmark-driver

@@ -1,110 +1,101 @@
 #!/usr/bin/env ruby
 $:.unshift File.expand_path('../lib', __dir__)
 
-require 'benchmark/driver'
-require 'benchmark/driver/yaml_parser'
+require 'benchmark_driver'
 require 'optparse'
 require 'yaml'
 
-#
+
 # Parse command line options
-#
-options = {}
-parser = OptionParser.new do |o|
-  o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML]"
-  o.on('-e', '--executables [EXECS]', 'Ruby executables (e1::path1,arg1,...; e2::path2,arg2;...)') do |e|
-    abort '-e, --executable must take argument but not given' if e.nil?
-    options[:execs] ||= []
-    e.split(';').each do |name_path|
-      options[:execs] << Benchmark::Driver::Configuration::Executable.parse(name_path)
+config = BenchmarkDriver::Config.new.tap do |c|
+  executables = []
+  bundler = false
+  parser = OptionParser.new do |o|
+    o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML]"
+    o.on('-r', '--runner [TYPE]', 'Specify runner type: ips, time, memory (default: ips)') do |d|
+      abort '-r, --runner must take argument but not given' if d.nil?
+      c.runner_type = d
     end
-  end
-  o.on('--rbenv [VERSIONS]', 'Ruby executables in rbenv (x.x.x,arg1,...;y.y.y,arg2,...;...)') do |r|
-    abort '--rbenv must take argument but not given' if r.nil?
-    options[:execs] ||= []
-    r.split(';').each do |spec|
-      options[:execs] << Benchmark::Driver::Configuration::Executable.parse_rbenv(spec)
+    o.on('-o', '--output [TYPE]', 'Specify output type: compare, simple, markdown (default: compare)') do |out|
+      abort '-o, --output must take argument but not given' if out.nil?
+      c.output_type = out
     end
-  end
-  o.on('-o', '--output [TYPE]', 'Specify output type (ips, time, memory, markdown)') do |t|
-    abort '-o, --output must take argument but not given' if t.nil?
-    options[:output] = t
-  end
-  o.on('-c', '--compare', 'Compare results (currently only supported in ips output)') do |v|
-    options[:compare] = v
-  end
-  o.on('-r', '--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result') do |v|
-    begin
-      options[:repeat_count] = Integer(v)
-    rescue ArgumentError
-      abort "-r, --repeat-count must take Integer, but got #{v.inspect}"
+    o.on('-e', '--executables [EXECS]', 'Ruby executables (e1::path1,arg1,...; e2::path2,arg2;...)') do |e|
+      abort '--executable must take argument but not given' if e.nil?
+      e.split(';').each do |name_path|
+        name, path = name_path.split('::', 2)
+        command = (path || name).split(',') # if `::` is not given, regard whole string as path
+        command[0] = File.expand_path(command[0])
+        executables << BenchmarkDriver::Config::Executable.new(name: name, command: command)
+      end
+    end
+    o.on('--rbenv [VERSIONS]', 'Ruby executables in rbenv (x.x.x,arg1,...;y.y.y,arg2,...;...)') do |r|
+      abort '--rbenv must take argument but not given' if r.nil?
+      r.split(';').each do |spec|
+        version, *args = spec.split(',')
+        executables << BenchmarkDriver::Config::Executable.new(
+          name: version,
+          command: [BenchmarkDriver::Rbenv.ruby_path(version), *args],
+        )
+      end
+    end
+    o.on('--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result (TODO)') do |v|
+      begin
+        c.repeat_count = Integer(v)
+      rescue ArgumentError
+        abort "-r, --repeat-count must take Integer, but got #{v.inspect}"
+      end
+    end
+    o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
+      bundler = v
+    end
+    o.on('--filter [REGEXP]', 'Filter out benchmarks with given regexp') do |v|
+      c.filters << Regexp.compile(v)
+    end
+    o.on('--run-duration [SECONDS]', 'Warmup esitmates loop_count to run for this duration (default: 3)') do |v|
+      begin
+        c.run_duration = Integer(v)
+      rescue ArgumentError
+        abort "--run-duration must take Integer, but got #{v.inspect}"
+      end
     end
   end
-  o.on('--filter [REGEXP]', 'Filter out benchmarks with given regexp') do |v|
-    abort '--filter can be used only once' if options.key?(:filter)
-    options[:filter] = v
+  c.paths = parser.parse!(ARGV)
+  if c.paths.empty?
+    abort "No YAML file is specified!\n\n#{parser.help}"
   end
-  o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
-    options[:bundler] = v
+
+  # Configs that need to be set lazily
+  unless executables.empty?
+    c.executables = executables
   end
-  o.on('--dir', 'Override __dir__ from "/tmp" to actual directory of YAML') do |v|
-    options[:dir] = v
+  if bundler
+    c.executables.each do |exec|
+      exec.command << '-rbundler/setup'
+    end
   end
-end
-args = parser.parse!(ARGV)
-if args.empty?
-  abort "No YAML file is specified!\n\n#{parser.help}"
+
+  c.freeze
 end
 
-#
-# Parse benchmark definitions
-#
-jobs = args.flat_map do |path|
-  yaml = YAML.load(File.read(path))
-  Benchmark::Driver::Configuration.symbolize_keys!(yaml)
+# Parse benchmark job definitions
+jobs = config.paths.flat_map do |path|
+  job = YAML.load(File.read(path))
+  job = { 'type' => config.runner_type }.merge!(job)
 
   begin
-    Benchmark::Driver::YamlParser.parse(yaml)
+    BenchmarkDriver::JobParser.parse(job)
   rescue ArgumentError
     $stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
     $stderr.puts '  YAML format may be wrong. See error below:'
     $stderr.puts
     raise
   end
-end
-
-#
-# Proceed parsed options
-#
-config = Benchmark::Driver::Configuration.new(jobs)
-config.runner_options = Benchmark::Driver::Configuration::RunnerOptions.new
-config.output_options = Benchmark::Driver::Configuration::OutputOptions.new(:ips)
-
-options.each do |key, value|
-  case key
-  when :bundler
-    config.runner_options.bundler = value
-  when :compare
-    config.output_options.compare = value
-  when :dir
-    dir = File.dirname(path)
-    config.jobs.each do |job|
-      job.prelude = "__dir__ = #{dir.dump}.freeze; #{job.prelude}"
-    end
-  when :execs
-    config.runner_options.executables = options.delete(:execs)
-  when :filter
-    filter = Regexp.compile(value)
-    config.jobs.select! do |job|
-      job.name.match(filter)
-    end
-  when :output
-    config.output_options.type = value.to_sym
-  when :repeat_count
-    config.runner_options.repeat_count = value
-  else
-    raise "Unhandled option: #{key.inspect}"
+end.select do |job|
+  config.filters.all? do |filter|
+    job.name.match(filter)
   end
 end
 
-Benchmark::Driver.run(config)
+# Run jobs
+BenchmarkDriver::Runner.run(jobs, config: config)