benchmark_driver 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 95e89d2f7ab5cee3192bf505ab51a848615bee2c2b2f021cd9c41e62d23c641a
-  data.tar.gz: 3973aae7876002ae741bb6443a5054f3e7a8be4069e4989d61ef2a0b02615766
+SHA1:
+  metadata.gz: 771d93093383594ac93a9f3456b646f9ef484578
+  data.tar.gz: 41b2b41209e834a8983dad25843188325f8fb009
 SHA512:
-  metadata.gz: 546445be415fd833e7f180d853c5b44440f4ca76cf2a77cd5dfbef0d38fba7441aeefa71a99dc3ab97f6d2f7050d293670c70fc520d446228da680041a1d9b75
-  data.tar.gz: b309b8421f76a5039788556786678d863ed11d85bd891a6ab1bb1ffdf203a96db8d536cb3cc4ff1973b78bdaf296e87464ae00a8dbff6e3537045ffc5e844991
+  metadata.gz: 697ed2bab69c631bb9104adae293105b323b7ccb3321c123c85a18f44f5bb5a4e352fc06257b637233d8dbe73cb7ebc19aa63f93c8f53e8b9dbf10daa4c13bf9
+  data.tar.gz: ba4fd20c4dc8b506fa2971416446ba86594604c336b0a728aef071d385a3f46e7ea41f34e6061c0ace32bb61412d488fdfd7d2719b39cd2541b9ef8cb3f453c3

data/.gitignore

@@ -1,12 +1,8 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/
 /pkg/
 /spec/reports/
 /tmp/
-
-# rspec failure tracking
-.rspec_status

data/.travis.yml

@@ -1,11 +1,15 @@
-sudo: false
 language: ruby
 rvm:
+  - 2.1.10
+  - 2.2.8
+  - 2.3.5
   - 2.4.2
+  - ruby-head
+cache: bundler
+branches:
+  only:
+    - master
 before_install: gem install bundler -v 1.15.4
 script:
-  - bundle exec rake
-
-  # Test some options
-  - bundle exec exe/benchmark-driver benchmarks/example_single.yml -e ruby1::ruby -e ruby2::ruby -i
-  - bundle exec exe/benchmark-driver benchmarks/example_single.yml -e ruby1::ruby -e ruby2::ruby -i 2
+  - bundle exec rake ruby_examples
+  - bundle exec rake yaml_examples

data/Gemfile

@@ -1,8 +1,13 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in benchmark_driver.gemspec
 gemspec
 
+# For my debugging
 gem 'pry'
+
+# For benchmark examples
+gem 'erubi'
+gem 'erubis'

data/Gemfile.lock

@@ -0,0 +1,30 @@
+PATH
+  remote: .
+  specs:
+    benchmark_driver (0.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    erubi (1.7.0)
+    erubis (2.7.0)
+    method_source (0.9.0)
+    pry (0.11.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (12.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  benchmark_driver!
+  bundler
+  erubi
+  erubis
+  pry
+  rake
+
+BUNDLED WITH
+   1.16.0

data/README.md

@@ -1,23 +1,104 @@
 # Benchmark::Driver [![Build Status](https://travis-ci.org/k0kubun/benchmark_driver.svg?branch=master)](https://travis-ci.org/k0kubun/benchmark_driver)
 
-Benchmark driver for different Ruby executables
+Fully-featured accurate benchmark driver for Ruby
+
+## Project Status
+
+**Under Construction**
+
+## Features
+NOTE: Pending ones are ~slashed~.
+
+### Accurate Measurement
+
+- Low overhead benchmark by running generated script instead of calling Proc
+- Running multiple times to minimize measurement errors
+- Profiling memory, high-precision real time, ~user time and system time~
+
+### Pluggable & Fully Featured
+
+- Flexible and real-time output format in ips, execution time, ~markdown table~, etc.
+- Runner and output are all pluggable
+- ~Integrated benchmark support using external libraries~
+
+### Flexible Interface
+
+- Ruby interface similar to benchmark stdlib, benchmark-ips
+- YAML input to easily manage structured benchmark set
+- Comparing multiple Ruby binaries, even with miniruby
 
 ## Installation
 
-    $ gem install benchmark_driver
+```
+$ gem install benchmark_driver
+```
 
 ## Usage
 
+### Ruby Interface: Compatible Mode
+
+This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's good for migration.
+
+```rb
+require 'benchmark/driver'
+require 'active_support/all'
+array = []
+
+Benchmark.driver do |x|
+  x.report('blank?') { array.blank? }
+  x.report('empty?') { array.empty? }
+  x.compare!
+end
+```
+
+### Ruby Interface: Low Overhead Mode
+
+This interface generates code to profile with low overhead and executes it.
+
+```rb
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
+
+  x.report('blank?', script: 'array.blank?')
+  x.report('empty?', script: 'array.empty?')
+end
+```
+
+or simply:
+
+```rb
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
+
+  x.report(script: 'array.blank?')
+  x.report(script: 'array.empty?')
+end
+```
+
+### Structured YAML Input
+
+With `benchmark-driver` command, you can describe benchmark with YAML input.
+
 ```
 $ benchmark-driver -h
 Usage: benchmark-driver [options] [YAML]
     -e, --executables [EXECS]        Ruby executables (e1::path1; e2::path2; e3::path3;...)
-    -i, --ips [SECONDS]              Measure IPS in duration seconds (default: 1)
-    -l, --loop-count [COUNT]         Measure execution time with loop count (default: 100000)
-    -v, --verbose
+        --rbenv [VERSIONS]           Ruby executables in rbenv (2.3.5;2.4.2;...)
+    -c, --compare                    Compare results (currently only supported in ips output)
+    -r, --repeat-count [NUM]         Try benchmark NUM times and use the fastest result
 ```
 
-### Running single script
+#### Running single script
 
 With following `example_single.yml`,
 
@@ -31,31 +112,19 @@ benchmark: erb.result
 you can benchmark the script with multiple ruby executables.
 
 ```
-$ benchmark-driver benchmarks/example_single.yml -e ruby1::ruby -e ruby2::ruby
-benchmark results:
-Execution time (sec)
-name             ruby1    ruby2
-example_single   0.958    0.972
-
-Speedup ratio: compare with the result of `ruby1' (greater is better)
-name             ruby2
-example_single   0.986
-```
-
-And you can change benchmark output to IPS (iteration per second) by `-i` option.
+$ exe/benchmark-driver examples/yaml/example_single.yml --rbenv '2.4.2;trunk' --compare
+Warming up --------------------------------------
+          erb.result    10.973k i/100ms
+Calculating -------------------------------------
+                          2.4.2       trunk
+          erb.result   109.268k    123.611k i/s -    548.675k in 4.017080s 4.438720s
 
-```
-$ benchmark-driver benchmarks/example_single.yml -e ruby1::ruby -e ruby2::ruby -i
-Result -------------------------------------------
-                         ruby1          ruby2
-  example_single   99414.1 i/s    99723.3 i/s
-
-Comparison: example_single
-           ruby2:      99723.3 i/s
-           ruby1:      99414.1 i/s - 1.00x slower
+Comparison:
+  erb.result (trunk):    123611.1 i/s
+  erb.result (2.4.2):    109268.4 i/s - 1.13x  slower
 ```
 
-### Running multiple scripts
+#### Running multiple scripts
 
 One YAML file can contain multiple benchmark scripts.
 With following `example_multi.yml`,
@@ -64,103 +133,42 @@ With following `example_multi.yml`,
 prelude: |
   a = 'a' * 100
   b = 'b' * 100
-benchmarks:
-  - name: join
-    benchmark: |
-      [a, b].join
-  - name: interpolation
-    benchmark: |
-      "#{a}#{b}"
+benchmark:
+  join: '[a, b].join'
+  str-interp: '"#{a}#{b}"'
 ```
 
 you can benchmark the scripts with multiple ruby executables.
 
 ```
-$ benchmark-driver benchmarks/example_multi.yml -e ruby1::ruby -e ruby2::ruby
-benchmark results:
-Execution time (sec)
-name             ruby1    ruby2
-join             0.022    0.022
-interpolation    0.026    0.026
-
-Speedup ratio: compare with the result of `ruby1' (greater is better)
-name             ruby2
-join             1.045
-interpolation    1.002
-```
+$ exe/benchmark-driver examples/yaml/example_multi.yml --rbenv '2.4.2;trunk' --compare
+Warming up --------------------------------------
+                join   515.787k i/100ms
+          str-interp   438.646k i/100ms
+Calculating -------------------------------------
+                          2.4.2       trunk
+                join     5.200M      4.740M i/s -     20.631M in 3.967750s 4.352565s
+          str-interp     4.306M      6.034M i/s -     21.932M in 4.075159s 3.634986s
 
-```
-$ benchmark-driver benchmarks/example_multi.yml -e ruby1::ruby -e ruby2::ruby -i
-Result -------------------------------------------
-                         ruby1          ruby2
-            join 4701954.3 i/s  4639520.3 i/s
-   interpolation 4263170.0 i/s  4044083.0 i/s
-
-Comparison: join
-           ruby1:    4701954.3 i/s
-           ruby2:    4639520.3 i/s - 1.01x slower
-
-Comparison: interpolation
-           ruby1:    4263170.0 i/s
-           ruby2:    4044083.0 i/s - 1.05x slower
-```
-
-### Configuring modes
-
-There are 2 modes:
-
-- Loop count: Enabled by `-l`. Optionally you can change count to loop by `-l COUNT`.
-- IPS: Enabled by `-i`. Optionally you can change duration by `-i DURATION`.
-
-Specifying both `-l` and `-i` is nonsense.
-
-### YAML syntax
-You can specify `benchmark:` or `benchmarks:`.
-
-#### Single
-```yml
-name: String # optional (default: file name)
-prelude: String # optional
-loop_count: Integer # optional
-benchmark: String # required
-```
-
-#### Multi
-
-```yml
-prelude: String # optional (shared)
-loop_count: Integer # optional (shared)
-benchmarks:
-  - name: String # required
-    prelude: String # optional (benchmark specific)
-    loop_count: Integer # optional (benchmark specific)
-    benchmark: String # required
-```
-
-### Debugging
-
-If you have a trouble like an unexpectedly fast result, you should check benchmark script by `-v`.
-
-```
-$ benchmark-driver benchmarks/example_multi.yml -v
---- Running "join" with "ruby" 957780 times ---
-a = 'a' * 100
-b = 'b' * 100
-
-
-i = 0
-while i < 957780
-  i += 1
-[a, b].join
-
-end
+Comparison:
+  str-interp (trunk):   6033674.6 i/s
+        join (2.4.2):   5199794.6 i/s - 1.16x  slower
+        join (trunk):   4740075.1 i/s - 1.27x  slower
+  str-interp (2.4.2):   4305563.1 i/s - 1.40x  slower
 ```
 
 ## TODO
-
-- Measure multiple times and use minimum result
-- Retry and reject negative result in ips mode
-- Change not to take long time for iteration count estimation in ips mode
+### Runner
+- [x] Call
+- [x] Exec
+- [ ] Eval
+
+### Output
+- [x] IPS
+- [x] Time
+- [ ] CPU/System/Real Time
+- [ ] Memory
+- [ ] Markdown Table
 
 ## Contributing
 
@@ -168,4 +176,4 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/k0kubu
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -1,15 +1,22 @@
 require 'bundler/gem_tasks'
+require 'shellwords'
 
-desc 'Run benchmarks in benchmarks'
-task :benchmarks do
-  require 'bundler'
-  require 'shellwords'
+task :ruby_examples do
+  Dir.glob(File.expand_path('./examples/*.rb', __dir__)).sort.each do |file|
+    Bundler.with_clean_env do
+      sh ['time', 'bundle', 'exec', 'ruby', file].shelljoin
+    end
+    puts
+  end
+end
 
-  Dir.glob(File.expand_path('./benchmarks/**/*.yml', __dir__)).sort.each do |path|
+task :yaml_examples do
+  Dir.glob(File.expand_path('./examples/yaml/*.yml', __dir__)).sort.each do |file|
     Bundler.with_clean_env do
-      sh [File.expand_path('./exe/benchmark-driver', __dir__), path].shelljoin
+      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', file].shelljoin
     end
+    puts
   end
 end
 
-task default: :benchmarks
+task default: [:ruby_examples, :yaml_examples]

data/benchmark_driver.gemspec

@@ -1,4 +1,3 @@
-# coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'benchmark/driver/version'
@@ -9,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Takashi Kokubun']
   spec.email         = ['takashikkbn@gmail.com']
 
-  spec.summary       = %q{Benchmark driver for different Ruby executables}
-  spec.description   = %q{Benchmark driver for different Ruby executables}
+  spec.summary       = 'Fully-featured accurate benchmark driver for Ruby'
+  spec.description   = 'Fully-featured accurate benchmark driver for Ruby'
   spec.homepage      = 'https://github.com/k0kubun/benchmark_driver'
   spec.license       = 'MIT'
 
@@ -20,7 +19,6 @@ Gem::Specification.new do |spec|
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
-  spec.required_ruby_version = '>= 2.3.0'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require "bundler/setup"
-require "benchmark_driver"
+require "benchmark/driver"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.