benchmark_driver 0.13.3 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a40d650e58bfb47dae94c1dfe4383dd0682a6749ae3c463cf10663ef8a69035
-  data.tar.gz: 57cfa42e50210ea8384b2ba2831553f6aaf3cdfe3b87d1466d8497344343bf6c
+  metadata.gz: 7493fee045d80ab0918a271649807b7a984e30a6c2e90aa39d4a35d8c19e4198
+  data.tar.gz: 04d607c363d64d0f2c2c3e2fd6c6f107d1ad924eb757e048f112131249660a6b
 SHA512:
-  metadata.gz: 4e4c9dcd507974ee703252c8049b7e538846ebf2ae6523303a20ed5e42d644a990de6890a6acb50dc9257e0292e95912ee895337697e28b1ba8cc72eff38f875
-  data.tar.gz: bd12fb18af42c59187ba1b8e4b0568f2ece0b46ec8857f1ca003907007756e4817effc55a7104b217bd3b8492db1a4ee97972e11fddb6472ed4d9d6ddb8c1756
+  metadata.gz: 496a23c97b27783b2fef8f1da1a31a10c4e1efa4b2c487acd400ee2687f2c58e5d458fa800355a6121cc4728a0af6077c87a4067b524f9e71506ef21cde74ebd
+  data.tar.gz: c65d611995c53f273a803234e9bd9c9d31cd6edbfc1a79a92066a5f6a438583cf601eb54319d3e81b18ab4c9f06a2e13b3e642155307e8555a1b2fd9f465ffca

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# v0.14.0
+
+- `benchmark-driver` command also takes `*.rb` file to run single-execution benchmark
+  - Non-`*.rb` filename is considered as YAML file as before
+- [breaking change] `--verbose` no longer takes LEVEL argument
+  - Added `-v` for `-v`/`-vv` instead of `--verbose 1` and `--verbose 2`.
+  - `--verbose --verbose` can also be used instead of `--verbose 2` if you want.
+- [breaking change] Use ` `(space) to split Ruby arguments in `-e`/`--executables`/`--rbenv`
+  - ex) `-e "/path/to/ruby --jit"` should be used instead of `-e "/path/to/ruby,--jit"`
+- [experimental] Return 0.0 as value when benchmark script fails on ips/time/memory/once runners
+  - Exit status is 0 for that case. `benchmark-driver` exits abnormaly only on its internal bug.
+
 # v0.13.3
 
 - Support `require: false` in contexts to skip automatic requirement with a gem name

data/README.md

@@ -70,16 +70,17 @@ With `benchmark-driver` command, you can describe benchmark with YAML input.
 
 ```
 $ benchmark-driver -h
-Usage: benchmark-driver [options] [YAML]
+Usage: benchmark-driver [options] [YAML|RUBY]
     -r, --runner [TYPE]              Specify runner type: ips, time, memory, once (default: ips)
     -o, --output [TYPE]              Specify output type: compare, simple, markdown, record (default: compare)
     -e, --executables [EXECS]        Ruby executables (e1::path1,arg1,...; e2::path2,arg2;...)
         --rbenv [VERSIONS]           Ruby executables in rbenv (x.x.x,arg1,...;y.y.y,arg2,...;...)
-        --repeat-count [NUM]         Try benchmark NUM times and use the fastest result (TODO)
+        --repeat-count [NUM]         Try benchmark NUM times and use the fastest result or the worst memory usage
+        --repeat-result [TYPE]       Yield "best", "average" or "worst" result with --repeat-count (default: best)
         --bundler                    Install and use gems specified in Gemfile
         --filter [REGEXP]            Filter out benchmarks with given regexp
-        --verbose [LEVEL]            Show some verbose outputs: 0, 1, 2 (default: 0)
         --run-duration [SECONDS]     Warmup estimates loop_count to run for this duration (default: 3)
+    -v, --verbose                    Verbose mode. Multiple -v options increase visilibity (max: 2)
 ```
 
 #### Running single script
@@ -363,16 +364,16 @@ If you benchmark can run with `ruby foo bar`, specify `foo bar` to `command:`.
 Then write `stdout_to_metrics:` to convert stdout to metrics. This runner can be used only with YAML interface for now.
 
 ```
-$ benchmark-driver benchmark.yml --verbose 1 --rbenv '2.6.0-dev;2.6.0-dev,--jit'
+$ benchmark-driver benchmark.yml --verbose 1 --rbenv '2.6.0-dev;2.6.0-dev --jit'
 2.6.0-dev: ruby 2.6.0dev (2018-03-21 trunk 62870) [x86_64-linux]
-2.6.0-dev,--jit: ruby 2.6.0dev (2018-03-21 trunk 62870) +JIT [x86_64-linux]
+2.6.0-dev --jit: ruby 2.6.0dev (2018-03-21 trunk 62870) +JIT [x86_64-linux]
 Calculating -------------------------------------
-                      2.6.0-dev  2.6.0-dev,--jit
+                      2.6.0-dev  2.6.0-dev --jit
            optcarrot     51.866           67.445 fps
 
 Comparison:
                         optcarrot
-     2.6.0-dev,--jit:        67.4 fps
+     2.6.0-dev --jit:        67.4 fps
            2.6.0-dev:        51.9 fps - 1.30x  slower
 ```
 

data/exe/benchmark-driver

@@ -3,6 +3,7 @@ $:.unshift File.expand_path('../lib', __dir__)
 
 require 'benchmark_driver'
 require 'optparse'
+require 'shellwords'
 require 'yaml'
 
 # Parse command line options
@@ -10,7 +11,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
   executables = []
   bundler = false
   parser = OptionParser.new do |o|
-    o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML]"
+    o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML|RUBY]"
     o.on('-r', '--runner [TYPE]', 'Specify runner type: ips, time, memory, once (default: ips)') do |d|
       abort '-r, --runner must take argument but not given' if d.nil?
       c.runner_type = d
@@ -24,7 +25,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
       e.split(';').each do |name_path|
         name, path = name_path.split('::', 2)
         path ||= name # if `::` is not given, regard whole string as path
-        command = path.split(',')
+        command = path.shellsplit
         command[0] = File.expand_path(command[0])
         executables << BenchmarkDriver::Config::Executable.new(name: name, command: command)
       end
@@ -54,13 +55,6 @@ config = BenchmarkDriver::Config.new.tap do |c|
     o.on('--filter [REGEXP]', 'Filter out benchmarks with given regexp') do |v|
       c.filters << Regexp.compile(v)
     end
-    o.on('--verbose [LEVEL]', 'Show some verbose outputs: 0, 1, 2 (default: 0)') do |v|
-      begin
-        c.verbose = Integer(v)
-      rescue ArgumentError
-        abort "--verbose must take Integer, but got #{v.inspect}"
-      end
-    end
     o.on('--run-duration [SECONDS]', 'Warmup estimates loop_count to run for this duration (default: 3)') do |v|
       begin
         c.run_duration = Float(v)
@@ -68,6 +62,9 @@ config = BenchmarkDriver::Config.new.tap do |c|
         abort "--run-duration must take Float, but got #{v.inspect}"
       end
     end
+    o.on('-v', '--verbose', 'Verbose mode. Multiple -v options increase visilibity (max: 2)') do |v|
+      c.verbose += 1
+    end
   end
   c.paths = parser.parse!(ARGV)
   if c.paths.empty?
@@ -89,8 +86,17 @@ end
 
 # Parse benchmark job definitions
 jobs = config.paths.flat_map do |path|
-  job = YAML.load(File.read(path))
-  job = { 'type' => config.runner_type }.merge!(job)
+  job = { 'type' => config.runner_type }
+
+  # Treat *.rb as a single-execution benchmark, others are considered as YAML definition
+  if path.end_with?('.rb')
+    name = File.basename(path).sub(/\.rb\z/, '')
+    script = File.read(path)
+    prelude = script.slice!(/\A(^#[^\n]+\n)+/m) || '' # preserve magic comment
+    job.merge!('prelude' => prelude, 'benchmark' => { name => script }, 'loop_count' => 1)
+  else
+    job.merge!(YAML.load_file(path))
+  end
 
   begin
     # `working_directory` is YAML-specific special parameter, mainly for "command_stdout"

data/lib/benchmark_driver/metric.rb

@@ -35,6 +35,7 @@ module BenchmarkDriver
     :environment, # @param [Hash] - Any other key -> value pairs to express the benchmark context
     defaults: { environment: {} },
   )
+  Result::ERROR = 0.0
 
   # A kind of thing to be measured
   Metric = ::BenchmarkDriver::Struct.new(

data/lib/benchmark_driver/output/compare.rb

@@ -116,7 +116,9 @@ class BenchmarkDriver::Output::Compare
   end
 
   def humanize(value, width = 10)
-    if value <= 0
+    if value == BenchmarkDriver::Result::ERROR
+      return " %#{width}s" % 'ERROR'
+    elsif value < 0
       raise ArgumentError.new("Non positive value: #{value.inspect}")
     end
 

data/lib/benchmark_driver/output/simple.rb

@@ -77,7 +77,9 @@ class BenchmarkDriver::Output::Simple
   end
 
   def humanize(value)
-    if value < 0
+    if value == BenchmarkDriver::Result::ERROR
+      return 'ERROR'
+    elsif value < 0
       raise ArgumentError.new("Negative value: #{value.inspect}")
     end
 

data/lib/benchmark_driver/rbenv.rb

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 module BenchmarkDriver
   module Rbenv
     # @param [String] version
@@ -13,7 +15,7 @@ module BenchmarkDriver
     def self.parse_spec(full_spec)
       name, spec = full_spec.split('::', 2)
       spec ||= name # if `::` is not given, regard whole string as spec
-      version, *args = spec.split(',')
+      version, *args = spec.shellsplit
       BenchmarkDriver::Config::Executable.new(
         name: name,
         command: [BenchmarkDriver::Rbenv.ruby_path(version), *args],

data/lib/benchmark_driver/runner/ips.rb

@@ -101,9 +101,13 @@ class BenchmarkDriver::Runner::Ips
 
     duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        execute(*context.executable.command, path)
+        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
+        if $?.success?
+          Float(f.read)
+        else
+          BenchmarkDriver::Result::ERROR
+        end
       end
-      Float(f.read)
     end
 
     value_duration(
@@ -119,7 +123,11 @@ class BenchmarkDriver::Runner::Ips
 
   # Overridden by BenchmarkDriver::Runner::Time
   def value_duration(duration:, loop_count:)
-    [loop_count.to_f / duration, duration]
+    if duration == BenchmarkDriver::Result::ERROR
+      [BenchmarkDriver::Result::ERROR, BenchmarkDriver::Result::ERROR]
+    else
+      [loop_count.to_f / duration, duration]
+    end
   end
 
   def with_script(script)
@@ -136,7 +144,7 @@ class BenchmarkDriver::Runner::Ips
   end
 
   def execute(*args)
-    stdout = IO.popen(args, &:read) # handle stdout?
+    stdout = IO.popen(args, &:read) # TODO: print stdout if verbose=2
     unless $?.success?
       raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
     end

data/lib/benchmark_driver/runner/memory.rb

@@ -68,14 +68,18 @@ class BenchmarkDriver::Runner::Memory
       loop_count: job.loop_count,
     )
 
-    output = with_script(benchmark.render) do |path|
-      execute('/usr/bin/time', *context.executable.command, path)
-    end
-
-    match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(output)
-    raise "Unexpected format given from /usr/bin/time:\n#{out}" unless match_data[:maxresident]
+    with_script(benchmark.render) do |path|
+      output = IO.popen(['/usr/bin/time', *context.executable.command, path], err: [:child, :out], &:read)
+      if $?.success?
+        match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(output)
+        raise "Unexpected format given from /usr/bin/time:\n#{out}" unless match_data[:maxresident]
 
-    Integer(match_data[:maxresident]) * 1000.0 # kilobytes -> bytes
+        Integer(match_data[:maxresident]) * 1000.0 # kilobytes -> bytes
+      else
+        $stdout.print(output)
+        BenchmarkDriver::Result::ERROR
+      end
+    end
   end
 
   def with_script(script)
@@ -91,14 +95,6 @@ class BenchmarkDriver::Runner::Memory
     end
   end
 
-  def execute(*args)
-    output = IO.popen(args, err: [:child, :out], &:read) # handle stdout?
-    unless $?.success?
-      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
-    end
-    output
-  end
-
   # @param [String] prelude
   # @param [String] script
   # @param [String] teardown

data/lib/benchmark_driver/runner/once.rb

@@ -35,8 +35,14 @@ class BenchmarkDriver::Runner::Once
         @output.with_job(name: job.name) do
           job.runnable_contexts(@contexts).each do |context|
             duration = run_benchmark(job, context: context) # no repeat support
+            if duration == BenchmarkDriver::Result::ERROR
+              value = BenchmarkDriver::Result::ERROR
+            else
+              value = 1.0 / duration
+            end
+
             @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
-              @output.report(values: { METRIC => 1.0 / duration }, duration: duration, loop_count: 1)
+              @output.report(values: { METRIC => value }, duration: duration, loop_count: 1)
             end
           end
         end
@@ -59,9 +65,13 @@ class BenchmarkDriver::Runner::Once
 
     Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        execute(*context.executable.command, path)
+        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
+        if $?.success?
+          Float(f.read)
+        else
+          BenchmarkDriver::Result::ERROR
+        end
       end
-      Float(f.read)
     end
   end
 

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.13.3'
+  VERSION = '0.14.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.13.3
+  version: 0.14.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-08 00:00:00.000000000 Z
+date: 2018-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler