benchmark_driver 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b444be9753246263a4b3a65f0fe33a07c62f4a538008245fb8a17b2e809f7b6
-  data.tar.gz: e4a43fb02781c343e266e8c5644d8fc4edf1568de93d620e90500f027bb4418a
+  metadata.gz: d2bcdf477d225e851726d6e8296af8b2ace8542c9b3cf3bc0d0d076dc140c8cb
+  data.tar.gz: 0d13f33d1fe60a936601377686a3121eee98972d587ad2a13e1828fa5a6a6588
 SHA512:
-  metadata.gz: 4c512a0e7bb65debbb79b2f312db9852db1625cfd3a46cab847b81f0d3f85d09caef043e09b1f47679f6b476ea98d2a913ea7fbd1a694aac4621a2e701256450
-  data.tar.gz: 5fd294728cf04e4524a62aa12916ba4c66b6e9f8dab498cb290a20d324a3aa60b5f53786d6bb150ec1a44d730770a40d77ab8e3f29fcd5c109ed85f5e02c074c
+  metadata.gz: bca45d1c0fd8608aa223aa02ce0e1873f6e9bd2a04edb9efebab5dddff0f68be77be3a318613fda3af152e197e3298d97959e7720daa3b2b4ea5e6fffc8ac7b3
+  data.tar.gz: '0908488a1465b40d808dfa9f3ead01d1b0647106ed8f88328e3aa1dd3a43d13ad90946f40b81094ba5327b6ff480d7f19060c814749bbe63731cbb0871170437'

data/.travis.yml

@@ -1,6 +1,5 @@
 language: ruby
 rvm:
-  - 2.1.10
   - 2.2.8
   - 2.3.6
   - 2.4.3

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# v0.10.1
+
+- Add "command\_stdout" runner to plug in existing benchmark
+- Explicitly bump supported Ruby version to >= 2.2
+  - v0.10.0 actually does not work with 2.1
+  - You can still benchmark Ruby 2.0, 2.1 by --executable, but you need to use newer Ruby for driver
+
 # v0.10.0
 
 - Add "record" output and "recorded" runner
@@ -24,6 +31,7 @@
      - So Ruby interface can't take Proc
   - YAML can have arbitrary format depending on the runner
 - `--compare` option is dropped and changed to `--output compare`
+- `--dir` option is dropped for now
 
 # v0.8.6
 

data/README.md

@@ -4,7 +4,9 @@ Fully-featured accurate benchmark driver for Ruby
 
 ## Project Status
 
-**Under Construction**
+Beta.
+
+Interface might be changed in the future, but it's almost fixed and many features can run well.
 
 ## Features
 ### Accurate Measurement

data/Rakefile

@@ -12,7 +12,9 @@ task :test do
   }.each do |runner, output|
     Bundler.with_clean_env do
       sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', runner, '-o', output].shelljoin
+      puts
       sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_hash, '-r', runner, '-o', output, '--run-duration', '1'].shelljoin
+      puts
     end
   end
 end
@@ -20,9 +22,13 @@ end
 task :test_record do
   blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
   sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', 'ips', '-o', 'record'].shelljoin
+  puts
   sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'compare'].shelljoin
+  puts
   sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'record'].shelljoin
+  puts
   sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'simple'].shelljoin
+  puts
 end
 
 task :test_ruby do

data/benchmark_driver.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
-  spec.required_ruby_version = '>= 2.1.0'
+  spec.required_ruby_version = '>= 2.2.0'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/exe/benchmark-driver

@@ -5,7 +5,6 @@ require 'benchmark_driver'
 require 'optparse'
 require 'yaml'
 
-
 # Parse command line options
 config = BenchmarkDriver::Config.new.tap do |c|
   executables = []
@@ -34,7 +33,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
       r.split(';').each do |spec|
         version, *args = spec.split(',')
         executables << BenchmarkDriver::Config::Executable.new(
-          name: version,
+          name: spec,
           command: [BenchmarkDriver::Rbenv.ruby_path(version), *args],
         )
       end

data/lib/benchmark_driver/output/compare.rb

@@ -55,7 +55,7 @@ class BenchmarkDriver::Output::Compare
     block.call
   ensure
     $stdout.print(@metrics_type.unit)
-    if job.loop_count
+    if job.respond_to?(:loop_count) && job.loop_count
       $stdout.print(" - #{humanize(job.loop_count)} times")
       if @job_metrics.all? { |metrics| metrics.duration }
         $stdout.print(" in")

data/lib/benchmark_driver/runner.rb

@@ -1,5 +1,6 @@
 module BenchmarkDriver
   module Runner
+    require 'benchmark_driver/runner/command_stdout'
     require 'benchmark_driver/runner/ips'
     require 'benchmark_driver/runner/memory'
     require 'benchmark_driver/runner/once'

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -0,0 +1,121 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metrics'
+require 'tempfile'
+require 'shellwords'
+
+# Run only once, for testing
+class BenchmarkDriver::Runner::CommandStdout
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,              # @param [String] name - This is mandatory for all runner
+    :command,           # @param [Array<String>]
+    :working_directory, # @param [String,NilClass]
+    :metrics_type,      # @param [BenchmarkDriver::Metrics::Type]
+    :stdout_to_metrics, # @param [String]
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [String] name
+    # @param [String] command
+    # @param [String,NilClass] working_directory
+    # @param [Hash] metrics_type
+    # @param [String] stdout_to_metrics
+    def parse(name:, command:, working_directory:, metrics_type:, stdout_to_metrics:)
+      Job.new(
+        name: name,
+        command: command.shellsplit,
+        working_directory: working_directory,
+        metrics_type: parse_metrics_type(metrics_type),
+        stdout_to_metrics: stdout_to_metrics,
+      )
+    end
+
+    private
+
+    def parse_metrics_type(unit:, larger_better: nil, worse_word: nil)
+      BenchmarkDriver::Metrics::Type.new(
+        unit: unit,
+        larger_better: larger_better,
+        worse_word: worse_word,
+      )
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output::*] output
+  def initialize(config:, output:)
+    @config = config
+    @output = output
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    metrics_type = jobs.first.metrics_type
+    @output.metrics_type = metrics_type
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(job) do
+          @config.executables.each do |exec|
+            best_value = with_repeat(metrics_type) do
+              stdout = with_chdir(job.working_directory) do
+                execute(*exec.command, *job.command)
+              end
+              StdoutToMetrics.new(
+                stdout: stdout,
+                stdout_to_metrics: job.stdout_to_metrics,
+              ).metrics_value
+            end
+
+            @output.report(
+              BenchmarkDriver::Metrics.new(
+                value: best_value,
+                executable: exec,
+              )
+            )
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def with_chdir(working_directory, &block)
+    if working_directory
+      Dir.chdir(working_directory) { block.call }
+    else
+      block.call
+    end
+  end
+
+  def execute(*args)
+    stdout = IO.popen(args, &:read)
+    unless $?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+    end
+    stdout
+  end
+
+  # Return multiple times and return the best metrics
+  def with_repeat(metrics_type, &block)
+    values = @config.repeat_count.times.map do
+      block.call
+    end
+    values.sort_by do |value|
+      if metrics_type.larger_better
+        value
+      else
+        -value
+      end
+    end.last
+  end
+
+  StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :stdout_to_metrics) do
+    def metrics_value
+      eval(stdout_to_metrics, binding)
+    end
+  end
+  private_constant :StdoutToMetrics
+end

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.10.0'
+  VERSION = '0.10.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Takashi Kokubun
@@ -82,6 +82,7 @@ files:
 - lib/benchmark_driver/rbenv.rb
 - lib/benchmark_driver/ruby_interface.rb
 - lib/benchmark_driver/runner.rb
+- lib/benchmark_driver/runner/command_stdout.rb
 - lib/benchmark_driver/runner/ips.rb
 - lib/benchmark_driver/runner/memory.rb
 - lib/benchmark_driver/runner/once.rb
@@ -101,7 +102,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="