benchmark_driver 0.10.16 → 0.11.0

data/lib/benchmark_driver/runner/recorded.rb

@@ -1,5 +1,5 @@
 require 'benchmark_driver/struct'
-require 'benchmark_driver/metrics'
+require 'benchmark_driver/metric'
 require 'tempfile'
 require 'shellwords'
 
@@ -8,51 +8,58 @@ class BenchmarkDriver::Runner::Recorded
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = ::BenchmarkDriver::Struct.new(
     :name,              # @param [String] name - This is mandatory for all runner
-    :job,               # @param [BenchmarkDriver::Runner::*::Job]
-    :warmup_metrics,    # @param [Hash]
-    :benchmark_metrics, # @param [Hash]
-    :metrics_type,      # @param [BenchmarkDriver::Metrics::Type]
+    :warmup_results,    # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
+    :benchmark_results, # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
+    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
   )
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   class << JobParser = Module.new
-    # @param [Hash] metrics_by_job
-    # @param [BenchmarkDriver::Metrics::Type] metrics_type
-    def parse(metrics_by_job:, metrics_type:)
-      metrics_by_job.map do |job, metrics_hash|
+    # @param [Hash{ String => Hash{ TrueClass,FalseClass => Hash{ BenchmarkDriver::Context => Hash{ BenchmarkDriver::Metric => Float } } } }] job_warmup_context_metric_value
+    # @param [BenchmarkDriver::Metrics::Type] metrics
+    def parse(job_warmup_context_metric_value:, metrics:)
+      job_warmup_context_metric_value.map do |job_name, warmup_context_values|
         Job.new(
-          name: job.name,
-          job: job,
-          warmup_metrics: metrics_hash.fetch(:warmup, []),
-          benchmark_metrics: metrics_hash.fetch(:benchmark),
-          metrics_type: metrics_type,
+          name: job_name,
+          warmup_results: warmup_context_values.fetch(true, {}),
+          benchmark_results: warmup_context_values.fetch(false, {}),
+          metrics: metrics,
         )
       end
     end
   end
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
-  # @param [BenchmarkDriver::Output::*] output
+  # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
     @config = config
     @output = output
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
-  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  # @param [Array<BenchmarkDriver::Runner::Recorded::Job>] record
   def run(records)
-    @output.metrics_type = records.first.metrics_type
+    @output.metrics = records.first.metrics
 
     records.each do |record|
-      unless record.warmup_metrics.empty?
+      unless record.warmup_results.empty?
         # TODO:
       end
     end
 
     @output.with_benchmark do
       records.each do |record|
-        @output.with_job(record.job) do
-          record.benchmark_metrics.each do |metrics|
-            @output.report(metrics)
+        @output.with_job(name: record.name) do
+          record.benchmark_results.each do |context, metric_values|
+            @output.with_context(
+              name: context.name,
+              executable: context.executable,
+              duration: context.duration,
+              loop_count: context.loop_count,
+            ) do
+              metric_values.each do |metric, value|
+                @output.report(value: value, metric: metric)
+              end
+            end
           end
         end
       end

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -0,0 +1,157 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'tempfile'
+require 'shellwords'
+require 'open3'
+
+# Use stdout of ruby command
+class BenchmarkDriver::Runner::RubyStdout
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,                   # @param [String] name - This is mandatory for all runner
+    :command,                # @param [Array<String>]
+    :working_directory,      # @param [String,NilClass]
+    :metrics,                # @param [Array<BenchmarkDriver::Metric>]
+    :value_from_stdout,      # @param [String]
+    :environment_from_stdout # @param [Hash{ String => String }]
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [String] name
+    # @param [String] command
+    # @param [String,NilClass] working_directory
+    # @param [Hash] metrics_type
+    # @param [String] stdout_to_metrics
+    def parse(name:, command:, working_directory: nil, metrics:, environment: {})
+      unless metrics.is_a?(Hash)
+        raise ArgumentError.new("metrics must be Hash, but got #{metrics.class}")
+      end
+      if metrics.size == 0
+        raise ArgumentError.new('At least one metric must be specified"')
+      elsif metrics.size != 1
+        raise NotImplementedError.new('Having multiple metrics is not supported yet')
+      end
+
+      metric, value_from_stdout = parse_metric(*metrics.first)
+      environment_from_stdout = Hash[environment.map { |k, v| [k, parse_environment(v)] }]
+
+      Job.new(
+        name: name,
+        command: command.shellsplit,
+        working_directory: working_directory,
+        metrics: [metric],
+        value_from_stdout: value_from_stdout,
+        environment_from_stdout: environment_from_stdout,
+      )
+    end
+
+    private
+
+    def parse_metric(name, unit:, from_stdout:, larger_better: true, worse_word: 'slower')
+      metric = BenchmarkDriver::Metric.new(
+        name: name,
+        unit: unit,
+        larger_better: larger_better,
+        worse_word: worse_word,
+      )
+      [metric, from_stdout]
+    end
+
+    def parse_environment(from_stdout:)
+      from_stdout
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  def initialize(config:, output:)
+    @config = config
+    @output = output
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    metric = jobs.first.metrics.first
+    @output.metrics = [metric]
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          @config.executables.each do |exec|
+            best_value, environment = with_repeat(metric) do
+              stdout = with_chdir(job.working_directory) do
+                with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
+              end
+              script = StdoutToMetrics.new(
+                stdout: stdout,
+                value_from_stdout: job.value_from_stdout,
+                environment_from_stdout: job.environment_from_stdout,
+              )
+              [script.value, script.environment]
+            end
+
+            @output.with_context(name: exec.name, executable: exec, environment: environment) do
+              @output.report(value: best_value, metric: metric)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def with_ruby_prefix(executable, &block)
+    env = ENV.to_h.dup
+    ENV['PATH'] = "#{File.dirname(executable.command.first)}:#{ENV['PATH']}"
+    block.call
+  ensure
+    ENV.replace(env)
+  end
+
+  def with_chdir(working_directory, &block)
+    if working_directory
+      Dir.chdir(working_directory) { block.call }
+    else
+      block.call
+    end
+  end
+
+  def execute(*args)
+    stdout, stderr, status = Open3.capture3(*args)
+    unless status.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus}):\n[stdout]:\n#{stdout}\n[stderr]:\n#{stderr}"
+    end
+    stdout
+  end
+
+  # Return multiple times and return the best metrics
+  def with_repeat(metric, &block)
+    value_environments = @config.repeat_count.times.map do
+      block.call
+    end
+    value_environments.sort_by do |value, _|
+      if metric.larger_better
+        value
+      else
+        -value
+      end
+    end.last
+  end
+
+  StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :value_from_stdout, :environment_from_stdout) do
+    def value
+      value = eval(value_from_stdout, binding)
+    end
+
+    def environment
+      ret = {}
+      environment_from_stdout.each do |name, from_stdout|
+        ret[name] = eval(from_stdout, binding)
+      end
+      ret
+    end
+  end
+  private_constant :StdoutToMetrics
+end

data/lib/benchmark_driver/runner/time.rb

@@ -6,18 +6,15 @@ class BenchmarkDriver::Runner::Time < BenchmarkDriver::Runner::Ips
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
 
-  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 's', larger_better: false)
+  METRIC = BenchmarkDriver::Metric.new(name: 'Execution time', unit: 's', larger_better: false)
 
-  # Overriding BenchmarkDriver::Runner::Ips#set_metrics_type
-  def set_metrics_type
-    @output.metrics_type = METRICS_TYPE
+  # Overriding BenchmarkDriver::Runner::Ips#metric
+  def metric
+    METRIC
   end
 
-  # Overriding BenchmarkDriver::Runner::Ips#build_metrics
-  def build_metrics(duration:, executable:, loop_count:)
-    BenchmarkDriver::Metrics.new(
-      value: duration,
-      executable: executable,
-    )
+  # Overriding BenchmarkDriver::Runner::Ips#value_duration
+  def value_duration(duration:, loop_count:)
+    [duration, duration]
   end
 end

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.10.16'
+  VERSION = '0.11.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.10.16
+  version: 0.11.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2018-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fully-featured accurate benchmark driver for Ruby
 email:
 - takashikkbn@gmail.com
@@ -47,26 +75,27 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmark-driver/.gitignore
+- benchmark-driver/CODE_OF_CONDUCT.md
+- benchmark-driver/Gemfile
+- benchmark-driver/LICENSE.txt
+- benchmark-driver/README.md
+- benchmark-driver/Rakefile
+- benchmark-driver/benchmark-driver.gemspec
+- benchmark-driver/bin/console
+- benchmark-driver/bin/setup
+- benchmark-driver/lib/benchmark-driver.rb
+- benchmark-driver/lib/benchmark/driver.rb
 - benchmark_driver.gemspec
 - bin/console
 - bin/setup
-- examples/exec_blank.rb
-- examples/exec_blank_simple.rb
-- examples/yaml/array_duration_time.yml
-- examples/yaml/array_loop.yml
-- examples/yaml/blank_hash.yml
-- examples/yaml/blank_hash_array.yml
-- examples/yaml/blank_loop.yml
-- examples/yaml/blank_string.yml
-- examples/yaml/blank_string_array.yml
-- examples/yaml/example_multi.yml
-- examples/yaml/example_single.yml
 - exe/benchmark-driver
 - images/optcarrot.png
 - lib/benchmark_driver.rb
@@ -74,7 +103,7 @@ files:
 - lib/benchmark_driver/default_job.rb
 - lib/benchmark_driver/default_job_parser.rb
 - lib/benchmark_driver/job_parser.rb
-- lib/benchmark_driver/metrics.rb
+- lib/benchmark_driver/metric.rb
 - lib/benchmark_driver/output.rb
 - lib/benchmark_driver/output/compare.rb
 - lib/benchmark_driver/output/markdown.rb
@@ -88,10 +117,11 @@ files:
 - lib/benchmark_driver/runner/memory.rb
 - lib/benchmark_driver/runner/once.rb
 - lib/benchmark_driver/runner/recorded.rb
+- lib/benchmark_driver/runner/ruby_stdout.rb
 - lib/benchmark_driver/runner/time.rb
 - lib/benchmark_driver/struct.rb
 - lib/benchmark_driver/version.rb
-homepage: https://github.com/k0kubun/benchmark_driver
+homepage: https://github.com/benchmark-driver/benchmark-driver
 licenses:
 - MIT
 metadata: {}
@@ -111,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby

data/examples/exec_blank.rb

@@ -1,13 +0,0 @@
-require 'benchmark_driver'
-
-Benchmark.driver do |x|
-  x.prelude <<-EOS
-    class Array
-      alias_method :blank?, :empty?
-    end
-    array = []
-  EOS
-  x.report 'Array#empty?', %{ array.empty? }
-  x.report 'Array#blank?', %{ array.blank? }
-  x.output 'markdown'
-end

data/examples/exec_blank_simple.rb

@@ -1,13 +0,0 @@
-require 'benchmark_driver'
-
-Benchmark.driver(output: :simple) do |x|
-  x.prelude <<-EOS
-    class Array
-      alias_method :blank?, :empty?
-    end
-    array = []
-  EOS
-  x.report %{ array.empty? }
-  x.report %{ array.blank? }
-  x.loop_count 10000000
-end

data/examples/yaml/array_duration_time.yml

@@ -1,2 +0,0 @@
-benchmark:
-  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/array_loop.yml

@@ -1,3 +0,0 @@
-loop_count: 6000000
-benchmark:
-  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/blank_hash.yml

@@ -1,8 +0,0 @@
-prelude: |
-  class Array
-    alias_method :blank?, :empty?
-  end
-  array = []
-benchmark:
-  empty: array.empty?
-  blank: array.blank?

data/examples/yaml/blank_hash_array.yml

@@ -1,10 +0,0 @@
-prelude: |
-  class Array
-    alias_method :blank?, :empty?
-  end
-  array = []
-benchmark:
-  - name: Array#empty?
-    script: array.empty?
-  - name: Array#blank?
-    script: array.blank?