benchmark_driver 0.3.0 → 0.4.0

data/lib/benchmark/output/memory.rb

@@ -0,0 +1,57 @@
+class Benchmark::Output::Memory
+  # This class requires runner to measure following fields in `Benchmark::Driver::BenchmarkResult` to show output.
+  REQUIRED_FIELDS = [:max_rss]
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @param [Array<Benchmark::Driver::Configuration::Executable>] executables
+  # @param [Benchmark::Driver::Configuration::OutputOptions] options
+  def initialize(jobs:, executables:, options:)
+    @jobs = jobs
+    @executables = executables
+    @options = options
+    @name_length = jobs.map { |j| j.name.size }.max
+  end
+
+  def start_warming
+    $stdout.print 'warming up...'
+  end
+
+  # @param [String] name
+  def warming(name)
+    # noop
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def warmup_stats(result)
+    $stdout.print '.'
+  end
+
+  def start_running
+    $stdout.puts if @jobs.any?(&:warmup_needed?)
+    $stdout.puts 'max resident memory (KB):'
+    $stdout.print("%-#{@name_length}s  " % 'ruby')
+    @executables.each do |executable|
+      $stdout.print('%-6s  ' % executable.name)
+    end
+    $stdout.puts
+  end
+
+  # @param [String] name
+  def running(name)
+    $stdout.print("%-#{@name_length}s  " % name)
+    @ran_num = 0
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def benchmark_stats(result)
+    $stdout.print('%-6d  ' % result.max_rss)
+    @ran_num += 1
+    if @ran_num == @executables.size
+      $stdout.puts
+    end
+  end
+
+  def finish
+    # compare is not implemented yet
+  end
+end

data/lib/benchmark/output/time.rb

@@ -0,0 +1,57 @@
+class Benchmark::Output::Time
+  # This class requires runner to measure following fields in `Benchmark::Driver::BenchmarkResult` to show output.
+  REQUIRED_FIELDS = [:real]
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @param [Array<Benchmark::Driver::Configuration::Executable>] executables
+  # @param [Benchmark::Driver::Configuration::OutputOptions] options
+  def initialize(jobs:, executables:, options:)
+    @jobs = jobs
+    @executables = executables
+    @options = options
+    @name_length = jobs.map { |j| j.name.size }.max
+  end
+
+  def start_warming
+    $stdout.print 'warming up...'
+  end
+
+  # @param [String] name
+  def warming(name)
+    # noop
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def warmup_stats(result)
+    $stdout.print '.'
+  end
+
+  def start_running
+    $stdout.puts if @jobs.any?(&:warmup_needed?)
+    $stdout.puts 'benchmark results (s):'
+    $stdout.print("%-#{@name_length}s  " % 'ruby')
+    @executables.each do |executable|
+      $stdout.print('%-6s  ' % executable.name)
+    end
+    $stdout.puts
+  end
+
+  # @param [String] name
+  def running(name)
+    $stdout.print("%-#{@name_length}s  " % name)
+    @ran_num = 0
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def benchmark_stats(result)
+    $stdout.print('%-6.3f  ' % result.real)
+    @ran_num += 1
+    if @ran_num == @executables.size
+      $stdout.puts
+    end
+  end
+
+  def finish
+    # compare is not implemented yet
+  end
+end

data/lib/benchmark/runner.rb

@@ -0,0 +1,13 @@
+module Benchmark::Runner
+  # Benchmark::Runner is pluggable.
+  # Create `Benchmark::Runner::FooBar` as benchmark-runner-foo_bar.gem and specify `runner: foo_bar`.
+  #
+  # @param [Symbol] name
+  def self.find(name)
+    class_name = Benchmark::Driver::Configuration.camelize(name.to_s)
+    Benchmark::Runner.const_get(class_name, false)
+  end
+end
+
+require 'benchmark/runner/call'
+require 'benchmark/runner/exec'

data/lib/benchmark/runner/call.rb

@@ -0,0 +1,97 @@
+require 'benchmark/driver/benchmark_result'
+require 'benchmark/driver/duration_runner'
+require 'benchmark/driver/repeatable_runner'
+require 'benchmark/driver/time'
+
+# Run benchmark by calling #call on running ruby.
+#
+# Multiple Ruby binaries: x
+# Memory output: x
+class Benchmark::Runner::Call
+  # This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
+  SUPPORTED_FIELDS = [:real]
+
+  WARMUP_DURATION    = 2
+  BENCHMARK_DURATION = 5
+
+  # @param [Benchmark::Driver::Configuration::RunnerOptions] options
+  # @param [Benchmark::Output::*] output - Object that responds to methods used in this class
+  def initialize(options, output:)
+    @options = options
+    @output  = output
+  end
+
+  # @param [Benchmark::Driver::Configuration] config
+  def run(config)
+    validate_config(config)
+
+    if config.jobs.any?(&:warmup_needed?)
+      run_warmup(config.jobs)
+    end
+
+    @output.start_running
+
+    config.jobs.each do |job|
+      @output.running(job.name)
+
+      result = Benchmark::Driver::RepeatableRunner.new(job).run(
+        runner: method(:call_times),
+        repeat_count: @options.repeat_count,
+      )
+
+      @output.benchmark_stats(result)
+    end
+
+    @output.finish
+  end
+
+  private
+
+  def validate_config(config)
+    if config.runner_options.executables_specified?
+      raise ArgumentError.new("Benchmark::Runner::Call can't run other Ruby executables")
+    end
+
+    config.jobs.each do |job|
+      unless job.script.respond_to?(:call)
+        raise NotImplementedError.new(
+          "#{self.class.name} only accepts objects that respond to :call, but got #{job.script.inspect}"
+        )
+      end
+    end
+  end
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
+  def run_warmup(jobs)
+    @output.start_warming
+
+    jobs.each do |job|
+      next if job.loop_count
+      @output.warming(job.name)
+
+      result = Benchmark::Driver::DurationRunner.new(job).run(
+        seconds:    WARMUP_DURATION,
+        unit_iters: 1,
+        runner:     method(:call_times),
+      )
+      job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
+
+      @output.warmup_stats(result)
+    end
+  end
+
+  def call_times(job, times)
+    script = job.script
+    i = 0
+
+    before = Benchmark::Driver::Time.now
+    while i < times
+      script.call
+      i += 1
+    end
+    after = Benchmark::Driver::Time.now
+
+    after.to_f - before.to_f
+  end
+end

data/lib/benchmark/runner/exec.rb

@@ -0,0 +1,190 @@
+require 'tempfile'
+require 'shellwords'
+require 'benchmark/driver/benchmark_result'
+require 'benchmark/driver/duration_runner'
+require 'benchmark/driver/repeatable_runner'
+require 'benchmark/driver/error'
+require 'benchmark/driver/time'
+
+# Run benchmark by executing another Ruby process.
+#
+# Multiple Ruby binaries: o
+# Memory output: o
+class Benchmark::Runner::Exec
+  # This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
+  SUPPORTED_FIELDS = [:real, :max_rss]
+
+  WARMUP_DURATION    = 1
+  BENCHMARK_DURATION = 4
+  GUESS_TIMES = [1, 1_000, 1_000_000, 10_000_000, 100_000_000]
+  GUESS_THRESHOLD = 0.4 # 400ms
+
+  # @param [Benchmark::Driver::Configuration::RunnerOptions] options
+  # @param [Benchmark::Output::*] output - Object that responds to methods used in this class
+  def initialize(options, output:)
+    @options = options
+    @output  = output
+  end
+
+  # @param [Benchmark::Driver::Configuration] config
+  def run(config)
+    validate_config(config)
+
+    if config.jobs.any?(&:warmup_needed?)
+      run_warmup(config.jobs)
+    end
+
+    @output.start_running
+
+    config.jobs.each do |job|
+      @output.running(job.name)
+
+      @options.executables.each do |executable|
+        result = run_benchmark(job, executable)
+        @output.benchmark_stats(result)
+      end
+    end
+
+    @output.finish
+  end
+
+  private
+
+  def validate_config(config)
+    config.jobs.each do |job|
+      unless job.script.is_a?(String)
+        raise NotImplementedError.new(
+          "#{self.class.name} only accepts String, but got #{job.script.inspect}"
+        )
+      end
+    end
+  end
+
+  # @param [Benchmark::Driver::Configuration::Job] job
+  # @param [Benchmark::Driver::Configuration::Executable] executable
+  def run_benchmark(job, executable)
+    fields = @output.class::REQUIRED_FIELDS
+    if fields == [:real]
+      Benchmark::Driver::RepeatableRunner.new(job).run(
+        runner: build_runner(executable.path),
+        repeat_count: @options.repeat_count,
+      ).tap do |result|
+        if result.real < 0
+          raise Benchmark::Driver::ExecutionTimeTooShort.new(job, result.iterations)
+        end
+      end
+    elsif fields == [:max_rss] # TODO: we can also capture other metrics with /usr/bin/time
+      raise '/usr/bin/time is not available' unless File.exist?('/usr/bin/time')
+
+      script = BenchmarkScript.new(job.prelude, job.script).full_script(job.loop_count)
+      with_file(script) do |script_path|
+        out = IO.popen(['/usr/bin/time', executable.path, script_path], err: [:child, :out], &:read)
+        match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(out)
+        raise "Unexpected format given from /usr/bin/time:\n#{out}" unless match_data[:maxresident]
+
+        Benchmark::Driver::BenchmarkResult.new(job).tap do |result|
+          result.max_rss = Integer(match_data[:maxresident])
+        end
+      end
+    else
+      raise "Unexpected REQUIRED_FIELDS for #{self.class.name}: #{fields.inspect}"
+    end
+  end
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
+  def run_warmup(jobs)
+    @output.start_warming
+
+    jobs.each do |job|
+      next if job.loop_count
+      @output.warming(job.name)
+
+      result = Benchmark::Driver::DurationRunner.new(job).run(
+        seconds:    WARMUP_DURATION,
+        unit_iters: guess_ip100ms(job),
+        runner:     build_runner, # TODO: should use executables instead of RbConfig.ruby
+      )
+      job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
+
+      if result.duration < 0
+        raise Benchmark::Driver::ExecutionTimeTooShort.new(job, result.iterations)
+      end
+      @output.warmup_stats(result)
+    end
+  end
+
+  # @param [Benchmark::Driver::Configuration::Job] job
+  def guess_ip100ms(job)
+    ip100ms = 0
+    GUESS_TIMES.each do |times|
+      seconds = build_runner.call(job, times) # TODO: should use executables instead of RbConfig.ruby
+      ip100ms = (times.to_f / (seconds * 10.0)).ceil # ceil for times=1
+      if GUESS_THRESHOLD < seconds
+        return ip100ms
+      end
+    end
+    if ip100ms < 0
+      raise Benchmark::Driver::ExecutionTimeTooShort.new(job, GUESS_TIMES.last)
+    end
+    ip100ms
+  end
+
+  # @param [String] path - Path to Ruby executable
+  # @return [Proc] - Lambda to run benchmark
+  def build_runner(path = RbConfig.ruby)
+    lambda do |job, times|
+      benchmark = BenchmarkScript.new(job.prelude, job.script)
+      measure_seconds(path, benchmark.full_script(times)) -
+        measure_seconds(path, benchmark.overhead_script(times))
+    end
+  end
+
+  def with_file(content, &block)
+    Tempfile.create(File.basename(__FILE__)) do |f|
+      f.write(content)
+      f.close
+      block.call(f.path)
+    end
+  end
+
+  def measure_seconds(ruby, script)
+    with_file(script) do |path|
+      cmd = [ruby, path].shelljoin
+
+      before = Benchmark::Driver::Time.now
+      system(cmd, out: File::NULL)
+      after = Benchmark::Driver::Time.now
+
+      after.to_f - before.to_f
+    end
+  end
+
+  class BenchmarkScript < Struct.new(:prelude, :script)
+    BATCH_SIZE = 50
+
+    def overhead_script(times)
+      raise ArgumentError.new("Negative times: #{times}") if times < 0
+      <<-RUBY
+#{prelude}
+__benchmark_driver_i = 0
+while __benchmark_driver_i < #{times / BATCH_SIZE}
+  __benchmark_driver_i += 1
+end
+      RUBY
+    end
+
+    def full_script(times)
+      raise ArgumentError.new("Negative times: #{times}") if times < 0
+      <<-RUBY
+#{prelude}
+__benchmark_driver_i = 0
+while __benchmark_driver_i < #{times / BATCH_SIZE}
+  __benchmark_driver_i += 1
+  #{"#{script};" * BATCH_SIZE}
+end
+#{"#{script};" * (times % BATCH_SIZE)}
+      RUBY
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-22 00:00:00.000000000 Z
+date: 2017-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,7 +38,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Benchmark driver for different Ruby executables
+description: Fully-featured accurate benchmark driver for Ruby
 email:
 - takashikkbn@gmail.com
 executables:
@@ -49,19 +49,49 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - benchmark_driver.gemspec
-- benchmarks/core/array.yml
-- benchmarks/example_multi.yml
-- benchmarks/example_single.yml
-- benchmarks/lib/erb.yml
 - bin/console
 - bin/setup
+- examples/call.rb
+- examples/call_blank.rb
+- examples/call_erb.rb
+- examples/call_interpolation.rb
+- examples/exec_blank.rb
+- examples/exec_interpolation.rb
+- examples/yaml/array_duration_time.yml
+- examples/yaml/array_loop.yml
+- examples/yaml/array_loop_memory.yml
+- examples/yaml/array_loop_time.yml
+- examples/yaml/blank_hash.yml
+- examples/yaml/blank_hash_array.yml
+- examples/yaml/blank_loop.yml
+- examples/yaml/blank_loop_time.yml
+- examples/yaml/blank_string.yml
+- examples/yaml/blank_string_array.yml
+- examples/yaml/example_multi.yml
+- examples/yaml/example_single.yml
 - exe/benchmark-driver
 - lib/benchmark/driver.rb
+- lib/benchmark/driver/benchmark_result.rb
+- lib/benchmark/driver/configuration.rb
+- lib/benchmark/driver/duration_runner.rb
+- lib/benchmark/driver/error.rb
+- lib/benchmark/driver/repeatable_runner.rb
+- lib/benchmark/driver/ruby_dsl_parser.rb
+- lib/benchmark/driver/time.rb
 - lib/benchmark/driver/version.rb
+- lib/benchmark/driver/yaml_parser.rb
+- lib/benchmark/output.rb
+- lib/benchmark/output/ips.rb
+- lib/benchmark/output/memory.rb
+- lib/benchmark/output/time.rb
+- lib/benchmark/runner.rb
+- lib/benchmark/runner/call.rb
+- lib/benchmark/runner/exec.rb
 - lib/benchmark_driver.rb
 homepage: https://github.com/k0kubun/benchmark_driver
 licenses:
@@ -75,7 +105,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -83,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
-summary: Benchmark driver for different Ruby executables
+summary: Fully-featured accurate benchmark driver for Ruby
 test_files: []