RubyGems - benchmark_driver - Versions diffs - 0.3.0 → 0.4.0 - Mend

benchmark_driver 0.3.0 → 0.4.0

Files changed (49) hide show

checksums.yaml +5 -5
data/.gitignore +0 -4
data/.travis.yml +10 -6
data/Gemfile +7 -2
data/Gemfile.lock +30 -0
data/README.md +125 -117
data/Rakefile +14 -7
data/benchmark_driver.gemspec +2 -4
data/bin/console +1 -1
data/examples/call.rb +12 -0
data/examples/call_blank.rb +13 -0
data/examples/call_erb.rb +33 -0
data/examples/call_interpolation.rb +13 -0
data/examples/exec_blank.rb +14 -0
data/examples/exec_interpolation.rb +15 -0
data/examples/yaml/array_duration_time.yml +3 -0
data/examples/yaml/array_loop.yml +3 -0
data/examples/yaml/array_loop_memory.yml +6 -0
data/examples/yaml/array_loop_time.yml +4 -0
data/examples/yaml/blank_hash.yml +8 -0
data/examples/yaml/blank_hash_array.yml +10 -0
data/examples/yaml/blank_loop.yml +9 -0
data/examples/yaml/blank_loop_time.yml +10 -0
data/examples/yaml/blank_string.yml +6 -0
data/examples/yaml/blank_string_array.yml +8 -0
data/examples/yaml/example_multi.yml +6 -0
data/{benchmarks → examples/yaml}/example_single.yml +0 -0
data/exe/benchmark-driver +44 -18
data/lib/benchmark/driver.rb +52 -257
data/lib/benchmark/driver/benchmark_result.rb +21 -0
data/lib/benchmark/driver/configuration.rb +65 -0
data/lib/benchmark/driver/duration_runner.rb +24 -0
data/lib/benchmark/driver/error.rb +16 -0
data/lib/benchmark/driver/repeatable_runner.rb +18 -0
data/lib/benchmark/driver/ruby_dsl_parser.rb +57 -0
data/lib/benchmark/driver/time.rb +12 -0
data/lib/benchmark/driver/version.rb +2 -2
data/lib/benchmark/driver/yaml_parser.rb +103 -0
data/lib/benchmark/output.rb +16 -0
data/lib/benchmark/output/ips.rb +114 -0
data/lib/benchmark/output/memory.rb +57 -0
data/lib/benchmark/output/time.rb +57 -0
data/lib/benchmark/runner.rb +13 -0
data/lib/benchmark/runner/call.rb +97 -0
data/lib/benchmark/runner/exec.rb +190 -0
metadata +40 -10
data/benchmarks/core/array.yml +0 -4
data/benchmarks/example_multi.yml +0 -10
data/benchmarks/lib/erb.yml +0 -30

data/examples/call.rb ADDED Viewed

@@ -0,0 +1,12 @@
+require 'benchmark/driver'
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+  small_a = "Hello"
+  small_b = "World"
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+end

data/examples/call_blank.rb ADDED Viewed

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+class Array
+  alias_method :blank?, :empty?
+end
+Benchmark.driver(runner: :call) do |x|
+  array = []
+  x.report('array.empty?') { array.empty? }
+  x.report('array.blank?') { array.blank? }
+  x.compare!
+end

data/examples/call_erb.rb ADDED Viewed

@@ -0,0 +1,33 @@
+require 'benchmark/driver'
+require 'erb'
+require 'erubi'
+require 'erubis'
+data = DATA.read
+mod = Module.new
+mod.instance_eval("def self.erb(title, content); #{ERB.new(data).src}; end", "(ERB)")
+mod.instance_eval("def self.erubis(title, content); #{Erubi::Engine.new(data).src}; end", "(Erubi)")
+mod.instance_eval("def self.erubi(title, content); #{Erubis::Eruby.new(data).src}; end", "(Erubis)")
+title = "hello world!"
+content = "hello world!\n" * 10
+Benchmark.driver do |x|
+  x.report("ERB #{RUBY_VERSION}")       { mod.erb(title, content) }
+  x.report("Erubis #{Erubis::VERSION}") { mod.erubis(title, content) }
+  x.report("Erubi #{Erubi::VERSION}")   { mod.erubi(title, content) }
+  x.compare!
+end
+__END__
+<html>
+  <head> <%= title %> </head>
+  <body>
+    <h1> <%= title %> </h1>
+    <p>
+      <%= content %>
+    </p>
+  </body>
+</html>

data/examples/call_interpolation.rb ADDED Viewed

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+  small_a = "Hello"
+  small_b = "World"
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+  x.compare!
+end

data/examples/exec_blank.rb ADDED Viewed

@@ -0,0 +1,14 @@
+require 'benchmark/driver'
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude = <<-EOS
+    class Array
+      alias_method :blank?, :empty?
+    end
+    array = []
+  EOS
+  x.report(script: 'array.empty?')
+  x.report(script: 'array.blank?')
+  x.compare!
+end

data/examples/exec_interpolation.rb ADDED Viewed

@@ -0,0 +1,15 @@
+require 'benchmark/driver'
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude = <<-EOS
+    large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+    large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+    small_a = "Hello"
+    small_b = "World"
+  EOS
+  x.report('large', script: '"#{large_a}, #{large_b}!"')
+  x.report('small', script: '"#{small_a}, #{small_b}!"')
+  x.compare!
+end

data/examples/yaml/array_duration_time.yml ADDED Viewed

@@ -0,0 +1,3 @@
+output: time
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/array_loop.yml ADDED Viewed

@@ -0,0 +1,3 @@
+loop_count: 6000000
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/array_loop_memory.yml ADDED Viewed

@@ -0,0 +1,6 @@
+output: memory
+loop_count: 6000000
+benchmark:
+  array10: a = [1,2,3,4,5,6,7,8,9,10]
+  array100: a = [1,2,3,4,5,6,7,8,9,10] * 10
+  array1000: a = [1,2,3,4,5,6,7,8,9,10] * 100

data/examples/yaml/array_loop_time.yml ADDED Viewed

@@ -0,0 +1,4 @@
+output: time
+loop_count: 6000000
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/blank_hash.yml ADDED Viewed

@@ -0,0 +1,8 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_hash_array.yml ADDED Viewed

@@ -0,0 +1,10 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  - name: Array#empty?
+    script: array.empty?
+  - name: Array#blank?
+    script: array.blank?

data/examples/yaml/blank_loop.yml ADDED Viewed

@@ -0,0 +1,9 @@
+loop_count: 20000000
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_loop_time.yml ADDED Viewed

@@ -0,0 +1,10 @@
+output: time
+loop_count: 20000000
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_string.yml ADDED Viewed

@@ -0,0 +1,6 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark: array.blank?

data/examples/yaml/blank_string_array.yml ADDED Viewed

@@ -0,0 +1,8 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  - array.empty?
+  - array.blank?

data/examples/yaml/example_multi.yml ADDED Viewed

@@ -0,0 +1,6 @@
+prelude: |
+  a = 'a' * 100
+  b = 'b' * 100
+benchmark:
+  join: '[a, b].join'
+  str-interp: '"#{a}#{b}"'

data/{benchmarks → examples/yaml}/example_single.yml RENAMED Viewed

File without changes

data/exe/benchmark-driver CHANGED Viewed

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
 $:.unshift File.expand_path('../lib', __dir__)
-require 'benchmark_driver'
+require 'benchmark/driver'
+require 'benchmark/driver/yaml_parser'
 require 'optparse'
 require 'yaml'
@@ -10,32 +11,57 @@ args = OptionParser.new do |o|
   o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML]"
   o.on('-e', '--executables [EXECS]', 'Ruby executables (e1::path1; e2::path2; e3::path3;...)') do |e|
     options[:execs] ||= []
-    e.split(/;/).each do |path|
-      options[:execs] << path
+    e.split(';').each do |name_path|
+      name, path = name_path.split('::', 2)
+      options[:execs] << Benchmark::Driver::Configuration::Executable.new(name, path || name)
     end
   end
-  o.on('-r', '--rbenv [VERSION]', 'Ruby executable in rbenv') do |r|
+  o.on('--rbenv [VERSIONS]', 'Ruby executables in rbenv (2.3.5;2.4.2;...)') do |r|
     options[:execs] ||= []
-    r.split(/;/).each do |version|
-      options[:execs] << "#{version}::#{`RBENV_VERSION='#{version}' rbenv which ruby`.rstrip}"
+    r.split(';').each do |version|
+      path = `RBENV_VERSION='#{version}' rbenv which ruby`.rstrip
+      abort "Failed to execute 'rbenv which ruby'" unless $?.success?
+      options[:execs] << Benchmark::Driver::Configuration::Executable.new(version, path)
     end
   end
-  o.on('-i [DURATION]', '--ips [SECONDS]', "Measure IPS in duration seconds (default: #{Benchmark::Driver::DEFAULT_IPS_DURATION})") do |i|
-    options[:measure_type] = 'ips'
-    options[:measure_num]  = Integer(i) if i
+  o.on('-c', '--compare', 'Compare results (currently only supported in ips output)') do |v|
+    options[:compare] = v
   end
-  o.on('-l [COUNT]', '--loop-count [COUNT]', "Measure execution time with loop count (default: #{Benchmark::Driver::DEFAULT_LOOP_COUNT})") do |l|
-    options[:measure_type] = 'loop_count'
-    options[:measure_num]  = Integer(l) if l
-  end
-  o.on('-v', '--verbose') do |v|
-    options[:verbose] = v
+  o.on('-r', '--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result') do |v|
+    begin
+      options[:repeat_count] = Integer(v)
+    rescue ArgumentError
+      abort "-r, --repeat-count must take Integer, but got #{v.inspect}"
+    end
   end
 end.parse!(ARGV)
 abort "No YAML file is specified" if args.empty?
-driver = Benchmark::Driver.new(options)
 args.each do |yaml|
-  default = { name: File.basename(yaml, '.*') }
-  driver.run(default.merge(YAML.load(File.read(yaml))))
+  yaml = YAML.load(File.read(yaml))
+  Benchmark::Driver::Configuration.symbolize_keys!(yaml)
+  begin
+    config = Benchmark::Driver::YamlParser.parse(yaml)
+  rescue ArgumentError
+    $stderr.puts "benchmark-driver: Failed to parse #{yaml.dump}."
+    $stderr.puts '  YAML format may be wrong. See error below:'
+    $stderr.puts
+    raise
+  end
+  options.each do |key, value|
+    case key
+    when :compare
+      config.output_options.compare = value
+    when :execs
+      config.runner_options.executables = value
+    when :repeat_count
+      config.runner_options.repeat_count = value
+    else
+      raise "Unhandled option: #{key.inspect}"
+    end
+  end
+  Benchmark::Driver.run(config)
 end

data/lib/benchmark/driver.rb CHANGED Viewed

@@ -1,273 +1,68 @@
-require 'benchmark'
-require 'benchmark/driver/version'
-require 'tempfile'
-class Benchmark::Driver
-  MEASURE_TYPES = %w[loop_count ips]
-  DEFAULT_LOOP_COUNT = 100_000
-  DEFAULT_IPS_DURATION = 1
-  # @param [String] measure_type - "loop_count"|"ips"
-  # @param [Integer,nil] measure_num - Loop count for "loop_type", duration seconds for "ips"
-  # @param [Array<String>] execs - ["path1", "path2"] or `["ruby1::path1", "ruby2::path2"]`
-  # @param [Boolean] verbose
-  def initialize(measure_type: 'loop_count', measure_num: nil, execs: ['ruby'], verbose: false)
-    unless MEASURE_TYPES.include?(measure_type)
-      abort "unsupported measure type: #{measure_type.dump}"
-    end
-    @measure_type = measure_type
-    @measure_num = measure_num
-    @execs = execs.map do |exec|
-      name, path = exec.split('::', 2)
-      Executable.new(name, path || name)
-    end
-    @verbose = verbose
-  end
-  # @param [Hash] root_hash
-  def run(root_hash)
-    root = BenchmarkRoot.new(Hash[root_hash.map { |k, v| [k.to_sym, v] }])
-    results = root.benchmarks.map do |benchmark|
-      metrics_by_exec = {}
-      iterations = calc_iterations(@execs.first, benchmark)
-      @execs.each do |exec|
-        if @verbose
-          puts "--- Running #{benchmark.name.dump} with #{exec.name.dump} #{iterations} times ---"
-          puts "#{benchmark.benchmark_script(iterations)}\n"
-        end
-        elapsed_time = run_benchmark(exec, benchmark, iterations)
-        metrics_by_exec[exec] = BenchmarkMetrics.new(iterations, elapsed_time)
-      end
-      BenchmarkResult.new(benchmark.name, metrics_by_exec)
-    end
-    puts if @verbose
-    case @measure_type
-    when 'loop_count'
-      LoopCountReporter.report(@execs, results)
-    when 'ips'
-      IpsReporter.report(@execs, results)
-    else
-      raise "unexpected measure type: #{@measure_type.dump}"
-    end
-  end
-  private
-  # Estimate iterations to finish benchmark within `@duration`.
-  def calc_iterations(exec, benchmark)
-    case @measure_type
-    when 'loop_count'
-      @measure_num || benchmark.loop_count || DEFAULT_LOOP_COUNT
-    when 'ips'
-      # TODO: Change to try from 1, 10, 100 ...
-      base = 1000
-      time = run_benchmark(exec, benchmark, base)
-      duration = @measure_num || DEFAULT_IPS_DURATION
-      (duration / time * base).to_i
-    else
-      raise "unexpected measure type: #{@measure_type.dump}"
-    end
-  end
-  def run_benchmark(exec, benchmark, iterations)
-    # TODO: raise error if negative
-    measure_script(exec.path, benchmark.benchmark_script(iterations)) -
-      measure_script(exec.path, benchmark.overhead_script(iterations))
-  end
-  def measure_script(ruby, script)
-    Tempfile.create(File.basename(__FILE__)) do |f|
-      f.write(script)
-      f.close
-      cmd = "#{ruby} #{f.path}"
-      Benchmark.measure { system(cmd, out: File::NULL) }.real
-    end
-  end
-  class BenchmarkRoot
-    # @param [String] name
-    # @param [String] prelude
-    # @param [Integer,nil] loop_count
-    # @param [String,nil]  benchmark  - For running single instant benchmark
-    # @param [Array<Hash>] benchmarks - For running multiple benchmarks
-    def initialize(name:, prelude: '', loop_count: nil, benchmark: nil, benchmarks: [])
-      if benchmark
-        unless benchmarks.empty?
-          raise ArgumentError.new("Only either :benchmark or :benchmarks can be specified")
-        end
-        @benchmarks = [BenchmarkScript.new(name: name, prelude: prelude, benchmark: benchmark)]
-      else
-        @benchmarks = benchmarks.map do |hash|
-          BenchmarkScript.new(Hash[hash.map { |k, v| [k.to_sym, v] }]).tap do |b|
-            b.inherit_root(prelude: prelude, loop_count: loop_count)
-          end
-        end
-      end
-    end
-    # @return [Array<BenchmarkScript>]
-    attr_reader :benchmarks
-  end
-  class BenchmarkScript
-    # @param [String] name
-    # @param [String] prelude
-    # @param [String] benchmark
-    def initialize(name:, prelude: '', loop_count: nil, benchmark:)
-      @name = name
-      @prelude = prelude
-      @loop_count = loop_count
-      @benchmark = benchmark
-    end
-    # @return [String]
-    attr_reader :name
-    # @return [Integer]
-    attr_reader :loop_count
-    def inherit_root(prelude:, loop_count:)
-      @prelude = "#{prelude}\n#{@prelude}"
-      if @loop_count.nil? && loop_count
-        @loop_count = loop_count
-      end
-    end
-    def overhead_script(iterations)
-      <<-RUBY
-#{@prelude}
-__benchmark_driver_i = 0
-while __benchmark_driver_i < #{iterations}
-  __benchmark_driver_i += 1
-end
-      RUBY
-    end
-    def benchmark_script(iterations)
-      <<-RUBY
-#{@prelude}
-__benchmark_driver_i = 0
-while __benchmark_driver_i < #{iterations}
-  __benchmark_driver_i += 1
-#{@benchmark}
-end
-      RUBY
-    end
-  end
-  class BenchmarkResult < Struct.new(
-    :name,            # @param [String]
-    :metrics_by_exec, # @param [Hash{ Executable => BenchmarkMetrics }]
-  )
-    def iterations_of(exec)
-      metrics_by_exec.fetch(exec).iterations
-    end
-    def elapsed_time_of(exec)
-      metrics_by_exec.fetch(exec).elapsed_time
-    end
-    def ips_of(exec)
-      iterations_of(exec) / elapsed_time_of(exec)
-    end
-  end
-  class BenchmarkMetrics < Struct.new(
-    :iterations,   # @param [Integer]
-    :elapsed_time, # @param [Float] - Elapsed time in seconds
-  )
-  end
-  class Executable < Struct.new(
-    :name, # @param [String]
-    :path, # @param [String]
-  )
-  end
-  module LoopCountReporter
+module Benchmark
+  module Driver
     class << self
-      # @param [Array<Executable>] execs
-      # @param [Array<BenchmarkResult>] results
-      def report(execs, results)
-        puts "benchmark results:"
-        puts "Execution time (sec)"
-        puts "#{'%-16s' % 'name'} #{execs.map { |e| "%-8s" % e.name }.join(' ')}"
-        results.each do |result|
-          print '%-16s ' % result.name
-          puts execs.map { |exec|
-            "%-8s" % ("%.3f" % result.elapsed_time_of(exec))
-          }.join(' ')
+      # Main function which is used by both RubyDriver and YamlDriver.
+      # @param [Benchmark::Driver::Configuration] config
+      def run(config)
+        validate_config(config)
+        runner_class = Runner.find(config.runner_options.type)
+        output_class = Output.find(config.output_options.type)
+        missing_fields = output_class::REQUIRED_FIELDS - runner_class::SUPPORTED_FIELDS
+        unless missing_fields.empty?
+          raise ArgumentError.new(
+            "#{output_class.name} requires #{missing_fields.inspect} fields "\
+            "which are not supported by #{runner_class.name}. Try using another runner."
+          )
         end
-        puts
-        if execs.size > 1
-          report_speedup(execs, results)
+        without_stdout_buffering do
+          runner = runner_class.new(
+            config.runner_options,
+            output: output_class.new(
+              jobs:        config.jobs,
+              executables: config.runner_options.executables,
+              options:     config.output_options,
+            ),
+          )
+          runner.run(config)
         end
+      rescue Benchmark::Driver::Error => e
+        $stderr.puts "\n\nFailed to execute benchmark!\n\n#{e.class.name}:\n  #{e.message}"
+        exit 1
       end
       private
-      def report_speedup(execs, results)
-        compared = execs.first
-        rest = execs - [compared]
+      def validate_config(config)
+        # TODO: make sure all scripts are the same class
+      end
-        puts "Speedup ratio: compare with the result of `#{compared.name}' (greater is better)"
-        puts "#{'%-16s' % 'name'} #{rest.map { |e| "%-8s" % e.name }.join(' ')}"
-        results.each do |result|
-          print '%-16s ' % result.name
-          puts rest.map { |exec|
-            "%-8s" % ("%.3f" % (result.ips_of(exec) / result.ips_of(compared)))
-          }.join(' ')
-        end
-        puts
+      # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+      #
+      # Currently benchmark_driver supports only output to stdout.
+      # In future exetension, this may be included in Output plugins.
+      def without_stdout_buffering
+        sync, $stdout.sync = $stdout.sync, true
+        yield
+      ensure
+        $stdout.sync = sync
       end
     end
   end
-  module IpsReporter
-    class << self
-      # @param [Array<Executable>] execs
-      # @param [Array<BenchmarkResult>] results
-      def report(execs, results)
-        puts "Result -------------------------------------------"
-        puts "#{' ' * 16} #{execs.map { |e| "%13s" % e.name }.join('  ')}"
-        results.each do |result|
-          print '%16s ' % result.name
-          puts execs.map { |exec|
-            "%13s" % ("%.1f i/s" % result.ips_of(exec))
-          }.join('  ')
-        end
-        puts
-        if execs.size > 1
-          compare(execs, results)
-        end
-      end
-      private
-      def compare(execs, results)
-        results.each do |result|
-          puts "Comparison: #{result.name}"
+  # RubyDriver entrypoint.
+  def self.driver(*args, &block)
+    dsl = Driver::RubyDslParser.new(*args)
+    block.call(dsl)
-          sorted = execs.sort_by { |e| -result.ips_of(e) }
-          first = sorted.first
-          sorted.each do |exec|
-            if exec == first
-              puts "%16s: %12s i/s" % [first.name, "%.1f" % result.ips_of(first)]
-            else
-              puts "%16s: %12s i/s - %.2fx slower" % [exec.name, "%.1f" % result.ips_of(exec), result.ips_of(first) / result.ips_of(exec)]
-            end
-          end
-          puts
-        end
-      end
-    end
+    Driver.run(dsl.configuration)
   end
 end
+require 'benchmark/output'
+require 'benchmark/runner'
+require 'benchmark/driver/error'
+require 'benchmark/driver/ruby_dsl_parser'
+require 'benchmark/driver/version'