RubyGems - benchmark_driver - Versions diffs - 0.3.0 → 0.4.0 - Mend

benchmark_driver 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (49) hide show

checksums.yaml +5 -5
data/.gitignore +0 -4
data/.travis.yml +10 -6
data/Gemfile +7 -2
data/Gemfile.lock +30 -0
data/README.md +125 -117
data/Rakefile +14 -7
data/benchmark_driver.gemspec +2 -4
data/bin/console +1 -1
data/examples/call.rb +12 -0
data/examples/call_blank.rb +13 -0
data/examples/call_erb.rb +33 -0
data/examples/call_interpolation.rb +13 -0
data/examples/exec_blank.rb +14 -0
data/examples/exec_interpolation.rb +15 -0
data/examples/yaml/array_duration_time.yml +3 -0
data/examples/yaml/array_loop.yml +3 -0
data/examples/yaml/array_loop_memory.yml +6 -0
data/examples/yaml/array_loop_time.yml +4 -0
data/examples/yaml/blank_hash.yml +8 -0
data/examples/yaml/blank_hash_array.yml +10 -0
data/examples/yaml/blank_loop.yml +9 -0
data/examples/yaml/blank_loop_time.yml +10 -0
data/examples/yaml/blank_string.yml +6 -0
data/examples/yaml/blank_string_array.yml +8 -0
data/examples/yaml/example_multi.yml +6 -0
data/{benchmarks → examples/yaml}/example_single.yml +0 -0
data/exe/benchmark-driver +44 -18
data/lib/benchmark/driver.rb +52 -257
data/lib/benchmark/driver/benchmark_result.rb +21 -0
data/lib/benchmark/driver/configuration.rb +65 -0
data/lib/benchmark/driver/duration_runner.rb +24 -0
data/lib/benchmark/driver/error.rb +16 -0
data/lib/benchmark/driver/repeatable_runner.rb +18 -0
data/lib/benchmark/driver/ruby_dsl_parser.rb +57 -0
data/lib/benchmark/driver/time.rb +12 -0
data/lib/benchmark/driver/version.rb +2 -2
data/lib/benchmark/driver/yaml_parser.rb +103 -0
data/lib/benchmark/output.rb +16 -0
data/lib/benchmark/output/ips.rb +114 -0
data/lib/benchmark/output/memory.rb +57 -0
data/lib/benchmark/output/time.rb +57 -0
data/lib/benchmark/runner.rb +13 -0
data/lib/benchmark/runner/call.rb +97 -0
data/lib/benchmark/runner/exec.rb +190 -0
metadata +40 -10
data/benchmarks/core/array.yml +0 -4
data/benchmarks/example_multi.yml +0 -10
data/benchmarks/lib/erb.yml +0 -30

data/examples/call.rb ADDED Viewed

@@ -0,0 +1,12 @@
+require 'benchmark/driver'
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+  small_a = "Hello"
+  small_b = "World"
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+end

data/examples/call_blank.rb ADDED Viewed

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+class Array
+  alias_method :blank?, :empty?
+end
+Benchmark.driver(runner: :call) do |x|
+  array = []
+  x.report('array.empty?') { array.empty? }
+  x.report('array.blank?') { array.blank? }
+  x.compare!
+end

data/examples/call_erb.rb ADDED Viewed

@@ -0,0 +1,33 @@
+require 'benchmark/driver'
+require 'erb'
+require 'erubi'
+require 'erubis'
+data = DATA.read
+mod = Module.new
+mod.instance_eval("def self.erb(title, content); #{ERB.new(data).src}; end", "(ERB)")
+mod.instance_eval("def self.erubis(title, content); #{Erubi::Engine.new(data).src}; end", "(Erubi)")
+mod.instance_eval("def self.erubi(title, content); #{Erubis::Eruby.new(data).src}; end", "(Erubis)")
+title = "hello world!"
+content = "hello world!\n" * 10
+Benchmark.driver do |x|
+  x.report("ERB #{RUBY_VERSION}")       { mod.erb(title, content) }
+  x.report("Erubis #{Erubis::VERSION}") { mod.erubis(title, content) }
+  x.report("Erubi #{Erubi::VERSION}")   { mod.erubi(title, content) }
+  x.compare!
+end
+__END__
+<html>
+  <head> <%= title %> </head>
+  <body>
+    <h1> <%= title %> </h1>
+    <p>
+      <%= content %>
+    </p>
+  </body>
+</html>

data/examples/call_interpolation.rb ADDED Viewed

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+  small_a = "Hello"
+  small_b = "World"
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+  x.compare!
+end

data/examples/exec_blank.rb ADDED Viewed

@@ -0,0 +1,14 @@
+require 'benchmark/driver'
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude = <<-EOS
+    class Array
+      alias_method :blank?, :empty?
+    end
+    array = []
+  EOS
+  x.report(script: 'array.empty?')
+  x.report(script: 'array.blank?')
+  x.compare!
+end

data/examples/exec_interpolation.rb ADDED Viewed

@@ -0,0 +1,15 @@
+require 'benchmark/driver'
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude = <<-EOS
+    large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+    large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+    small_a = "Hello"
+    small_b = "World"
+  EOS
+  x.report('large', script: '"#{large_a}, #{large_b}!"')
+  x.report('small', script: '"#{small_a}, #{small_b}!"')
+  x.compare!
+end

data/examples/yaml/array_duration_time.yml ADDED Viewed

@@ -0,0 +1,3 @@
+output: time
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/array_loop.yml ADDED Viewed

@@ -0,0 +1,3 @@
+loop_count: 6000000
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/array_loop_memory.yml ADDED Viewed

@@ -0,0 +1,6 @@
+output: memory
+loop_count: 6000000
+benchmark:
+  array10: a = [1,2,3,4,5,6,7,8,9,10]
+  array100: a = [1,2,3,4,5,6,7,8,9,10] * 10
+  array1000: a = [1,2,3,4,5,6,7,8,9,10] * 100

data/examples/yaml/array_loop_time.yml ADDED Viewed

@@ -0,0 +1,4 @@
+output: time
+loop_count: 6000000
+benchmark:
+  bm_vm2_array: a = [1,2,3,4,5,6,7,8,9,10]

data/examples/yaml/blank_hash.yml ADDED Viewed

@@ -0,0 +1,8 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_hash_array.yml ADDED Viewed

@@ -0,0 +1,10 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  - name: Array#empty?
+    script: array.empty?
+  - name: Array#blank?
+    script: array.blank?

data/examples/yaml/blank_loop.yml ADDED Viewed

@@ -0,0 +1,9 @@
+loop_count: 20000000
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_loop_time.yml ADDED Viewed

@@ -0,0 +1,10 @@
+output: time
+loop_count: 20000000
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  empty: array.empty?
+  blank: array.blank?

data/examples/yaml/blank_string.yml ADDED Viewed

@@ -0,0 +1,6 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark: array.blank?

data/examples/yaml/blank_string_array.yml ADDED Viewed

@@ -0,0 +1,8 @@
+prelude: |
+  class Array
+    alias_method :blank?, :empty?
+  end
+  array = []
+benchmark:
+  - array.empty?
+  - array.blank?

data/examples/yaml/example_multi.yml ADDED Viewed

@@ -0,0 +1,6 @@
+prelude: |
+  a = 'a' * 100
+  b = 'b' * 100
+benchmark:
+  join: '[a, b].join'
+  str-interp: '"#{a}#{b}"'

data/{benchmarks → examples/yaml}/example_single.yml RENAMED Viewed

File without changes

data/exe/benchmark-driver CHANGED Viewed

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
 $:.unshift File.expand_path('../lib', __dir__)
-require 'benchmark_driver'
+require 'benchmark/driver'
+require 'benchmark/driver/yaml_parser'
 require 'optparse'
 require 'yaml'
@@ -10,32 +11,57 @@ args = OptionParser.new do |o|
   o.banner = "Usage: #{File.basename($0, '.*')} [options] [YAML]"
   o.on('-e', '--executables [EXECS]', 'Ruby executables (e1::path1; e2::path2; e3::path3;...)') do |e|
     options[:execs] ||= []
-    e.split(/;/).each do |path|
-      options[:execs] << path
+    e.split(';').each do |name_path|
+      name, path = name_path.split('::', 2)
+      options[:execs] << Benchmark::Driver::Configuration::Executable.new(name, path || name)
     end
   end
-  o.on('-r', '--rbenv [VERSION]', 'Ruby executable in rbenv') do |r|
+  o.on('--rbenv [VERSIONS]', 'Ruby executables in rbenv (2.3.5;2.4.2;...)') do |r|
     options[:execs] ||= []
-    r.split(/;/).each do |version|
-      options[:execs] << "#{version}::#{`RBENV_VERSION='#{version}' rbenv which ruby`.rstrip}"
+    r.split(';').each do |version|
+      path = `RBENV_VERSION='#{version}' rbenv which ruby`.rstrip
+      abort "Failed to execute 'rbenv which ruby'" unless $?.success?
+      options[:execs] << Benchmark::Driver::Configuration::Executable.new(version, path)
     end
   end
-  o.on('-i [DURATION]', '--ips [SECONDS]', "Measure IPS in duration seconds (default: #{Benchmark::Driver::DEFAULT_IPS_DURATION})") do |i|
-    options[:measure_type] = 'ips'
-    options[:measure_num]  = Integer(i) if i
+  o.on('-c', '--compare', 'Compare results (currently only supported in ips output)') do |v|
+    options[:compare] = v
   end
-  o.on('-l [COUNT]', '--loop-count [COUNT]', "Measure execution time with loop count (default: #{Benchmark::Driver::DEFAULT_LOOP_COUNT})") do |l|
-    options[:measure_type] = 'loop_count'
-    options[:measure_num]  = Integer(l) if l
-  end
-  o.on('-v', '--verbose') do |v|
-    options[:verbose] = v
+  o.on('-r', '--repeat-count [NUM]', 'Try benchmark NUM times and use the fastest result') do |v|
+    begin
+      options[:repeat_count] = Integer(v)
+    rescue ArgumentError
+      abort "-r, --repeat-count must take Integer, but got #{v.inspect}"
+    end
   end
 end.parse!(ARGV)
 abort "No YAML file is specified" if args.empty?
-driver = Benchmark::Driver.new(options)
 args.each do |yaml|
-  default = { name: File.basename(yaml, '.*') }
-  driver.run(default.merge(YAML.load(File.read(yaml))))
+  yaml = YAML.load(File.read(yaml))
+  Benchmark::Driver::Configuration.symbolize_keys!(yaml)
+  begin
+    config = Benchmark::Driver::YamlParser.parse(yaml)
+  rescue ArgumentError
+    $stderr.puts "benchmark-driver: Failed to parse #{yaml.dump}."
+    $stderr.puts '  YAML format may be wrong. See error below:'
+    $stderr.puts
+    raise
+  end
+  options.each do |key, value|
+    case key
+    when :compare
+      config.output_options.compare = value
+    when :execs
+      config.runner_options.executables = value
+    when :repeat_count
+      config.runner_options.repeat_count = value
+    else
+      raise "Unhandled option: #{key.inspect}"
+    end
+  end
+  Benchmark::Driver.run(config)
 end

data/lib/benchmark/driver.rb CHANGED Viewed

@@ -1,273 +1,68 @@
-require 'benchmark'
-require 'benchmark/driver/version'
-require 'tempfile'
-class Benchmark::Driver
-  MEASURE_TYPES = %w[loop_count ips]
-  DEFAULT_LOOP_COUNT = 100_000
-  DEFAULT_IPS_DURATION = 1
-  # @param [String] measure_type - "loop_count"|"ips"
-  # @param [Integer,nil] measure_num - Loop count for "loop_type", duration seconds for "ips"
-  # @param [Array<String>] execs - ["path1", "path2"] or `["ruby1::path1", "ruby2::path2"]`
-  # @param [Boolean] verbose
-  def initialize(measure_type: 'loop_count', measure_num: nil, execs: ['ruby'], verbose: false)
-    unless MEASURE_TYPES.include?(measure_type)
-      abort "unsupported measure type: #{measure_type.dump}"
-    end
-    @measure_type = measure_type
-    @measure_num = measure_num
-    @execs = execs.map do |exec|
-      name, path = exec.split('::', 2)
-      Executable.new(name, path || name)
-    end
-    @verbose = verbose
-  end
-  # @param [Hash] root_hash
-  def run(root_hash)
-    root = BenchmarkRoot.new(Hash[root_hash.map { |k, v| [k.to_sym, v] }])
-    results = root.benchmarks.map do |benchmark|
-      metrics_by_exec = {}
-      iterations = calc_iterations(@execs.first, benchmark)
-      @execs.each do |exec|
-        if @verbose
-          puts "--- Running #{benchmark.name.dump} with #{exec.name.dump} #{iterations} times ---"
-          puts "#{benchmark.benchmark_script(iterations)}\n"
-        end
-        elapsed_time = run_benchmark(exec, benchmark, iterations)
-        metrics_by_exec[exec] = BenchmarkMetrics.new(iterations, elapsed_time)
-      end
-      BenchmarkResult.new(benchmark.name, metrics_by_exec)
-    end
-    puts if @verbose
-    case @measure_type
-    when 'loop_count'
-      LoopCountReporter.report(@execs, results)
-    when 'ips'
-      IpsReporter.report(@execs, results)
-    else
-      raise "unexpected measure type: #{@measure_type.dump}"
-    end
-  end
-  private
-  # Estimate iterations to finish benchmark within `@duration`.
-  def calc_iterations(exec, benchmark)
-    case @measure_type
-    when 'loop_count'
-      @measure_num || benchmark.loop_count || DEFAULT_LOOP_COUNT
-    when 'ips'
-      # TODO: Change to try from 1, 10, 100 ...
-      base = 1000
-      time = run_benchmark(exec, benchmark, base)
-      duration = @measure_num || DEFAULT_IPS_DURATION
-      (duration / time * base).to_i
-    else
-      raise "unexpected measure type: #{@measure_type.dump}"
-    end
-  end
-  def run_benchmark(exec, benchmark, iterations)
-    # TODO: raise error if negative
-    measure_script(exec.path, benchmark.benchmark_script(iterations)) -
-      measure_script(exec.path, benchmark.overhead_script(iterations))
-  end
-  def measure_script(ruby, script)
-    Tempfile.create(File.basename(__FILE__)) do |f|
-      f.write(script)
-      f.close
-      cmd = "#{ruby} #{f.path}"
-      Benchmark.measure { system(cmd, out: File::NULL) }.real
-    end
-  end
-  class BenchmarkRoot
-    # @param [String] name
-    # @param [String] prelude
-    # @param [Integer,nil] loop_count
-    # @param [String,nil]  benchmark  - For running single instant benchmark
-    # @param [Array<Hash>] benchmarks - For running multiple benchmarks
-    def initialize(name:, prelude: '', loop_count: nil, benchmark: nil, benchmarks: [])
-      if benchmark
-        unless benchmarks.empty?
-          raise ArgumentError.new("Only either :benchmark or :benchmarks can be specified")
-        end
-        @benchmarks = [BenchmarkScript.new(name: name, prelude: prelude, benchmark: benchmark)]
-      else
-        @benchmarks = benchmarks.map do |hash|
-          BenchmarkScript.new(Hash[hash.map { |k, v| [k.to_sym, v] }]).tap do |b|
-            b.inherit_root(prelude: prelude, loop_count: loop_count)
-          end
-        end
-      end
-    end
-    # @return [Array<BenchmarkScript>]
-    attr_reader :benchmarks
-  end
-  class BenchmarkScript
-    # @param [String] name
-    # @param [String] prelude
-    # @param [String] benchmark
-    def initialize(name:, prelude: '', loop_count: nil, benchmark:)
-      @name = name
-      @prelude = prelude
-      @loop_count = loop_count
-      @benchmark = benchmark
-    end
-    # @return [String]
-    attr_reader :name
-    # @return [Integer]
-    attr_reader :loop_count
-    def inherit_root(prelude:, loop_count:)
-      @prelude = "#{prelude}\n#{@prelude}"
-      if @loop_count.nil? && loop_count
-        @loop_count = loop_count
-      end
-    end
-    def overhead_script(iterations)
-      <<-RUBY
-#{@prelude}
-__benchmark_driver_i = 0
-while __benchmark_driver_i < #{iterations}
-  __benchmark_driver_i += 1
-end
-      RUBY
-    end
-    def benchmark_script(iterations)
-      <<-RUBY
-#{@prelude}
-__benchmark_driver_i = 0
-while __benchmark_driver_i < #{iterations}
-  __benchmark_driver_i += 1
-#{@benchmark}
-end
-      RUBY
-    end
-  end
-  class BenchmarkResult < Struct.new(
-    :name,            # @param [String]
-    :metrics_by_exec, # @param [Hash{ Executable => BenchmarkMetrics }]
-  )
-    def iterations_of(exec)
-      metrics_by_exec.fetch(exec).iterations
-    end
-    def elapsed_time_of(exec)
-      metrics_by_exec.fetch(exec).elapsed_time
-    end
-    def ips_of(exec)
-      iterations_of(exec) / elapsed_time_of(exec)
-    end
-  end
-  class BenchmarkMetrics < Struct.new(
-    :iterations,   # @param [Integer]
-    :elapsed_time, # @param [Float] - Elapsed time in seconds
-  )
-  end
-  class Executable < Struct.new(
-    :name, # @param [String]
-    :path, # @param [String]
-  )
-  end
-  module LoopCountReporter
+module Benchmark
+  module Driver
     class << self
-      # @param [Array<Executable>] execs
-      # @param [Array<BenchmarkResult>] results
-      def report(execs, results)
-        puts "benchmark results:"
-        puts "Execution time (sec)"
-        puts "#{'%-16s' % 'name'} #{execs.map { |e| "%-8s" % e.name }.join(' ')}"
-        results.each do |result|
-          print '%-16s ' % result.name
-          puts execs.map { |exec|
-            "%-8s" % ("%.3f" % result.elapsed_time_of(exec))
-          }.join(' ')
+      # Main function which is used by both RubyDriver and YamlDriver.
+      # @param [Benchmark::Driver::Configuration] config
+      def run(config)
+        validate_config(config)
+        runner_class = Runner.find(config.runner_options.type)
+        output_class = Output.find(config.output_options.type)
+        missing_fields = output_class::REQUIRED_FIELDS - runner_class::SUPPORTED_FIELDS
+        unless missing_fields.empty?
+          raise ArgumentError.new(
+            "#{output_class.name} requires #{missing_fields.inspect} fields "\
+            "which are not supported by #{runner_class.name}. Try using another runner."
+          )
         end
-        puts
-        if execs.size > 1
-          report_speedup(execs, results)
+        without_stdout_buffering do
+          runner = runner_class.new(
+            config.runner_options,
+            output: output_class.new(
+              jobs:        config.jobs,
+              executables: config.runner_options.executables,
+              options:     config.output_options,
+            ),
+          )
+          runner.run(config)
         end
+      rescue Benchmark::Driver::Error => e
+        $stderr.puts "\n\nFailed to execute benchmark!\n\n#{e.class.name}:\n  #{e.message}"
+        exit 1
       end
       private
-      def report_speedup(execs, results)
-        compared = execs.first
-        rest = execs - [compared]
+      def validate_config(config)
+        # TODO: make sure all scripts are the same class
+      end
-        puts "Speedup ratio: compare with the result of `#{compared.name}' (greater is better)"
-        puts "#{'%-16s' % 'name'} #{rest.map { |e| "%-8s" % e.name }.join(' ')}"
-        results.each do |result|
-          print '%-16s ' % result.name
-          puts rest.map { |exec|
-            "%-8s" % ("%.3f" % (result.ips_of(exec) / result.ips_of(compared)))
-          }.join(' ')
-        end
-        puts
+      # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+      #
+      # Currently benchmark_driver supports only output to stdout.
+      # In future exetension, this may be included in Output plugins.
+      def without_stdout_buffering
+        sync, $stdout.sync = $stdout.sync, true
+        yield
+      ensure
+        $stdout.sync = sync
       end
     end
   end
-  module IpsReporter
-    class << self
-      # @param [Array<Executable>] execs
-      # @param [Array<BenchmarkResult>] results
-      def report(execs, results)
-        puts "Result -------------------------------------------"
-        puts "#{' ' * 16} #{execs.map { |e| "%13s" % e.name }.join('  ')}"
-        results.each do |result|
-          print '%16s ' % result.name
-          puts execs.map { |exec|
-            "%13s" % ("%.1f i/s" % result.ips_of(exec))
-          }.join('  ')
-        end
-        puts
-        if execs.size > 1
-          compare(execs, results)
-        end
-      end
-      private
-      def compare(execs, results)
-        results.each do |result|
-          puts "Comparison: #{result.name}"
+  # RubyDriver entrypoint.
+  def self.driver(*args, &block)
+    dsl = Driver::RubyDslParser.new(*args)
+    block.call(dsl)
-          sorted = execs.sort_by { |e| -result.ips_of(e) }
-          first = sorted.first
-          sorted.each do |exec|
-            if exec == first
-              puts "%16s: %12s i/s" % [first.name, "%.1f" % result.ips_of(first)]
-            else
-              puts "%16s: %12s i/s - %.2fx slower" % [exec.name, "%.1f" % result.ips_of(exec), result.ips_of(first) / result.ips_of(exec)]
-            end
-          end
-          puts
-        end
-      end
-    end
+    Driver.run(dsl.configuration)
   end
 end
+require 'benchmark/output'
+require 'benchmark/runner'
+require 'benchmark/driver/error'
+require 'benchmark/driver/ruby_dsl_parser'
+require 'benchmark/driver/version'