benchmark_driver 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28707c030056cf68bac9e42f5de7acbfe8fa2700a4f23c95ee23983a76aba56b
-  data.tar.gz: 3734f5f19eabda233d2ad189f898c4214282c1fa7d341fafdabf34b026fe31ae
+  metadata.gz: 5c1f7fc0a8d066c0e6fec248737f0a6a0b62cd84b15ed40121f56817a9c6322a
+  data.tar.gz: 95c7c9a3c4a847ac77592c4cd3dbb0a3f2068e83de442bd8184e999dda9979a9
 SHA512:
-  metadata.gz: 275941867a95295a4cd351009a69d84ad38c88d30b1e066a3835581c5d1b6c7c46fcc503e8980e04dcfeaac9d61e9bdbe22bdba56b6b979f3d16fa727372650e
-  data.tar.gz: e67add21317516a0dc3fbeadfb6cb01723260fdbac1fcfaddc06d53e25b5e309c71fdf3dabd8cc14aadff6e223fba28b5fbdb1bdfb08e3cc1a2fcc074849dd7d
+  metadata.gz: c0b0e1be74c8e7a8a7b849828daffd6e586a84c3d2a188c1a84882a40ec83ca7717fc66c62d3d723d53b184e582b01cdc14ca74384c3cdfb93e2b80693593706
+  data.tar.gz: b4343234565046d92eb320eb3b5c9ffb7a73560ea9906cd7d541e7f9d065080fecc73f8d6542555abfe95a57e7438e797109acd5412ae6654b0c92631c3a2034

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# v0.7.0
+
+- Change Ruby interface for specifying prelude and script
+  - #prelude= is remaed to #prelude
+  - `script:` is no longer a keyword argument
+- Add Eval runner and it's made default for Ruby interface when script is String
+
 # v0.6.2
 
 - Resurrect support of Ruby interface dropped at v0.6.0

data/README.md

@@ -42,9 +42,9 @@ This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's go
 ```rb
 require 'benchmark/driver'
 require 'active_support/all'
-array = []
 
 Benchmark.driver do |x|
+  array = []
   x.report('blank?') { array.blank? }
   x.report('empty?') { array.empty? }
   x.compare!
@@ -57,15 +57,12 @@ This interface generates code to profile with low overhead and executes it.
 
 ```rb
 require 'benchmark/driver'
+require 'active_support/all'
 
 Benchmark.driver do |x|
-  x.prelude = <<~RUBY
-    require 'active_support/all'
-    array = []
-  RUBY
-
-  x.report('blank?', script: 'array.blank?')
-  x.report('empty?', script: 'array.empty?')
+  x.prelude %{ array = [] }
+  x.report 'blank?', %{ array.blank? }
+  x.report 'empty?', %{ array.empty? }
 end
 ```
 
@@ -73,15 +70,12 @@ or simply:
 
 ```rb
 require 'benchmark/driver'
+require 'active_support/all'
 
 Benchmark.driver do |x|
-  x.prelude = <<~RUBY
-    require 'active_support/all'
-    array = []
-  RUBY
-
-  x.report(script: 'array.blank?')
-  x.report(script: 'array.empty?')
+  x.prelude %{ array = [] }
+  x.report %{ array.blank?' }
+  x.report %{ array.empty?' }
 end
 ```
 

data/examples/eval_blank.rb

@@ -0,0 +1,12 @@
+require 'benchmark/driver'
+
+class Array
+  alias_method :blank?, :empty?
+end
+
+Benchmark.driver(runner: :eval) do |x|
+  x.prelude %{ array = [] }
+  x.report 'Array#empty?', %{ array.empty? }
+  x.report 'Array#blank?', %{ array.blank? }
+  x.compare!
+end

data/examples/{exec_interpolation.rb → eval_interpolation.rb}

@@ -1,7 +1,7 @@
 require 'benchmark/driver'
 
 Benchmark.driver do |x|
-  x.prelude = <<-EOS
+  x.prelude <<-EOS
     large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
     large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
 
@@ -9,7 +9,7 @@ Benchmark.driver do |x|
     small_b = "World"
   EOS
 
-  x.report('large', script: '"#{large_a}, #{large_b}!"')
-  x.report('small', script: '"#{small_a}, #{small_b}!"')
+  x.report('large', '"#{large_a}, #{large_b}!"')
+  x.report('small', '"#{small_a}, #{small_b}!"')
   x.compare!
 end

data/examples/exec_blank.rb

@@ -1,14 +1,13 @@
 require 'benchmark/driver'
 
 Benchmark.driver(runner: :exec) do |x|
-  x.prelude = <<-EOS
+  x.prelude <<-EOS
     class Array
       alias_method :blank?, :empty?
     end
     array = []
   EOS
-
-  x.report(script: 'array.empty?')
-  x.report(script: 'array.blank?')
+  x.report 'Array#empty?', %{ array.empty? }
+  x.report 'Array#blank?', %{ array.blank? }
   x.compare!
 end

data/examples/exec_blank_simple.rb

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude <<-EOS
+    class Array
+      alias_method :blank?, :empty?
+    end
+    array = []
+  EOS
+  x.report %{ array.empty? }
+  x.report %{ array.blank? }
+  x.compare!
+end

data/lib/benchmark/driver.rb

@@ -57,11 +57,19 @@ module Benchmark
         unless config.jobs.all? { |j| j.script.is_a?(script_class) }
           raise InvalidConfig.new('Benchmark scripts include both String and Proc. Only either of them should be specified.')
         end
+
+        # TODO: invalidate prelude for call runner
       end
 
       def runner_type_for(config)
         script_class = config.jobs.first.script.class
-        script_class == Proc ? :call : :exec
+        if script_class == Proc
+          :call
+        elsif config.runner_options.executables_specified?
+          :exec
+        else
+          :eval
+        end
       end
 
       # benchmark_driver ouputs logs ASAP. This enables sync flag for it.

data/lib/benchmark/driver/ruby_dsl_parser.rb

@@ -22,25 +22,23 @@ class Benchmark::Driver::RubyDslParser
     end
   end
 
-  # @param [String] prelude - Script required for benchmark whose execution time is not measured.
-  def prelude=(prelude)
-    unless prelude.is_a?(String)
-      raise ArgumentError.new("prelude must be String but got #{prelude.inspect}")
+  # @param [String] prelude_script - Script required for benchmark whose execution time is not measured.
+  def prelude(prelude_script)
+    unless prelude_script.is_a?(String)
+      raise ArgumentError.new("prelude must be String but got #{prelude_script.inspect}")
     end
     unless @prelude.nil?
       raise ArgumentError.new("prelude is already set:\n#{@prelude}")
     end
 
-    @prelude = prelude
+    @prelude = prelude_script
   end
 
   # @param [String,nil] name   - Name shown on result output. This must be provided if block is given.
   # @param [String,nil] script - Benchmarked script in String. Only either of script or block must be provided.
   # @param [Proc,nil]   block  - Benchmarked Proc object.
-  def report(name = nil, script: nil, &block)
-    if script.nil? && !block_given?
-      raise ArgumentError.new('script or block must be provided')
-    elsif !script.nil? && block_given?
+  def report(name = nil, script = nil, &block)
+    if !script.nil? && block_given?
       raise ArgumentError.new('script and block cannot be specified at the same time')
     elsif name.nil? && block_given?
       raise ArgumentError.new('name must be specified if block is given')
@@ -50,7 +48,7 @@ class Benchmark::Driver::RubyDslParser
       raise ArgumentError.new("script must be String but got #{script.inspect}")
     end
 
-    @jobs << Benchmark::Driver::Configuration::Job.new(name || script, script || block)
+    @jobs << Benchmark::Driver::Configuration::Job.new(name, script || block || name)
   end
 
   def compare!

data/lib/benchmark/driver/version.rb

@@ -1,5 +1,5 @@
 module Benchmark
   module Driver
-    VERSION = '0.6.2'
+    VERSION = '0.7.0'
   end
 end

data/lib/benchmark/runner.rb

@@ -10,4 +10,5 @@ module Benchmark::Runner
 end
 
 require 'benchmark/runner/call'
+require 'benchmark/runner/eval'
 require 'benchmark/runner/exec'

data/lib/benchmark/runner/eval.rb

@@ -0,0 +1,152 @@
+require 'benchmark/driver/benchmark_result'
+require 'benchmark/driver/duration_runner'
+require 'benchmark/driver/repeatable_runner'
+require 'benchmark/driver/time'
+
+# Run benchmark by calling compiled script on running ruby.
+#
+# Multiple Ruby binaries: x
+# Memory output: x
+class Benchmark::Runner::Eval
+  # This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
+  SUPPORTED_FIELDS = [:real]
+
+  WARMUP_DURATION    = 2
+  BENCHMARK_DURATION = 5
+  GUESS_TIMES = [1, 1_000, 1_000_000, 10_000_000, 100_000_000]
+  GUESS_THRESHOLD = 0.4 # 400ms
+
+  # @param [Benchmark::Driver::Configuration::RunnerOptions] options
+  # @param [Benchmark::Output::*] output - Object that responds to methods used in this class
+  def initialize(options, output:)
+    @options = options
+    @output  = output
+  end
+
+  # @param [Benchmark::Driver::Configuration] config
+  def run(config)
+    validate_config(config)
+
+    if config.jobs.any?(&:warmup_needed?)
+      run_warmup(config.jobs)
+    end
+
+    @output.start_running
+
+    config.jobs.each do |job|
+      @output.running(job.name)
+
+      result = Benchmark::Driver::RepeatableRunner.new(job).run(
+        runner: method(:eval_times),
+        repeat_count: @options.repeat_count,
+      )
+
+      @output.benchmark_stats(result)
+    end
+
+    @output.finish
+  end
+
+  private
+
+  def validate_config(config)
+    if config.runner_options.executables_specified?
+      raise ArgumentError.new("#{self.class.name} can't run other Ruby executables")
+    end
+
+    config.jobs.each do |job|
+      unless job.script.is_a?(String)
+        raise NotImplementedError.new(
+          "#{self.class.name} only accepts String, but got #{job.script.inspect}"
+        )
+      end
+    end
+  end
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
+  def run_warmup(jobs)
+    @output.start_warming
+
+    jobs.each do |job|
+      next if job.loop_count
+      @output.warming(job.name)
+
+      result = Benchmark::Driver::DurationRunner.new(job).run(
+        seconds:    WARMUP_DURATION,
+        unit_iters: guess_ip100ms(job),
+        runner:     method(:eval_times),
+      )
+      job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
+
+      @output.warmup_stats(result)
+    end
+  end
+
+  # @param [Benchmark::Driver::Configuration::Job] job
+  def guess_ip100ms(job)
+    ip100ms = 0
+    GUESS_TIMES.each do |times|
+      seconds = eval_times(job, times)
+      ip100ms = (times.to_f / (seconds * 10.0)).ceil # ceil for times=1
+      if GUESS_THRESHOLD < seconds
+        return ip100ms
+      end
+    end
+    if ip100ms < 0
+      raise Benchmark::Driver::ExecutionTimeTooShort.new(job, GUESS_TIMES.last)
+    end
+    ip100ms
+  end
+
+  def eval_times(job, times)
+    benchmark = BenchmarkScript.new(job.prelude, job.script)
+    mod = Module.new
+    benchmark.compile_overhead!(mod, times)
+    benchmark.compile_full_script!(mod, times)
+
+    before = Benchmark::Driver::Time.now
+    mod.overhead
+    after = Benchmark::Driver::Time.now
+    overhead_duration = after.to_f - before.to_f
+
+    before = Benchmark::Driver::Time.now
+    mod.full_script
+    after = Benchmark::Driver::Time.now
+    full_script_duration = after.to_f - before.to_f
+
+    full_script_duration - overhead_duration
+  end
+
+  class BenchmarkScript < Struct.new(:prelude, :script)
+    BATCH_SIZE = 1000
+
+    def compile_overhead!(mod, times)
+      raise ArgumentError.new("Negative times: #{times}") if times < 0
+      mod.module_eval(<<-RUBY)
+def self.overhead
+  #{prelude}
+  __benchmark_driver_i = 0
+  while __benchmark_driver_i < #{times / BATCH_SIZE}
+    __benchmark_driver_i += 1
+  end
+end
+      RUBY
+    end
+
+    def compile_full_script!(mod, times)
+      raise ArgumentError.new("Negative times: #{times}") if times < 0
+      mod.module_eval(<<-RUBY)
+def self.full_script
+  #{prelude}
+  __benchmark_driver_i = 0
+  while __benchmark_driver_i < #{times / BATCH_SIZE}
+    __benchmark_driver_i += 1
+    #{"#{script};" * BATCH_SIZE}
+  end
+  #{"#{script};" * (times % BATCH_SIZE)}
+end
+      RUBY
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Takashi Kokubun
@@ -61,8 +61,10 @@ files:
 - examples/call_blank.rb
 - examples/call_erb.rb
 - examples/call_interpolation.rb
+- examples/eval_blank.rb
+- examples/eval_interpolation.rb
 - examples/exec_blank.rb
-- examples/exec_interpolation.rb
+- examples/exec_blank_simple.rb
 - examples/yaml/array_duration_time.yml
 - examples/yaml/array_loop.yml
 - examples/yaml/blank_hash.yml
@@ -91,6 +93,7 @@ files:
 - lib/benchmark/output/time.rb
 - lib/benchmark/runner.rb
 - lib/benchmark/runner/call.rb
+- lib/benchmark/runner/eval.rb
 - lib/benchmark/runner/exec.rb
 - lib/benchmark_driver.rb
 homepage: https://github.com/k0kubun/benchmark_driver