benchmark_driver 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38e9cc1bf336f881a4fc545120cb2203159ac9a5daf7330e7cb0bceb1eeacd42
-  data.tar.gz: ee7acb4616b4296164adad96e0118496a02b3d4cf2fe90441dd932d329613cb8
+  metadata.gz: 28707c030056cf68bac9e42f5de7acbfe8fa2700a4f23c95ee23983a76aba56b
+  data.tar.gz: 3734f5f19eabda233d2ad189f898c4214282c1fa7d341fafdabf34b026fe31ae
 SHA512:
-  metadata.gz: 12eec26e9a0f747120ffe1a13aeb75bbe5aac4d9b96d4b8c5ddb01b1998f849ce2c570d5db486144ef3b622a7548a1e5adb040159993923b2dbc60de9e2053c0
-  data.tar.gz: 790cc1ef422e7eebd2f2a83de6c1feeea6d8f313f28042c15350f8a36019dce706605fc3c1a00728c7b31023586395f99a06ceeb6cc28421cb69ad8eed1eec99
+  metadata.gz: 275941867a95295a4cd351009a69d84ad38c88d30b1e066a3835581c5d1b6c7c46fcc503e8980e04dcfeaac9d61e9bdbe22bdba56b6b979f3d16fa727372650e
+  data.tar.gz: e67add21317516a0dc3fbeadfb6cb01723260fdbac1fcfaddc06d53e25b5e309c71fdf3dabd8cc14aadff6e223fba28b5fbdb1bdfb08e3cc1a2fcc074849dd7d

data/.travis.yml

@@ -11,4 +11,5 @@ branches:
     - master
 before_install: gem install bundler -v 1.15.4
 script:
+  - bundle exec rake ruby_examples
   - bundle exec rake yaml_examples

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.6.2
+
+- Resurrect support of Ruby interface dropped at v0.6.0
+- Invalidate wrong configuration
+- Decide runner type automatically for Ruby interface
+
 # v0.6.1
 
 - Support markdown output

data/README.md

@@ -18,11 +18,12 @@ NOTE: Pending ones are ~slashed~.
 ### Pluggable & Fully Featured
 
 - Flexible and real-time output format in ips, execution time, ~markdown table~, etc.
-- Output format is pluggable
+- Runner and output are all pluggable
 - ~Integrated benchmark support using external libraries~
 
 ### Flexible Interface
 
+- Ruby interface similar to benchmark stdlib, benchmark-ips
 - YAML input to easily manage structured benchmark set
 - Comparing multiple Ruby binaries, even with miniruby
 
@@ -34,6 +35,58 @@ $ gem install benchmark_driver
 
 ## Usage
 
+### Ruby Interface: Compatible Mode
+
+This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's good for migration.
+
+```rb
+require 'benchmark/driver'
+require 'active_support/all'
+array = []
+
+Benchmark.driver do |x|
+  x.report('blank?') { array.blank? }
+  x.report('empty?') { array.empty? }
+  x.compare!
+end
+```
+
+### Ruby Interface: Low Overhead Mode
+
+This interface generates code to profile with low overhead and executes it.
+
+```rb
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
+
+  x.report('blank?', script: 'array.blank?')
+  x.report('empty?', script: 'array.empty?')
+end
+```
+
+or simply:
+
+```rb
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  x.prelude = <<~RUBY
+    require 'active_support/all'
+    array = []
+  RUBY
+
+  x.report(script: 'array.blank?')
+  x.report(script: 'array.empty?')
+end
+```
+
+### Structured YAML Input
+
 With `benchmark-driver` command, you can describe benchmark with YAML input.
 
 ```
@@ -45,7 +98,7 @@ Usage: benchmark-driver [options] [YAML]
     -r, --repeat-count [NUM]         Try benchmark NUM times and use the fastest result
 ```
 
-### Running single script
+#### Running single script
 
 With following `example_single.yml`,
 
@@ -71,7 +124,7 @@ Comparison:
   erb.result (2.4.2):    109268.4 i/s - 1.13x  slower
 ```
 
-### Running multiple scripts
+#### Running multiple scripts
 
 One YAML file can contain multiple benchmark scripts.
 With following `example_multi.yml`,
@@ -106,14 +159,15 @@ Comparison:
 
 ## TODO
 ### Runner
+- [x] Call
 - [x] Exec
 - [ ] Eval
 
 ### Output
 - [x] IPS
 - [x] Time
-- [x] Memory
 - [ ] CPU/System/Real Time
+- [ ] Memory
 - [ ] Markdown Table
 
 ## Contributing

data/Rakefile

@@ -1,6 +1,15 @@
 require 'bundler/gem_tasks'
 require 'shellwords'
 
+task :ruby_examples do
+  Dir.glob(File.expand_path('./examples/*.rb', __dir__)).sort.each do |file|
+    Bundler.with_clean_env do
+      sh ['time', 'bundle', 'exec', 'ruby', file].shelljoin
+    end
+    puts
+  end
+end
+
 task :yaml_examples do
   Dir.glob(File.expand_path('./examples/yaml/*.yml', __dir__)).sort.each do |file|
     Bundler.with_clean_env do
@@ -10,4 +19,4 @@ task :yaml_examples do
   end
 end
 
-task default: :yaml_examples
+task default: [:ruby_examples, :yaml_examples]

data/examples/call.rb

@@ -0,0 +1,12 @@
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+
+  small_a = "Hello"
+  small_b = "World"
+
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+end

data/examples/call_blank.rb

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+
+class Array
+  alias_method :blank?, :empty?
+end
+
+Benchmark.driver(runner: :call) do |x|
+  array = []
+
+  x.report('array.empty?') { array.empty? }
+  x.report('array.blank?') { array.blank? }
+  x.compare!
+end

data/examples/call_erb.rb

@@ -0,0 +1,33 @@
+require 'benchmark/driver'
+require 'erb'
+require 'erubi'
+require 'erubis'
+
+data = DATA.read
+
+mod = Module.new
+mod.instance_eval("def self.erb(title, content); #{ERB.new(data).src}; end", "(ERB)")
+mod.instance_eval("def self.erubis(title, content); #{Erubi::Engine.new(data).src}; end", "(Erubi)")
+mod.instance_eval("def self.erubi(title, content); #{Erubis::Eruby.new(data).src}; end", "(Erubis)")
+
+title = "hello world!"
+content = "hello world!\n" * 10
+
+Benchmark.driver do |x|
+  x.report("ERB #{RUBY_VERSION}")       { mod.erb(title, content) }
+  x.report("Erubis #{Erubis::VERSION}") { mod.erubis(title, content) }
+  x.report("Erubi #{Erubi::VERSION}")   { mod.erubi(title, content) }
+  x.compare!
+end
+
+__END__
+
+<html>
+  <head> <%= title %> </head>
+  <body>
+    <h1> <%= title %> </h1>
+    <p>
+      <%= content %>
+    </p>
+  </body>
+</html>

data/examples/call_interpolation.rb

@@ -0,0 +1,13 @@
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+
+  small_a = "Hello"
+  small_b = "World"
+
+  x.report('large') { "#{large_a}, #{large_b}!" }
+  x.report('small') { "#{small_a}, #{small_b}!" }
+  x.compare!
+end

data/examples/exec_blank.rb

@@ -0,0 +1,14 @@
+require 'benchmark/driver'
+
+Benchmark.driver(runner: :exec) do |x|
+  x.prelude = <<-EOS
+    class Array
+      alias_method :blank?, :empty?
+    end
+    array = []
+  EOS
+
+  x.report(script: 'array.empty?')
+  x.report(script: 'array.blank?')
+  x.compare!
+end

data/examples/exec_interpolation.rb

@@ -0,0 +1,15 @@
+require 'benchmark/driver'
+
+Benchmark.driver do |x|
+  x.prelude = <<-EOS
+    large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
+    large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
+
+    small_a = "Hello"
+    small_b = "World"
+  EOS
+
+  x.report('large', script: '"#{large_a}, #{large_b}!"')
+  x.report('small', script: '"#{small_a}, #{small_b}!"')
+  x.compare!
+end

data/exe/benchmark-driver

@@ -3,7 +3,6 @@ $:.unshift File.expand_path('../lib', __dir__)
 
 require 'benchmark/driver'
 require 'benchmark/driver/bundle_installer'
-require 'benchmark/driver/configuration'
 require 'benchmark/driver/yaml_parser'
 require 'optparse'
 require 'yaml'

data/lib/benchmark/driver.rb

@@ -1,10 +1,23 @@
 module Benchmark
+  # RubyDriver entrypoint.
+  def self.driver(*args, &block)
+    dsl = Driver::RubyDslParser.new(*args)
+    block.call(dsl)
+
+    Driver.run(dsl.configuration)
+  end
+
   module Driver
+    class InvalidConfig < StandardError; end
+
     class << self
-      # Main function which is used by exe/benchmark-driver.
+      # Main function which is used by both RubyDriver and YamlDriver.
       # @param [Benchmark::Driver::Configuration] config
       def run(config)
         validate_config(config)
+        if config.runner_options.type.nil?
+          config.runner_options.type = runner_type_for(config)
+        end
 
         runner_class = Runner.find(config.runner_options.type)
         output_class = Output.find(config.output_options.type)
@@ -36,7 +49,19 @@ module Benchmark
       private
 
       def validate_config(config)
-        # TODO: make sure all scripts are the same class
+        if config.jobs.empty?
+          raise InvalidConfig.new('No benchmark script is specified')
+        end
+
+        script_class = config.jobs.first.script.class
+        unless config.jobs.all? { |j| j.script.is_a?(script_class) }
+          raise InvalidConfig.new('Benchmark scripts include both String and Proc. Only either of them should be specified.')
+        end
+      end
+
+      def runner_type_for(config)
+        script_class = config.jobs.first.script.class
+        script_class == Proc ? :call : :exec
       end
 
       # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
@@ -56,4 +81,5 @@ end
 require 'benchmark/output'
 require 'benchmark/runner'
 require 'benchmark/driver/error'
+require 'benchmark/driver/ruby_dsl_parser'
 require 'benchmark/driver/version'

data/lib/benchmark/driver/ruby_dsl_parser.rb

@@ -0,0 +1,59 @@
+require 'benchmark/driver/configuration'
+
+class Benchmark::Driver::RubyDslParser
+  # @param [Symbol,nil] runner - If this is nil, this is automatically decided by Benchmark::Driver#runner_type_for
+  # @param [Symbol] output
+  def initialize(runner: nil, output: :ips)
+    @prelude = nil
+    @jobs = []
+    @runner_options = Benchmark::Driver::Configuration::RunnerOptions.new(runner)
+    @output_options = Benchmark::Driver::Configuration::OutputOptions.new(output)
+  end
+
+  # API to fetch configuration parsed from DSL
+  # @return [Benchmark::Driver::Configuration]
+  def configuration
+    @jobs.each do |job|
+      job.prelude = @prelude
+    end
+    Benchmark::Driver::Configuration.new(@jobs).tap do |c|
+      c.runner_options = @runner_options
+      c.output_options = @output_options
+    end
+  end
+
+  # @param [String] prelude - Script required for benchmark whose execution time is not measured.
+  def prelude=(prelude)
+    unless prelude.is_a?(String)
+      raise ArgumentError.new("prelude must be String but got #{prelude.inspect}")
+    end
+    unless @prelude.nil?
+      raise ArgumentError.new("prelude is already set:\n#{@prelude}")
+    end
+
+    @prelude = prelude
+  end
+
+  # @param [String,nil] name   - Name shown on result output. This must be provided if block is given.
+  # @param [String,nil] script - Benchmarked script in String. Only either of script or block must be provided.
+  # @param [Proc,nil]   block  - Benchmarked Proc object.
+  def report(name = nil, script: nil, &block)
+    if script.nil? && !block_given?
+      raise ArgumentError.new('script or block must be provided')
+    elsif !script.nil? && block_given?
+      raise ArgumentError.new('script and block cannot be specified at the same time')
+    elsif name.nil? && block_given?
+      raise ArgumentError.new('name must be specified if block is given')
+    elsif !name.nil? && !name.is_a?(String)
+      raise ArgumentError.new("name must be String but got #{name.inspect}")
+    elsif !script.nil? && !script.is_a?(String)
+      raise ArgumentError.new("script must be String but got #{script.inspect}")
+    end
+
+    @jobs << Benchmark::Driver::Configuration::Job.new(name || script, script || block)
+  end
+
+  def compare!
+    @output_options.compare = true
+  end
+end

data/lib/benchmark/driver/version.rb

@@ -1,5 +1,5 @@
 module Benchmark
   module Driver
-    VERSION = '0.6.1'
+    VERSION = '0.6.2'
   end
 end

data/lib/benchmark/runner.rb

@@ -2,8 +2,6 @@ module Benchmark::Runner
   # Benchmark::Runner is pluggable.
   # Create `Benchmark::Runner::FooBar` as benchmark-runner-foo_bar.gem and specify `runner: foo_bar`.
   #
-  # Currently this has no options other than Exec, but this will have Eval for sure.
-  #
   # @param [Symbol] name
   def self.find(name)
     class_name = Benchmark::Driver::Configuration.camelize(name.to_s)
@@ -11,4 +9,5 @@ module Benchmark::Runner
   end
 end
 
+require 'benchmark/runner/call'
 require 'benchmark/runner/exec'

data/lib/benchmark/runner/call.rb

@@ -0,0 +1,97 @@
+require 'benchmark/driver/benchmark_result'
+require 'benchmark/driver/duration_runner'
+require 'benchmark/driver/repeatable_runner'
+require 'benchmark/driver/time'
+
+# Run benchmark by calling #call on running ruby.
+#
+# Multiple Ruby binaries: x
+# Memory output: x
+class Benchmark::Runner::Call
+  # This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
+  SUPPORTED_FIELDS = [:real]
+
+  WARMUP_DURATION    = 2
+  BENCHMARK_DURATION = 5
+
+  # @param [Benchmark::Driver::Configuration::RunnerOptions] options
+  # @param [Benchmark::Output::*] output - Object that responds to methods used in this class
+  def initialize(options, output:)
+    @options = options
+    @output  = output
+  end
+
+  # @param [Benchmark::Driver::Configuration] config
+  def run(config)
+    validate_config(config)
+
+    if config.jobs.any?(&:warmup_needed?)
+      run_warmup(config.jobs)
+    end
+
+    @output.start_running
+
+    config.jobs.each do |job|
+      @output.running(job.name)
+
+      result = Benchmark::Driver::RepeatableRunner.new(job).run(
+        runner: method(:call_times),
+        repeat_count: @options.repeat_count,
+      )
+
+      @output.benchmark_stats(result)
+    end
+
+    @output.finish
+  end
+
+  private
+
+  def validate_config(config)
+    if config.runner_options.executables_specified?
+      raise ArgumentError.new("Benchmark::Runner::Call can't run other Ruby executables")
+    end
+
+    config.jobs.each do |job|
+      unless job.script.respond_to?(:call)
+        raise NotImplementedError.new(
+          "#{self.class.name} only accepts objects that respond to :call, but got #{job.script.inspect}"
+        )
+      end
+    end
+  end
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
+  def run_warmup(jobs)
+    @output.start_warming
+
+    jobs.each do |job|
+      next if job.loop_count
+      @output.warming(job.name)
+
+      result = Benchmark::Driver::DurationRunner.new(job).run(
+        seconds:    WARMUP_DURATION,
+        unit_iters: 1,
+        runner:     method(:call_times),
+      )
+      job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
+
+      @output.warmup_stats(result)
+    end
+  end
+
+  def call_times(job, times)
+    script = job.script
+    i = 0
+
+    before = Benchmark::Driver::Time.now
+    while i < times
+      script.call
+      i += 1
+    end
+    after = Benchmark::Driver::Time.now
+
+    after.to_f - before.to_f
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-16 00:00:00.000000000 Z
+date: 2017-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,6 +57,12 @@ files:
 - bin/bench
 - bin/console
 - bin/setup
+- examples/call.rb
+- examples/call_blank.rb
+- examples/call_erb.rb
+- examples/call_interpolation.rb
+- examples/exec_blank.rb
+- examples/exec_interpolation.rb
 - examples/yaml/array_duration_time.yml
 - examples/yaml/array_loop.yml
 - examples/yaml/blank_hash.yml
@@ -74,6 +80,7 @@ files:
 - lib/benchmark/driver/duration_runner.rb
 - lib/benchmark/driver/error.rb
 - lib/benchmark/driver/repeatable_runner.rb
+- lib/benchmark/driver/ruby_dsl_parser.rb
 - lib/benchmark/driver/time.rb
 - lib/benchmark/driver/version.rb
 - lib/benchmark/driver/yaml_parser.rb
@@ -83,6 +90,7 @@ files:
 - lib/benchmark/output/memory.rb
 - lib/benchmark/output/time.rb
 - lib/benchmark/runner.rb
+- lib/benchmark/runner/call.rb
 - lib/benchmark/runner/exec.rb
 - lib/benchmark_driver.rb
 homepage: https://github.com/k0kubun/benchmark_driver