benchmark_driver 0.3.0 → 0.4.0

data/lib/benchmark/driver/benchmark_result.rb

@@ -0,0 +1,21 @@
+# @param [Benchmark::Driver::Configuration::Job] job
+# @param [Integer] iterations - Executed iterations of benchmark script in the job
+# @param [Float] real - Real time taken by the job
+# @param [Integer] max_rss - Maximum resident set size of the process during its lifetime, in Kilobytes.
+class Benchmark::Driver::BenchmarkResult < Struct.new(:job, :iterations, :real, :max_rss)
+  alias :duration :real
+
+  def ips
+    iterations / real
+  end
+
+  def ip100ms
+    ips / 10
+  end
+
+  def iterations
+    # runner's warmup uses `result.ips` to calculate `job.loop_count`, and thus
+    # at that moment `job.loop_count` isn't available and we need to use `super`.
+    super || job.loop_count
+  end
+end

data/lib/benchmark/driver/configuration.rb

@@ -0,0 +1,65 @@
+# @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+# @param [Benchmark::Driver::Configuration::RunnerOptions] runner_options
+# @param [Benchmark::Driver::Configuration::OutputOptions] output_options
+class Benchmark::Driver::Configuration < Struct.new(:jobs, :runner_options, :output_options)
+  # @param [String,nil] name
+  # @param [String,Proc] sctipt
+  # @param [String,nil] prelude
+  # @param [Integer,nil] loop_count - If this is nil, loop count is automatically estimated by warmup.
+  class Job < Struct.new(:name, :script, :prelude, :loop_count)
+    # @param [Integer,nil] guessed_count - Set by runner only when loop_count is nil. This is not configuration.
+    attr_accessor :guessed_count
+
+    def warmup_needed?
+      # This needs to check original configuration
+      self[:loop_count].nil?
+    end
+
+    def loop_count
+      super || guessed_count
+    end
+  end
+
+  # @param [String] name
+  # @param [String] path
+  Executable = Struct.new(:name, :path)
+
+  DEFAULT_EXECUTABLES = [Executable.new(RUBY_VERSION, RbConfig.ruby)]
+
+  # @param [Symbol] type - Type of runner
+  # @param [Array<Benchmark::Driver::Configuration::Executable>] executables
+  # @param [Integer,nil] repeat_count - Times to repeat benchmarks. When this is not nil, benchmark_driver will use the best result.
+  class RunnerOptions < Struct.new(:type, :executables, :repeat_count)
+    def initialize(*)
+      super
+      self.executables = DEFAULT_EXECUTABLES
+    end
+
+    def executables_specified?
+      executables != DEFAULT_EXECUTABLES
+    end
+  end
+
+  # @param [Symbol] type - Type of output
+  # @param [TrueClass,FalseClass] compare
+  OutputOptions = Struct.new(:type, :compare)
+
+  # @param [Object] config
+  def self.symbolize_keys!(config)
+    case config
+    when Hash
+      config.keys.each do |key|
+        config[key.to_sym] = symbolize_keys!(config.delete(key))
+      end
+    when Array
+      config.map! { |c| symbolize_keys!(c) }
+    end
+    config
+  end
+
+  # @param [String] str
+  def self.camelize(str)
+    return str if str !~ /_/ && str =~ /[A-Z]+.*/
+    str.split('_').map { |e| e.capitalize }.join
+  end
+end

data/lib/benchmark/driver/duration_runner.rb

@@ -0,0 +1,24 @@
+class Benchmark::Driver::DurationRunner
+  # @param [Benchmark::Driver::Configuration::Job] job
+  def initialize(job)
+    @job = job
+  end
+
+  # @param [Integer,Float] seconds
+  # @param [Integer] unit_iters
+  # @param [Proc] runner - should take (job, unit_iters) and return duration.
+  # @return [Benchmark::Driver::BenchmarkResult]
+  def run(seconds:, unit_iters:, runner:)
+    real_time  = 0.0
+    iterations = 0
+    unit_iters = unit_iters.to_i
+
+    benchmark_until = Benchmark::Driver::Time.now + seconds
+    while Benchmark::Driver::Time.now < benchmark_until
+      real_time  += runner.call(@job, unit_iters)
+      iterations += unit_iters
+    end
+
+    Benchmark::Driver::BenchmarkResult.new(@job, iterations, real_time)
+  end
+end

data/lib/benchmark/driver/error.rb

@@ -0,0 +1,16 @@
+module Benchmark::Driver
+  class Error < StandardError
+  end
+
+  class ExecutionTimeTooShort < Error
+    def initialize(job, iterations)
+      @job = job
+      @iterations = iterations
+    end
+
+    def message
+      "Execution time of job #{@job.name.dump} was too short in #{@iterations} executions;\n  "\
+        'Please retry, try slower script or increase loop_count.'
+    end
+  end
+end

data/lib/benchmark/driver/repeatable_runner.rb

@@ -0,0 +1,18 @@
+class Benchmark::Driver::RepeatableRunner
+  # @param [Benchmark::Driver::Configuration::Job] job
+  def initialize(job)
+    @job = job
+  end
+
+  # @param [Integer] repeat_count
+  # @param [Proc] runner - should take (job, unit_iters) and return duration.
+  # @return [Benchmark::Driver::BenchmarkResult]
+  def run(repeat_count:, runner:)
+    real_times = (repeat_count || 1).times.map do
+      runner.call(@job, @job.loop_count)
+    end
+    Benchmark::Driver::BenchmarkResult.new(@job).tap do |result|
+      result.real = real_times.select { |d| d > 0 }.min || real_times.max
+    end
+  end
+end

data/lib/benchmark/driver/ruby_dsl_parser.rb

@@ -0,0 +1,57 @@
+require 'benchmark/driver/configuration'
+
+class Benchmark::Driver::RubyDslParser
+  def initialize(runner: :call, output: :ips)
+    @prelude = nil
+    @jobs = []
+    @runner_options = Benchmark::Driver::Configuration::RunnerOptions.new(runner)
+    @output_options = Benchmark::Driver::Configuration::OutputOptions.new(output)
+  end
+
+  # API to fetch configuration parsed from DSL
+  # @return [Benchmark::Driver::Configuration]
+  def configuration
+    @jobs.each do |job|
+      job.prelude = @prelude
+    end
+    Benchmark::Driver::Configuration.new(@jobs).tap do |c|
+      c.runner_options = @runner_options
+      c.output_options = @output_options
+    end
+  end
+
+  # @param [String] prelude - Script required for benchmark whose execution time is not measured.
+  def prelude=(prelude)
+    unless prelude.is_a?(String)
+      raise ArgumentError.new("prelude must be String but got #{prelude.inspect}")
+    end
+    unless @prelude.nil?
+      raise ArgumentError.new("prelude is already set:\n#{@prelude}")
+    end
+
+    @prelude = prelude
+  end
+
+  # @param [String,nil] name   - Name shown on result output. This must be provided if block is given.
+  # @param [String,nil] script - Benchmarked script in String. Only either of script or block must be provided.
+  # @param [Proc,nil]   block  - Benchmarked Proc object.
+  def report(name = nil, script: nil, &block)
+    if script.nil? && !block_given?
+      raise ArgumentError.new('script or block must be provided')
+    elsif !script.nil? && block_given?
+      raise ArgumentError.new('script and block cannot be specified at the same time')
+    elsif name.nil? && block_given?
+      raise ArgumentError.new('name must be specified if block is given')
+    elsif !name.nil? && !name.is_a?(String)
+      raise ArgumentError.new("name must be String but got #{name.inspect}")
+    elsif !script.nil? && !script.is_a?(String)
+      raise ArgumentError.new("script must be String but got #{script.inspect}")
+    end
+
+    @jobs << Benchmark::Driver::Configuration::Job.new(name || script, script || block)
+  end
+
+  def compare!
+    @output_options.compare = true
+  end
+end

data/lib/benchmark/driver/time.rb

@@ -0,0 +1,12 @@
+module Benchmark::Driver::Time
+  if defined?(Process::CLOCK_MONOTONIC)
+    def self.now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  else
+    $stderr.puts "Process::CLOCK_MONOTONIC was unavailable. Using Time."
+    def self.now
+      ::Time.now
+    end
+  end
+end

data/lib/benchmark/driver/version.rb

@@ -1,5 +1,5 @@
 module Benchmark
-  class Driver
-    VERSION = '0.3.0'
+  module Driver
+    VERSION = '0.4.0'
   end
 end

data/lib/benchmark/driver/yaml_parser.rb

@@ -0,0 +1,103 @@
+module Benchmark::Driver::YamlParser
+  DEFAULT_RUNNER = :exec # In YamlParser, we can't use :call.
+  DEFAULT_OUTPUT = :ips
+
+  class << self
+    # @param [String] prelude
+    # @param [Integer,nil] loop_count
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    # @param [String,Symbol,Hash{ Symbol => Integer,TrueClass,FalseClass }] runner
+    # @param [String,Symbol,Hash{ Symbol => Integer,TrueClass,FalseClass }] output
+    # @return [Benchmark::Driver::Configuration]
+    def parse(prelude: '', loop_count: nil, benchmark:, runner: {}, output: {})
+      jobs = parse_benchmark(benchmark)
+      jobs.each do |job|
+        job.prelude = prelude
+        job.loop_count ||= loop_count
+      end
+
+      config = Benchmark::Driver::Configuration.new(jobs)
+      config.runner_options = parse_runner(runner)
+      config.output_options = parse_output(output)
+      config
+    end
+
+    private
+
+    # @param [String,Symbol,Hash{ Symbol => Integer,TrueClass,FalseClass }] runner
+    def parse_runner(runner)
+      case runner
+      when String, Symbol
+        Benchmark::Driver::Configuration::RunnerOptions.new(runner.to_sym)
+      when Hash
+        parse_runner_options(runner)
+      else
+        raise ArgumentError.new("Expected String, Symbol or Hash in runner, but got: #{runner.inspect}")
+      end
+    end
+
+    def parse_runner_options(type: DEFAULT_RUNNER, repeat_count: nil)
+      Benchmark::Driver::Configuration::RunnerOptions.new.tap do |r|
+        r.type = type.to_sym
+        r.repeat_count = Integer(repeat_count) if repeat_count
+      end
+    end
+
+    # @param [String,Symbol,Hash{ Symbol => Integer,TrueClass,FalseClass }] output
+    def parse_output(output)
+      case output
+      when String, Symbol
+        Benchmark::Driver::Configuration::OutputOptions.new(output.to_sym)
+      when Hash
+        parse_output_options(output)
+      else
+        raise ArgumentError.new("Expected String, Symbol or Hash in output, but got: #{output.inspect}")
+      end
+    end
+
+    def parse_output_options(type: DEFAULT_OUTPUT, compare: false)
+      Benchmark::Driver::Configuration::OutputOptions.new.tap do |r|
+        r.type = type.to_sym
+        r.compare = compare
+      end
+    end
+
+    # Parse "benchmark" declarative. This may have multiple benchmarks.
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    def parse_benchmark(benchmark)
+      case benchmark
+      when String
+        [parse_each_benchmark(benchmark)]
+      when Array
+        benchmark.map { |b| parse_each_benchmark(b) }
+      when Hash
+        benchmark.map do |key, value|
+          Benchmark::Driver::Configuration::Job.new(key.to_s, value)
+        end
+      else
+        raise ArgumentError.new("benchmark must be String, Array or Hash, but got: #{benchmark.inspect}")
+      end
+    end
+
+    # Parse one benchmark specified in "benchmark" declarative.
+    # @param [String,Hash{ Symbol => String }>] job
+    def parse_each_benchmark(benchmark)
+      case benchmark
+      when String
+        Benchmark::Driver::Configuration::Job.new(benchmark, benchmark)
+      when Hash
+        parse_job(benchmark)
+      else
+        raise ArgumentError.new("Expected String or Hash in element of benchmark, but got: #{benchmark.inspect}")
+      end
+    end
+
+    # @param [String,nil] name
+    # @param [String] script
+    # TODO: support benchmark-specific prelude
+    def parse_job(name: nil, script:)
+      name = script if name.nil?
+      Benchmark::Driver::Configuration::Job.new(name, script)
+    end
+  end
+end

data/lib/benchmark/output.rb

@@ -0,0 +1,16 @@
+module Benchmark::Output
+  class << self
+    # Benchmark::Output is pluggable.
+    # Create `Benchmark::Output::FooBar` as benchmark-output-foo_bar.gem and specify `output: foo_bar`.
+    #
+    # @param [Symbol] name
+    def find(name)
+      class_name = Benchmark::Driver::Configuration.camelize(name.to_s)
+      Benchmark::Output.const_get(class_name, false)
+    end
+  end
+end
+
+require 'benchmark/output/ips'
+require 'benchmark/output/memory'
+require 'benchmark/output/time'

data/lib/benchmark/output/ips.rb

@@ -0,0 +1,114 @@
+class Benchmark::Output::Ips
+  # This class requires runner to measure following fields in `Benchmark::Driver::BenchmarkResult` to show output.
+  REQUIRED_FIELDS = [:real]
+
+  NAME_LENGTH = 20
+
+  # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
+  # @param [Array<Benchmark::Driver::Configuration::Executable>] executables
+  # @param [Benchmark::Driver::Configuration::OutputOptions] options
+  def initialize(jobs:, executables:, options:)
+    @jobs        = jobs
+    @executables = executables
+    @options     = options
+    @results     = []
+    @name_by_result = {}
+  end
+
+  def start_warming
+    $stdout.puts 'Warming up --------------------------------------'
+  end
+
+  # @param [String] name
+  def warming(name)
+    if name.length > NAME_LENGTH
+      $stdout.puts(name)
+    else
+      $stdout.print("%#{NAME_LENGTH}s" % name)
+    end
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def warmup_stats(result)
+    $stdout.puts "#{humanize(result.ip100ms)} i/100ms"
+  end
+
+  def start_running
+    $stdout.puts 'Calculating -------------------------------------'
+    if @executables.size > 1
+      $stdout.print(' ' * NAME_LENGTH)
+      @executables.each do |executable|
+        $stdout.print(" %10s " % executable.name)
+      end
+      $stdout.puts
+    end
+  end
+
+  def running(name)
+    warming(name)
+    @row_results = []
+  end
+
+  # @param [Benchmark::Driver::BenchmarkResult] result
+  def benchmark_stats(result)
+    executable = @executables[@row_results.size]
+    $stdout.print("#{humanize(result.ips, [10, executable.name.length].max)} ")
+
+    @results << result
+    @row_results << result
+    if @row_results.size == @executables.size
+      $stdout.print("i/s - #{humanize(result.iterations)} in")
+      @row_results.each do |r|
+        $stdout.print(" %3.6fs" % r.real)
+      end
+      $stdout.puts
+    end
+
+    @name_by_result[result] = executable.name
+  end
+
+  def finish
+    if @results.size > 1 && @options.compare
+      compare
+    end
+  end
+
+  private
+
+  def humanize(value, width = 10)
+    scale = (Math.log10(value) / 3).to_i
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'B'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        scale = 0
+        ' '
+      end
+    "%#{width}.3f#{suffix}" % (value.to_f / (1000 ** scale))
+  end
+
+  def compare
+    $stdout.puts("\nComparison:")
+    results = @results.sort_by { |r| -r.ips }
+    first   = results.first
+
+    results.each do |result|
+      if result == first
+        slower = ''
+      else
+        slower = '- %.2fx  slower' % (first.ips / result.ips)
+      end
+
+      name = result.job.name
+      if @executables.size > 1
+        name = "#{name} (#{@name_by_result.fetch(result)})"
+      end
+      $stdout.puts("%#{NAME_LENGTH}s: %11.1f i/s #{slower}" % [name, result.ips])
+    end
+    $stdout.puts
+  end
+end