benchmark_driver_monotonic_raw 0.14.13

data/lib/benchmark_driver/runner/once.rb

@@ -0,0 +1,118 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+require 'shellwords'
+
+# Run only once, for testing
+class BenchmarkDriver::Runner::Once
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    jobs = jobs.map do |job|
+      Job.new(job.to_h.merge(loop_count: 1)) # to show this on output
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          job.runnable_contexts(@contexts).each do |context|
+            duration = run_benchmark(job, context: context) # no repeat support
+            if BenchmarkDriver::Result::ERROR.equal?(duration)
+              value = BenchmarkDriver::Result::ERROR
+            else
+              value = 1.0 / duration
+            end
+
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+              @output.report(values: { METRIC => value }, duration: duration, loop_count: 1)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [Float] duration
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render(result: f.path)) do |path|
+        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
+        if $?.success?
+          Float(f.read)
+        else
+          BenchmarkDriver::Result::ERROR
+        end
+      end
+    end
+  end
+
+  def with_script(script)
+    if @config.verbose >= 2
+      sep = '-' * 30
+      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+    end
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  def execute(*args)
+    output = IO.popen(args, err: [:child, :out], &:read) # handle stdout?
+    unless $?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+    end
+    output
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      <<-RUBY
+#{prelude}
+__bmdv_before = Time.now
+#{script}
+__bmdv_after = Time.now
+File.write(#{result.dump}, (__bmdv_after - __bmdv_before).inspect)
+#{teardown}
+      RUBY
+    end
+  end
+  private_constant :BenchmarkScript
+end

data/lib/benchmark_driver/runner/recorded.rb

@@ -0,0 +1,73 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'tempfile'
+require 'shellwords'
+
+# Run only once, for testing
+class BenchmarkDriver::Runner::Recorded
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,              # @param [String] name - This is mandatory for all runner
+    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
+    :warmup_results,    # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
+    :benchmark_results, # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
+    :contexts,          # @param [Array<BenchmarkDriver::Context>]
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [Hash{ BenchmarkDriver::Job => Hash{ TrueClass,FalseClass => Hash{ BenchmarkDriver::Context => BenchmarkDriver::Result } } }] job_warmup_context_result
+    # @param [BenchmarkDriver::Metrics::Type] metrics
+    def parse(job_warmup_context_result:, metrics:)
+      job_warmup_context_result.map do |job, warmup_context_result|
+        Job.new(
+          name: job.name,
+          warmup_results: warmup_context_result.fetch(true, {}),
+          benchmark_results: warmup_context_result.fetch(false, {}),
+          metrics: metrics,
+          contexts: warmup_context_result.values.map(&:keys).flatten!.tap(&:uniq!),
+        )
+      end
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Runner::Recorded::Job>] record
+  def run(records)
+    records.each do |record|
+      unless record.warmup_results.empty?
+        # TODO:
+      end
+    end
+
+    @output.with_benchmark do
+      records.each do |record|
+        @output.with_job(name: record.name) do
+          record.benchmark_results.each do |context, result|
+            @output.with_context(
+              name: context.name,
+              executable: context.executable,
+              gems: context.gems,
+              prelude: context.prelude,
+            ) do
+              @output.report(
+                values: result.values,
+                duration: result.duration,
+                loop_count: result.loop_count,
+                environment: result.environment,
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -0,0 +1,146 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'tempfile'
+require 'shellwords'
+require 'open3'
+
+# Use stdout of ruby command
+class BenchmarkDriver::Runner::RubyStdout
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,                   # @param [String] name - This is mandatory for all runner
+    :metrics,                # @param [Array<BenchmarkDriver::Metric>]
+    :command,                # @param [Array<String>]
+    :working_directory,      # @param [String,NilClass]
+    :value_from_stdout,      # @param [String]
+    :environment_from_stdout # @param [Hash{ String => String }]
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [String] name
+    # @param [String] command
+    # @param [String,NilClass] working_directory
+    # @param [Hash] metrics_type
+    # @param [String] stdout_to_metrics
+    def parse(name:, command:, working_directory: nil, metrics:, environment: {})
+      unless metrics.is_a?(Hash)
+        raise ArgumentError.new("metrics must be Hash, but got #{metrics.class}")
+      end
+      if metrics.size == 0
+        raise ArgumentError.new('At least one metric must be specified"')
+      elsif metrics.size != 1
+        raise NotImplementedError.new('Having multiple metrics is not supported yet')
+      end
+
+      metric, value_from_stdout = parse_metric(*metrics.first)
+      environment_from_stdout = Hash[environment.map { |k, v| [k, parse_environment(v)] }]
+
+      Job.new(
+        name: name,
+        command: command.shellsplit,
+        working_directory: working_directory,
+        metrics: [metric],
+        value_from_stdout: value_from_stdout,
+        environment_from_stdout: environment_from_stdout,
+      )
+    end
+
+    private
+
+    def parse_metric(name, unit:, from_stdout:, larger_better: true, worse_word: 'slower')
+      metric = BenchmarkDriver::Metric.new(
+        name: name,
+        unit: unit,
+        larger_better: larger_better,
+        worse_word: worse_word,
+      )
+      [metric, from_stdout]
+    end
+
+    def parse_environment(from_stdout:)
+      from_stdout
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    metric = jobs.first.metrics.first
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          @contexts.each do |context|
+            exec = context.executable
+            repeat_params = { config: @config, larger_better: metric.larger_better }
+            value, environment = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
+              stdout = with_chdir(job.working_directory) do
+                with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
+              end
+              script = StdoutToMetrics.new(
+                stdout: stdout,
+                value_from_stdout: job.value_from_stdout,
+                environment_from_stdout: job.environment_from_stdout,
+              )
+              [script.value, script.environment]
+            end
+
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(values: { metric => value }, environment: environment)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def with_ruby_prefix(executable, &block)
+    env = ENV.to_h.dup
+    ENV['PATH'] = "#{File.dirname(executable.command.first)}:#{ENV['PATH']}"
+    block.call
+  ensure
+    ENV.replace(env)
+  end
+
+  def with_chdir(working_directory, &block)
+    if working_directory
+      Dir.chdir(working_directory) { block.call }
+    else
+      block.call
+    end
+  end
+
+  def execute(*args)
+    stdout, stderr, status = Open3.capture3(*args)
+    unless status.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus}):\n[stdout]:\n#{stdout}\n[stderr]:\n#{stderr}"
+    end
+    stdout
+  end
+
+  StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :value_from_stdout, :environment_from_stdout) do
+    def value
+      eval(value_from_stdout, binding)
+    end
+
+    def environment
+      ret = {}
+      environment_from_stdout.each do |name, from_stdout|
+        ret[name] = eval(from_stdout, binding)
+      end
+      ret
+    end
+  end
+  private_constant :StdoutToMetrics
+end

data/lib/benchmark_driver/runner/time.rb

@@ -0,0 +1,20 @@
+require 'benchmark_driver/runner/ips'
+
+class BenchmarkDriver::Runner::Time < BenchmarkDriver::Runner::Ips
+  METRIC = BenchmarkDriver::Metric.new(name: 'Execution time', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  # Overriding BenchmarkDriver::Runner::Ips#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Ips#value_duration
+  def value_duration(duration:, loop_count:)
+    [duration, duration]
+  end
+end

data/lib/benchmark_driver/struct.rb

@@ -0,0 +1,98 @@
+# Extended Struct with:
+#   * Polyfilled `keyword_init: true`
+#   * Default value configuration
+#   * Deeply freezing members
+module BenchmarkDriver
+  class ::Struct
+    SUPPORT_KEYWORD_P = begin
+                          ::Struct.new(:a, keyword_init: true)
+                          true
+                        rescue TypeError
+                          false
+                        end
+  end
+
+  class << Struct = Module.new
+    # @param [Array<Symbol>] args
+    # @param [Hash{ Symbol => Object }] defaults
+    def new(*args, defaults: {}, &block)
+      # Polyfill `keyword_init: true`
+      if ::Struct::SUPPORT_KEYWORD_P
+        klass = ::Struct.new(*args, keyword_init: true, &block)
+      else
+        klass = keyword_init_struct(*args, &block)
+      end
+
+      # Default value config
+      configure_defaults(klass, defaults)
+
+      # Force deeply freezing members
+      force_deep_freeze(klass)
+
+      klass
+    end
+
+    private
+
+    # Polyfill for Ruby < 2.5.0
+    def keyword_init_struct(*args, &block)
+      ::Struct.new(*args).tap do |klass|
+        klass.prepend(Module.new {
+          # @param [Hash{ Symbol => Object }] args
+          def initialize(**args)
+            args.each do |key, value|
+              unless members.include?(key)
+                raise ArgumentError.new("unknown keywords: #{key}")
+                next
+              end
+
+              public_send("#{key}=", value)
+            end
+          end
+        })
+        klass.prepend(Module.new(&block))
+      end
+    end
+
+    def configure_defaults(klass, defaults)
+      class << klass
+        attr_accessor :defaults
+      end
+      klass.defaults = defaults
+
+      klass.prepend(Module.new {
+        def initialize(**)
+          super
+          self.class.defaults.each do |key, value|
+            if public_send(key).nil?
+              begin
+                value = value.dup
+              rescue TypeError # for Ruby <= 2.3, like `true.dup`
+              end
+              public_send("#{key}=", value)
+            end
+          end
+        end
+      })
+
+      def klass.inherited(child)
+        child.defaults = self.defaults
+      end
+    end
+
+    def force_deep_freeze(klass)
+      klass.class_eval do
+        def freeze
+          members.each do |member|
+            value = public_send(member)
+            if value.is_a?(Array)
+              value.each(&:freeze)
+            end
+            value.freeze
+          end
+          super
+        end
+      end
+    end
+  end
+end