benchmark_driver 0.10.16 → 0.11.0

data/lib/benchmark_driver/output/simple.rb

@@ -1,15 +1,14 @@
 class BenchmarkDriver::Output::Simple
   NAME_LENGTH = 8
 
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  attr_writer :metrics_type
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  attr_writer :metrics
 
-  # @param [Array<BenchmarkDriver::*::Job>] jobs
-  # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  def initialize(jobs:, executables:)
-    @jobs = jobs
-    @executables = executables
-    @name_length = jobs.map { |j| j.name.size }.max
+  # @param [Array<String>] job_names
+  # @param [Array<String>] context_names
+  def initialize(job_names:, context_names:)
+    @context_names = context_names
+    @name_length = job_names.map(&:size).max
   end
 
   def with_warmup(&block)
@@ -25,13 +24,13 @@ class BenchmarkDriver::Output::Simple
     @with_benchmark = true
     without_stdout_buffering do
       # Show header
-      $stdout.puts "benchmark results (#{@metrics_type.unit}):"
+      $stdout.puts "#{@metrics.first.name} (#{@metrics.first.unit}):"
 
       # Show executable names
-      if @executables.size > 1
+      if @context_names.size > 1
         $stdout.print("#{' ' * @name_length}  ")
-        @executables.each do |executable|
-          $stdout.print("%#{NAME_LENGTH}s  " % executable.name)
+        @context_name.each do |context_name|
+          $stdout.print("%#{NAME_LENGTH}s  " % context_name)
         end
         $stdout.puts
       end
@@ -42,7 +41,7 @@ class BenchmarkDriver::Output::Simple
     @with_benchmark = false
   end
 
-  # @param [BenchmarkDriver::*::Job] job
+  # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
     if @with_benchmark
       $stdout.print("%-#{@name_length}s  " % job.name)
@@ -54,10 +53,16 @@ class BenchmarkDriver::Output::Simple
     end
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
-  def report(metrics)
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    block.call
+  end
+
+  # @param [Float] value
+  # @param [BenchmarkDriver::Metric] metic
+  def report(value:, metric:)
     if @with_benchmark
-      $stdout.print("%#{NAME_LENGTH}s  " % humanize(metrics.value))
+      $stdout.print("%#{NAME_LENGTH}s  " % humanize(value))
     else
       $stdout.print '.'
     end

data/lib/benchmark_driver/runner.rb

@@ -5,6 +5,7 @@ module BenchmarkDriver
     require 'benchmark_driver/runner/memory'
     require 'benchmark_driver/runner/once'
     require 'benchmark_driver/runner/recorded'
+    require 'benchmark_driver/runner/ruby_stdout'
     require 'benchmark_driver/runner/time'
   end
 
@@ -28,9 +29,10 @@ module BenchmarkDriver
 
       jobs.group_by(&:class).each do |klass, jobs_group|
         runner = runner_for(klass)
-        output = Output.find(config.output_type).new(
-          jobs: jobs,
-          executables: config.executables,
+        output = Output.new(
+          type: config.output_type,
+          job_names: jobs.map(&:name),
+          context_names: config.executables.map(&:name),
         )
         with_clean_env do
           runner.new(config: runner_config, output: output).run(jobs)

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -1,17 +1,17 @@
 require 'benchmark_driver/struct'
-require 'benchmark_driver/metrics'
+require 'benchmark_driver/metric'
 require 'tempfile'
 require 'shellwords'
 require 'open3'
 
-# Run only once, for testing
+# Use stdout of ruby command
 class BenchmarkDriver::Runner::CommandStdout
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = ::BenchmarkDriver::Struct.new(
     :name,              # @param [String] name - This is mandatory for all runner
     :command,           # @param [Array<String>]
     :working_directory, # @param [String,NilClass]
-    :metrics_type,      # @param [BenchmarkDriver::Metrics::Type]
+    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
     :stdout_to_metrics, # @param [String]
   )
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
@@ -26,24 +26,27 @@ class BenchmarkDriver::Runner::CommandStdout
         name: name,
         command: command.shellsplit,
         working_directory: working_directory,
-        metrics_type: parse_metrics_type(metrics_type),
+        metrics: parse_metrics(metrics_type),
         stdout_to_metrics: stdout_to_metrics,
       )
     end
 
     private
 
-    def parse_metrics_type(unit:, larger_better: nil, worse_word: nil)
-      BenchmarkDriver::Metrics::Type.new(
+    def parse_metrics(unit:, name: nil, larger_better: nil, worse_word: nil)
+      name ||= unit
+      metric = BenchmarkDriver::Metric.new(
+        name: name,
         unit: unit,
         larger_better: larger_better,
         worse_word: worse_word,
       )
+      [metric]
     end
   end
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
-  # @param [BenchmarkDriver::Output::*] output
+  # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
     @config = config
     @output = output
@@ -52,14 +55,14 @@ class BenchmarkDriver::Runner::CommandStdout
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
-    metrics_type = jobs.first.metrics_type
-    @output.metrics_type = metrics_type
+    metric = jobs.first.metrics.first
+    @output.metrics = [metric]
 
     @output.with_benchmark do
       jobs.each do |job|
-        @output.with_job(job) do
+        @output.with_job(name: job.name) do
           @config.executables.each do |exec|
-            best_value = with_repeat(metrics_type) do
+            best_value = with_repeat(metric) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
               end
@@ -69,12 +72,9 @@ class BenchmarkDriver::Runner::CommandStdout
               ).metrics_value
             end
 
-            @output.report(
-              BenchmarkDriver::Metrics.new(
-                value: best_value,
-                executable: exec,
-              )
-            )
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(value: best_value, metric: metric)
+            end
           end
         end
       end
@@ -108,12 +108,12 @@ class BenchmarkDriver::Runner::CommandStdout
   end
 
   # Return multiple times and return the best metrics
-  def with_repeat(metrics_type, &block)
+  def with_repeat(metric, &block)
     values = @config.repeat_count.times.map do
       block.call
     end
     values.sort_by do |value|
-      if metrics_type.larger_better
+      if metric.larger_better
         value
       else
         -value

data/lib/benchmark_driver/runner/ips.rb

@@ -1,5 +1,5 @@
 require 'benchmark_driver/struct'
-require 'benchmark_driver/metrics'
+require 'benchmark_driver/metric'
 require 'benchmark_driver/default_job'
 require 'benchmark_driver/default_job_parser'
 require 'tempfile'
@@ -12,10 +12,10 @@ class BenchmarkDriver::Runner::Ips
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
 
-  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
-  # @param [BenchmarkDriver::Output::*] output
+  # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
     @config = config
     @output = output
@@ -24,19 +24,23 @@ class BenchmarkDriver::Runner::Ips
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
-    set_metrics_type
+    @output.metrics = [metric]
 
     if jobs.any? { |job| job.loop_count.nil? }
       @output.with_warmup do
         jobs = jobs.map do |job|
           next job if job.loop_count # skip warmup if loop_count is set
 
-          @output.with_job(job) do
-            result = run_warmup(job, exec: job.runnable_execs(@config.executables).first)
-            metrics = build_metrics(result)
-            @output.report(metrics)
+          @output.with_job(name: job.name) do
+            executable = job.runnable_execs(@config.executables).first
+            duration, loop_count = run_warmup(job, exec: executable)
+            value, duration = value_duration(duration: duration, loop_count: loop_count)
 
-            loop_count = (result.fetch(:loop_count).to_f * @config.run_duration / result.fetch(:duration)).floor
+            @output.with_context(name: executable.name, executable: executable, duration: duration, loop_count: loop_count) do
+              @output.report(value: value, metric: metric)
+            end
+
+            loop_count = (loop_count.to_f * @config.run_duration / duration).floor
             Job.new(job.to_h.merge(loop_count: loop_count))
           end
         end
@@ -45,12 +49,14 @@ class BenchmarkDriver::Runner::Ips
 
     @output.with_benchmark do
       jobs.each do |job|
-        @output.with_job(job) do
+        @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
-            best_metrics = with_repeat(@config.repeat_count) do
+            value, duration = with_repeat(@config.repeat_count) do
               run_benchmark(job, exec: exec)
             end
-            @output.report(best_metrics)
+            @output.with_context(name: exec.name, executable: exec, duration: duration) do
+              @output.report(value: value, metric: metric)
+            end
           end
         end
       end
@@ -71,23 +77,23 @@ class BenchmarkDriver::Runner::Ips
       second_warmup_duration: @config.run_duration / 3.0, # default: 1.0
     )
 
-    hash = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+    duration, loop_count = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(warmup.render(result: f.path)) do |path|
         execute(*exec.command, path)
       end
       eval(f.read)
     end
 
-    hash.merge(executable: exec)
+    [duration, loop_count]
   end
 
   # Return multiple times and return the best metrics
   def with_repeat(repeat_times, &block)
-    all_metrics = repeat_times.times.map do
+    value_durations = repeat_times.times.map do
       block.call
     end
-    all_metrics.sort_by do |metrics|
-      metrics.value
+    value_durations.sort_by do |value, _|
+      metric.larger_better ? value : -value
     end.last
   end
 
@@ -109,25 +115,20 @@ class BenchmarkDriver::Runner::Ips
       Float(f.read)
     end
 
-    build_metrics(
+    value_duration(
       loop_count: job.loop_count,
       duration: duration,
-      executable: exec,
     )
   end
 
   # This method is overridden by BenchmarkDriver::Runner::Time
-  def build_metrics(duration:, executable:, loop_count:)
-    BenchmarkDriver::Metrics.new(
-      value: loop_count.to_f / duration,
-      duration: duration,
-      executable: executable,
-    )
+  def metric
+    METRIC
   end
 
-  # This method is overridden by BenchmarkDriver::Runner::Time
-  def set_metrics_type
-    @output.metrics_type = METRICS_TYPE
+  # Overridden by BenchmarkDriver::Runner::Time
+  def value_duration(duration:, loop_count:)
+    [loop_count.to_f / duration, duration]
   end
 
   def with_script(script)
@@ -186,7 +187,7 @@ end
 
 #{teardown}
 
-File.write(#{result.dump}, { duration: __bmdv_duration, loop_count: __bmdv_loops }.inspect)
+File.write(#{result.dump}, [__bmdv_duration, __bmdv_loops].inspect)
       RUBY
     end
   end

data/lib/benchmark_driver/runner/memory.rb

@@ -1,5 +1,5 @@
 require 'benchmark_driver/struct'
-require 'benchmark_driver/metrics'
+require 'benchmark_driver/metric'
 require 'benchmark_driver/default_job'
 require 'benchmark_driver/default_job_parser'
 require 'tempfile'
@@ -12,10 +12,12 @@ class BenchmarkDriver::Runner::Memory
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
 
-  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'bytes', larger_better: false, worse_word: 'larger')
+  METRIC = BenchmarkDriver::Metric.new(
+    name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger',
+  )
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
-  # @param [BenchmarkDriver::Output::*] output
+  # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
     @config = config
     @output = output
@@ -29,7 +31,7 @@ class BenchmarkDriver::Runner::Memory
       raise "memory output is not supported for '#{Etc.uname[:sysname]}' for now"
     end
 
-    @output.metrics_type = METRICS_TYPE
+    @output.metrics = [METRIC]
 
     if jobs.any? { |job| job.loop_count.nil? }
       jobs = jobs.map do |job|
@@ -39,12 +41,14 @@ class BenchmarkDriver::Runner::Memory
 
     @output.with_benchmark do
       jobs.each do |job|
-        @output.with_job(job) do
+        @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
-            best_metrics = with_repeat(@config.repeat_count) do
+            best_value = with_repeat(@config.repeat_count) do
               run_benchmark(job, exec: exec)
             end
-            @output.report(best_metrics)
+            @output.with_context(name: exec.name, executable: exec, loop_count: job.loop_count) do
+              @output.report(value: best_value, metric: METRIC)
+            end
           end
         end
       end
@@ -53,14 +57,12 @@ class BenchmarkDriver::Runner::Memory
 
   private
 
-  # Return multiple times and return the best metrics
+  # Return multiple times and return the best value (smallest usage)
   def with_repeat(repeat_times, &block)
-    all_metrics = repeat_times.times.map do
+    values = repeat_times.times.map do
       block.call
     end
-    all_metrics.sort_by do |metrics|
-      metrics.value
-    end.first
+    values.sort.first
   end
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
@@ -81,10 +83,7 @@ class BenchmarkDriver::Runner::Memory
     match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(output)
     raise "Unexpected format given from /usr/bin/time:\n#{out}" unless match_data[:maxresident]
 
-    BenchmarkDriver::Metrics.new(
-      value: Integer(match_data[:maxresident]) * 1000.0, # kilobytes -> bytes
-      executable: exec,
-    )
+    Integer(match_data[:maxresident]) * 1000.0 # kilobytes -> bytes
   end
 
   def with_script(script)

data/lib/benchmark_driver/runner/once.rb

@@ -1,5 +1,5 @@
 require 'benchmark_driver/struct'
-require 'benchmark_driver/metrics'
+require 'benchmark_driver/metric'
 require 'benchmark_driver/default_job'
 require 'benchmark_driver/default_job_parser'
 require 'tempfile'
@@ -12,10 +12,10 @@ class BenchmarkDriver::Runner::Once
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
 
-  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
-  # @param [BenchmarkDriver::Output::*] output
+  # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
     @config = config
     @output = output
@@ -24,7 +24,7 @@ class BenchmarkDriver::Runner::Once
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
-    @output.metrics_type = METRICS_TYPE
+    @output.metrics = [METRIC]
 
     jobs = jobs.map do |job|
       Job.new(job.to_h.merge(loop_count: 1)) # to show this on output
@@ -32,10 +32,12 @@ class BenchmarkDriver::Runner::Once
 
     @output.with_benchmark do
       jobs.each do |job|
-        @output.with_job(job) do
+        @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
-            metrics = run_benchmark(job, exec: exec) # no repeat support
-            @output.report(metrics)
+            duration = run_benchmark(job, exec: exec) # no repeat support
+            @output.with_context(name: exec.name, executable: exec, duration: duration, loop_count: 1) do
+              @output.report(value: 1.0 / duration, metric: METRIC)
+            end
           end
         end
       end
@@ -46,7 +48,7 @@ class BenchmarkDriver::Runner::Once
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
   # @param [BenchmarkDriver::Config::Executable] exec
-  # @return [BenchmarkDriver::Metrics]
+  # @return [Float] duration
   def run_benchmark(job, exec:)
     benchmark = BenchmarkScript.new(
       prelude:    job.prelude,
@@ -55,18 +57,12 @@ class BenchmarkDriver::Runner::Once
       loop_count: job.loop_count,
     )
 
-    duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
         execute(*exec.command, path)
       end
       Float(f.read)
     end
-
-    BenchmarkDriver::Metrics.new(
-      value: 1.0 / duration,
-      duration: duration,
-      executable: exec,
-    )
   end
 
   def with_script(script)