benchmark_driver 0.11.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 606b300f9ffeb798a6ac330be1e97795898afa7c
-  data.tar.gz: fb3cfa3a44eff01b189635c8ebaffcb73737567d
+  metadata.gz: cc2f6826606613671a2e6b8d0384421924bd9487
+  data.tar.gz: b310edeb8fa144e6eb6569b670ae7291e5a77a70
 SHA512:
-  metadata.gz: 8401a061574125ed5af0be54293fb2936bf8f856abb23c242232dff213770a008726839742353194d790ee6e883a00cfc1c47d42e820c7e7936f39e9d3c1d317
-  data.tar.gz: 7f742808cd90e3cdb2b983bced6a8a224a9be18c3a5121e679316edaba9648bff90711888d20fe06274c16eefe5340d6c0b52ef5c417fe1ebe0495167be23cae
+  metadata.gz: a121e71123b95d5dba0f130f2efa65aaeb9adb42c4e9046703c3afcbd3726a698b7f33e870d3c873a5eda7bbaea20bce2061f87a085118a9d8a6b36b183c6e1a
+  data.tar.gz: 942d159581bf537e62c15b46dba3769c56eb981d98d7d7d1055b872b004cdbc9b85713d2447e009b9b61f53357c45d8a116c19aa730af3541e94af1fe4336303

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.12.0
+
+- [breaking change] Plugin interface is changed again
+- Fix bugs in a case that multiple YAMLs with different types are specified
+- Output plugin is now ensured to yield the same metrics
+
 # v0.11.1
 
 - Add `--repeat-result` option to return the best, the worst or an average result with `--repeat-count`

data/lib/benchmark_driver/bulk_output.rb

@@ -11,18 +11,16 @@ module BenchmarkDriver
   # only `#bulk_output` that takes all inputs at once.
   class BulkOutput
     # @param [Array<BenchmarkDriver::Metric>] metrics
-    attr_writer :metrics
-
-    # @param [Array<String>] job_names
-    # @param [Array<String>] context_names
-    def initialize(job_names:, context_names:)
-      # noop
+    # @param [Array<BenchmarkDriver::Job>] jobs
+    # @param [Array<BenchmarkDriver::Context>] contexts
+    def initialize(metrics:, jobs:, contexts:)
+      @metrics = metrics
     end
 
     # The main API you need to override if you make a class inherit `BenchmarkDriver::BulkOutput`.
     # @param [Hash{ BenchmarkDriver::Job => Hash{ BenchmarkDriver::Context => { BenchmarkDriver::Metric => Float } } }] result
     # @param [Array<BenchmarkDriver::Metric>] metrics
-    def bulk_output(result:, metrics:)
+    def bulk_output(job_context_result:, metrics:)
       raise NotImplementedError.new("#{self.class} must override #bulk_output")
     end
 
@@ -31,13 +29,11 @@ module BenchmarkDriver
     end
 
     def with_benchmark(&block)
-      @result = Hash.new do |h1, job|
-        h1[job] = Hash.new do |h2, context|
-          h2[context] = {}
-        end
+      @job_context_result = Hash.new do |hash, job|
+        hash[job] = {}
       end
       result = block.call
-      bulk_output(result: @result, metrics: @metrics)
+      bulk_output(job_context_result: @job_context_result, metrics: @metrics)
       result
     end
 
@@ -53,10 +49,9 @@ module BenchmarkDriver
       block.call
     end
 
-    # @param [Float] value
-    # @param [BenchmarkDriver::Metric] metic
-    def report(value:, metric:)
-      @result[@job][@context][metric] = value
+    # @param [BenchmarkDriver::Result] result
+    def report(result)
+      @job_context_result[@job][@context] = result
     end
   end
 end

data/lib/benchmark_driver/default_job.rb

@@ -3,6 +3,7 @@ require 'benchmark_driver/struct'
 module BenchmarkDriver
   DefaultJob = ::BenchmarkDriver::Struct.new(
     :name,       # @param [String] name - This is mandatory for all runner
+    :metrics,    # @param [Array<BenchmarkDriver::Metric>] - This is mandatory for all runner too, set by job parser.
     :script,     # @param [String] benchmark
     :prelude,    # @param [String,nil] prelude (optional)
     :teardown,   # @param [String,nil] after (optional)

data/lib/benchmark_driver/default_job_parser.rb

@@ -1,14 +1,13 @@
 module BenchmarkDriver
   module DefaultJobParser
     # Build default JobParser for given job klass
-    def self.for(klass)
+    def self.for(klass:, metrics:)
       Module.new.tap do |parser|
         class << parser
           include DefaultJobParser
         end
-        parser.define_singleton_method(:job_class) do
-          klass
-        end
+        parser.define_singleton_method(:job_class) { klass }
+        parser.define_singleton_method(:job_metrics) { metrics }
       end
     end
 
@@ -21,6 +20,7 @@ module BenchmarkDriver
     # @return [Array<BenchmarkDriver::Default::Job>]
     def parse(prelude: nil, benchmark:, teardown: nil, loop_count: nil, required_ruby_version: nil)
       parse_benchmark(benchmark).each do |job|
+        job.metrics = job_metrics
         job.prelude.prepend("#{prelude}\n") if prelude
         job.teardown.prepend("#{teardown}\n") if teardown
         job.loop_count ||= loop_count

data/lib/benchmark_driver/metric.rb

@@ -4,57 +4,65 @@ require 'benchmark_driver/struct'
 module BenchmarkDriver
   # BenchmarkDriver returns benchmark results with the following nested Hash structure:
   # {
-  #   BenchmarkDriver::Job => {
-  #     BenchmarkDriver::Context => {
-  #       BenchmarkDriver::Metric => Float
-  #     }
+  #   #<BenchmarkDriver::Job> => {
+  #     #<BenchmarkDriver::Context> => #<BenchmarkDriver::Result
+  #       metrics: {
+  #         #<BenchmarkDriver::Metric> => Float
+  #       }
+  #     >
   #   }
   # }
 
-  # A kind of thing to be measured
-  Metric = ::BenchmarkDriver::Struct.new(
-    :name,          # @param [String] - Metric name or description like "Max Resident Set Size"
-    :unit,          # @param [String] - A unit like "MiB"
-    :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
-    :worse_word,    # @param [String] - A label shown when the value is worse.
-    defaults: { larger_better: true, worse_word: 'slower' },
+  # Holding identifier of measured workload
+  Job = ::BenchmarkDriver::Struct.new(
+    :name, # @param [String] - Name of the benchmark task
   )
 
-  # Benchmark conditions used to measure a metric
+  # Benchmark conditions that can be known before running benchmark
   Context = ::BenchmarkDriver::Struct.new(
     :name,        # @param [String] - Name of the context
     :executable,  # @param [BenchmarkDriver::Config::Executable] - Measured Ruby executable
     :gems,        # @param [Hash{ String => String,nil }] - Gem -> version pairs used for the benchmark
     :prelude,     # @param [String,nil] - Context specific setup script (optional)
+    defaults: { gems: {} },
+  )
+
+  # Everything that can be known after running benchmark
+  Result = ::BenchmarkDriver::Struct.new(
+    :values,      # @param [Hash{ BenchmarkDriver::Metric => Float }] - Main benchmark results
     :duration,    # @param [Float,nil] - Time taken to run the benchmark job (optional)
     :loop_count,  # @param [Integer,nil] - Times to run the benchmark job (optional)
     :environment, # @param [Hash] - Any other key -> value pairs to express the benchmark context
-    defaults: { gems: {}, environment: {} },
+    defaults: { environment: {} },
   )
 
-  # Holding identifier of measured workload
-  Job = ::BenchmarkDriver::Struct.new(
-    :name, # @param [String] - Name of the benchmark task
+  # A kind of thing to be measured
+  Metric = ::BenchmarkDriver::Struct.new(
+    :name,          # @param [String] - Metric name or description like "Max Resident Set Size"
+    :unit,          # @param [String] - A unit like "MiB"
+    :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+    :worse_word,    # @param [String] - A label shown when the value is worse.
+    defaults: { larger_better: true, worse_word: 'slower' },
   )
 
-  #=[RubyBench mapping]=================================|
+  #=[RubyBench mapping]=======================================|
   #
   # BenchmarkRun:
-  #   result              -> { context.name => value }  | { "default"=>"44.666666666666664", "default_jit"=>"59.333333333333336" }
-  #   environment         -> context                    | "---\nRuby version: 'ruby 2.6.0dev (2018-05-14 trunk 63417) [x86_64-linux]\n\n'\nChecksum: '59662'\n"
-  #   initiator           -> (not supported)            | #<Commit sha1: "6f0de6ed9...", message: "error.c: check redefined ...", url: "https://github.com/tgxworld/ruby/commit/6f0de6ed98...", repo_id: 6>
+  #   result              -> { context.name => result.value } | { "default"=>"44.666666666666664", "default_jit"=>"59.333333333333336" }
+  #   environment         -> result.environment               | "---\nRuby version: 'ruby 2.6.0dev (2018-05-14 trunk 63417) [x86_64-linux]\n\n'\nChecksum: '59662'\n"
+  #   initiator           -> (not supported)                  | #<Commit sha1: "6f0de6ed9...", message: "error.c: check redefined ...", url: "https://github.com/tgxworld/ruby/commit/6f0de6ed98...", repo_id: 6>
   #
   #   BenchmarkType:
-  #     category          -> job.name                   | "app_erb", "Optcarrot Lan_Master.nes"
-  #     script_url        -> (not supported)            | "https://raw.githubusercontent.com/mame/optcarrot/master/lib/optcarrot/nes.rb"
-  #     repo              -> (not supported)            | #<Repo name: "ruby", url: "https://github.com/tgxworld/ruby">
-  #     repo.organization -> (not supported)            | #<Organization name: "ruby", url: "https://github.com/tgxworld/">
+  #     category          -> job.name                         | "app_erb", "Optcarrot Lan_Master.nes"
+  #     script_url        -> (not supported)                  | "https://raw.githubusercontent.com/mame/optcarrot/master/lib/optcarrot/nes.rb"
+  #     repo              -> (not supported)                  | #<Repo name: "ruby", url: "https://github.com/tgxworld/ruby">
+  #     repo.organization -> (not supported)                  | #<Organization name: "ruby", url: "https://github.com/tgxworld/">
   #
   #   BenchmarkResultType:
-  #     name              -> metric.name                | "Number of frames"
-  #     unit              -> metric.unit                | "fps"
+  #     name              -> metric.name                      | "Number of frames"
+  #     unit              -> metric.unit                      | "fps"
   #
-  #=====================================================|
+  #===========================================================|
 
   #----
   # legacy

data/lib/benchmark_driver/output.rb

@@ -8,12 +8,12 @@ module BenchmarkDriver
   #   metrics=
   #   with_warmup
   #     with_job(name:)
-  #       with_context(name:, executable:, duration: nil, loop_count: nil)
-  #         report(value:, metric:)
+  #       with_context(name:, executable:)
+  #         report(values:, duration: nil, loop_count: nil, environment: {})
   #   with_benchmark
   #     with_job(name:)
-  #       with_context(name:, executable:, duration: nil, loop_count: nil)
-  #         report(value:, metric:)
+  #       with_context(name:, executable:)
+  #         report(values:, duration: nil, loop_count: nil, environment: {})
   class Output
     require 'benchmark_driver/output/compare'
     require 'benchmark_driver/output/markdown'
@@ -24,9 +24,10 @@ module BenchmarkDriver
     # Create `BenchmarkDriver::Output::Foo` as benchmark_dirver-output-foo.gem and specify `-o foo`.
     #
     # @param [String] type
-    # @param [Array<String>] job_names
-    # @param [Array<String>] context_names
-    def initialize(type:, job_names:, context_names:)
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    # @param [Array<BenchmarkDriver::Job>] jobs
+    # @param [Array<BenchmarkDriver::Context>] contexts
+    def initialize(type:, metrics:, jobs:, contexts:)
       if type.include?(':')
         raise ArgumentError.new("Output type '#{type}' cannot contain ':'")
       end
@@ -35,8 +36,9 @@ module BenchmarkDriver
       camelized = type.split('_').map(&:capitalize).join
 
       @output = ::BenchmarkDriver::Output.const_get(camelized, false).new(
-        job_names: job_names,
-        context_names: context_names,
+        metrics: metrics,
+        jobs: jobs,
+        contexts: contexts,
       )
     end
 
@@ -65,10 +67,8 @@ module BenchmarkDriver
     # @param [BenchmarkDriver::Config::Executable] executable
     # @param [Float] duration
     # @param [Integer] loop_count
-    def with_context(name:, executable:, duration: nil, loop_count: nil, environment: {}, &block)
-      context = BenchmarkDriver::Context.new(
-        name: name, executable: executable, duration: duration, loop_count: loop_count, environment: environment,
-      )
+    def with_context(name:, executable:, &block)
+      context = BenchmarkDriver::Context.new(name: name, executable: executable)
       @output.with_context(context) do
         block.call
       end
@@ -76,8 +76,14 @@ module BenchmarkDriver
 
     # @param [Float] value
     # @param [BenchmarkDriver::Metric] metic
-    def report(value:, metric:)
-      @output.report(value: value, metric: metric)
+    def report(values:, duration: nil, loop_count: nil, environment: {})
+      result = BenchmarkDriver::Result.new(
+        values: values,
+        duration: duration,
+        loop_count: loop_count,
+        environment: environment,
+      )
+      @output.report(result)
     end
   end
 end

data/lib/benchmark_driver/output/compare.rb

@@ -3,14 +3,13 @@ class BenchmarkDriver::Output::Compare
   NAME_LENGTH = 20
 
   # @param [Array<BenchmarkDriver::Metric>] metrics
-  attr_writer :metrics
-
-  # @param [Array<String>] job_names
-  # @param [Array<String>] context_names
-  def initialize(job_names:, context_names:)
-    @job_names = job_names
-    @context_names = context_names
-    @name_length = [job_names.map(&:length).max, NAME_LENGTH].max
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @job_names = jobs.map(&:name)
+    @context_names = contexts.map(&:name)
+    @name_length = [@job_names.map(&:length).max, NAME_LENGTH].max
   end
 
   def with_warmup(&block)
@@ -22,8 +21,8 @@ class BenchmarkDriver::Output::Compare
   end
 
   def with_benchmark(&block)
-    @job_context_values = Hash.new do |h1, k1|
-      h1[k1] = Hash.new { |h2, k2| h2[k2] = [] }
+    @job_context_result = Hash.new do |hash, job|
+      hash[job] = {}
     end
 
     without_stdout_buffering do
@@ -55,14 +54,15 @@ class BenchmarkDriver::Output::Compare
       $stdout.print("%#{@name_length}s" % name)
     end
     @job = name
+    @job_results = []
     @job_contexts = []
     block.call
   ensure
     $stdout.print(@metrics.first.unit)
-    loop_count = @job_contexts.first.loop_count
-    if loop_count && @job_contexts.all? { |c| c.loop_count == loop_count }
+    loop_count = @job_results.first.loop_count
+    if loop_count && @job_results.all? { |r| r.loop_count == loop_count }
       $stdout.print(" - #{humanize(loop_count)} times")
-      if @job_contexts.all? { |context| !context.duration.nil? }
+      if @job_results.all? { |result| !result.duration.nil? }
         $stdout.print(" in")
         show_durations
       end
@@ -77,28 +77,28 @@ class BenchmarkDriver::Output::Compare
     block.call
   end
 
-  # @param [Float] value
-  # @param [BenchmarkDriver::Metric] metic
-  def report(value:, metric:)
-    if defined?(@job_context_values)
-      @job_context_values[@job][@context] << value
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    @job_results << result
+    if defined?(@job_context_result)
+      @job_context_result[@job][@context] = result
     end
 
-    $stdout.print("#{humanize(value, [10, @context.name.length].max)} ")
+    $stdout.print("#{humanize(result.values.values.first, [10, @context.name.length].max)} ")
   end
 
   private
 
   def show_durations
-    @job_contexts.each do |context|
-      $stdout.print(' %3.6fs' % context.duration)
+    @job_results.each do |result|
+      $stdout.print(' %3.6fs' % result.duration)
     end
 
     # Show pretty seconds / clocks too. As it takes long width, it's shown only with a single executable.
-    if @job_contexts.size == 1
-      context = @job_contexts.first
-      sec = context.duration
-      iter = context.loop_count
+    if @job_results.size == 1
+      result = @job_results.first
+      sec = result.duration
+      iter = result.loop_count
       if File.exist?('/proc/cpuinfo') && (clks = estimate_clock(sec, iter)) < 1_000
         $stdout.print(" (#{pretty_sec(sec, iter)}/i, #{clks}clocks/i)")
       else
@@ -160,8 +160,8 @@ class BenchmarkDriver::Output::Compare
 
   def compare_jobs
     $stdout.puts "\nComparison:"
-    results = @job_context_values.flat_map do |job, context_values|
-      context_values.map { |context, values| Result.new(job: job, value: values.first, executable: context.executable) }
+    results = @job_context_result.flat_map do |job, context_result|
+      context_result.map { |context, result| Result.new(job: job, value: result.values.values.first, executable: context.executable) }
     end
     show_results(results, show_executable: false)
   end
@@ -169,10 +169,10 @@ class BenchmarkDriver::Output::Compare
   def compare_executables
     $stdout.puts "\nComparison:"
 
-    @job_context_values.each do |job, context_values|
+    @job_context_result.each do |job, context_result|
       $stdout.puts("%#{@name_length + 2 + 11}s" % job)
-      results = context_values.flat_map do |context, values|
-        values.map { |value| Result.new(job: job, value: value, executable: context.executable) }
+      results = context_result.flat_map do |context, result|
+        result.values.values.map { |value| Result.new(job: job, value: value, executable: context.executable) }
       end
       show_results(results, show_executable: true)
     end

data/lib/benchmark_driver/output/markdown.rb

@@ -2,13 +2,12 @@ class BenchmarkDriver::Output::Markdown
   NAME_LENGTH = 8
 
   # @param [Array<BenchmarkDriver::Metric>] metrics
-  attr_writer :metrics
-
-  # @param [Array<String>] job_names
-  # @param [Array<String>] context_names
-  def initialize(job_names:, context_names:)
-    @context_names = context_names
-    @name_length = job_names.map(&:size).max
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @context_names = contexts.map(&:name)
+    @name_length = jobs.map(&:name).map(&:size).max
   end
 
   def with_warmup(&block)
@@ -63,11 +62,10 @@ class BenchmarkDriver::Output::Markdown
     block.call
   end
 
-  # @param [Floa] value
-  # @param [BenchmarkDriver::Metric] metic
-  def report(value:, metric:)
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
     if @with_benchmark
-      $stdout.print("|%#{NAME_LENGTH}s" % humanize(value))
+      $stdout.print("|%#{NAME_LENGTH}s" % humanize(result.values.fetch(@metrics.first)))
     else
       $stdout.print '.'
     end

data/lib/benchmark_driver/output/record.rb

@@ -1,14 +1,13 @@
 class BenchmarkDriver::Output::Record
   # @param [Array<BenchmarkDriver::Metric>] metrics
-  attr_writer :metrics
-
-  # @param [Array<String>] job_names
-  # @param [Array<String>] context_names
-  def initialize(job_names:, context_names:)
-    @job_warmup_context_metric_value = Hash.new do |h1, k1|
-      h1[k1] = Hash.new do |h2, k2|
-        h2[k2] = Hash.new do |h3, k3|
-          h3[k3] = {}
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @job_warmup_context_result = Hash.new do |h1, job|
+      h1[job] = Hash.new do |h2, warmup|
+        h2[warmup] = Hash.new do |h3, context|
+          h3[context] = {}
         end
       end
     end
@@ -33,7 +32,7 @@ class BenchmarkDriver::Output::Record
 
   # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
-    @job = job.name
+    @job = job
     block.call
   end
 
@@ -43,11 +42,10 @@ class BenchmarkDriver::Output::Record
     block.call
   end
 
-  # @param [Float] value
-  # @param [BenchmarkDriver::Metric] metic
-  def report(value:, metric:)
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
     $stdout.print '.'
-    @job_warmup_context_metric_value[@job][!@with_benchmark][@context][metric] = value
+    @job_warmup_context_result[@job][!@with_benchmark][@context] = result
   end
 
   private
@@ -56,7 +54,7 @@ class BenchmarkDriver::Output::Record
     jobs = @benchmark_metrics
     yaml = {
       'type' => 'recorded',
-      'job_warmup_context_metric_value' => @job_warmup_context_metric_value,
+      'job_warmup_context_result' => @job_warmup_context_result,
       'metrics' => @metrics,
     }.to_yaml
     File.write('benchmark_driver.record.yml', yaml)

data/lib/benchmark_driver/output/simple.rb

@@ -2,13 +2,12 @@ class BenchmarkDriver::Output::Simple
   NAME_LENGTH = 8
 
   # @param [Array<BenchmarkDriver::Metric>] metrics
-  attr_writer :metrics
-
-  # @param [Array<String>] job_names
-  # @param [Array<String>] context_names
-  def initialize(job_names:, context_names:)
-    @context_names = context_names
-    @name_length = job_names.map(&:size).max
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @context_names = contexts.map(&:name)
+    @name_length = jobs.map(&:name).map(&:size).max
   end
 
   def with_warmup(&block)
@@ -58,11 +57,10 @@ class BenchmarkDriver::Output::Simple
     block.call
   end
 
-  # @param [Float] value
-  # @param [BenchmarkDriver::Metric] metic
-  def report(value:, metric:)
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
     if @with_benchmark
-      $stdout.print("%#{NAME_LENGTH}s  " % humanize(value))
+      $stdout.print("%#{NAME_LENGTH}s  " % humanize(result.values.fetch(@metrics.first)))
     else
       $stdout.print '.'
     end

data/lib/benchmark_driver/runner.rb

@@ -28,15 +28,20 @@ module BenchmarkDriver
         verbose: config.verbose,
       )
 
-      jobs.group_by(&:class).each do |klass, jobs_group|
-        runner = runner_for(klass)
-        output = Output.new(
-          type: config.output_type,
-          job_names: jobs.map(&:name),
-          context_names: config.executables.map(&:name),
-        )
-        with_clean_env do
-          runner.new(config: runner_config, output: output).run(jobs)
+      jobs.group_by(&:class).each do |klass, klass_jobs|
+        klass_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
+          runner = runner_for(klass)
+          output = Output.new(
+            type: config.output_type,
+            metrics: metrics,
+            jobs: jobs.map { |job| BenchmarkDriver::Job.new(name: job.name) },
+            contexts: config.executables.map { |exec|
+              BenchmarkDriver::Context.new(name: exec.name, executable: exec)
+            },
+          )
+          with_clean_env do
+            runner.new(config: runner_config, output: output).run(metrics_jobs)
+          end
         end
       end
     end

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -9,9 +9,9 @@ class BenchmarkDriver::Runner::CommandStdout
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = ::BenchmarkDriver::Struct.new(
     :name,              # @param [String] name - This is mandatory for all runner
+    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
     :command,           # @param [Array<String>]
     :working_directory, # @param [String,NilClass]
-    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
     :stdout_to_metrics, # @param [String]
   )
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
@@ -56,7 +56,6 @@ class BenchmarkDriver::Runner::CommandStdout
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
     metric = jobs.first.metrics.first
-    @output.metrics = [metric]
 
     @output.with_benchmark do
       jobs.each do |job|
@@ -73,7 +72,7 @@ class BenchmarkDriver::Runner::CommandStdout
             end
 
             @output.with_context(name: exec.name, executable: exec) do
-              @output.report(value: value, metric: metric)
+              @output.report(values: { metric => value })
             end
           end
         end

data/lib/benchmark_driver/runner/ips.rb

@@ -7,12 +7,12 @@ require 'shellwords'
 
 # Show iteration per second.
 class BenchmarkDriver::Runner::Ips
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
-  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-
-  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
@@ -24,8 +24,6 @@ class BenchmarkDriver::Runner::Ips
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
-    @output.metrics = [metric]
-
     if jobs.any? { |job| job.loop_count.nil? }
       @output.with_warmup do
         jobs = jobs.map do |job|
@@ -36,8 +34,8 @@ class BenchmarkDriver::Runner::Ips
             duration, loop_count = run_warmup(job, exec: executable)
             value, duration = value_duration(duration: duration, loop_count: loop_count)
 
-            @output.with_context(name: executable.name, executable: executable, duration: duration, loop_count: loop_count) do
-              @output.report(value: value, metric: metric)
+            @output.with_context(name: executable.name, executable: executable) do
+              @output.report(values: { metric => value }, duration: duration, loop_count: loop_count)
             end
 
             loop_count = (loop_count.to_f * @config.run_duration / duration).floor
@@ -55,8 +53,8 @@ class BenchmarkDriver::Runner::Ips
             value, duration = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               run_benchmark(job, exec: exec)
             end
-            @output.with_context(name: exec.name, executable: exec, duration: duration) do
-              @output.report(value: value, metric: metric)
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(values: { metric => value }, duration: duration, loop_count: job.loop_count)
             end
           end
         end

data/lib/benchmark_driver/runner/memory.rb

@@ -7,15 +7,15 @@ require 'shellwords'
 
 # Max resident set size
 class BenchmarkDriver::Runner::Memory
-  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
-  Job = Class.new(BenchmarkDriver::DefaultJob)
-  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
-  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-
   METRIC = BenchmarkDriver::Metric.new(
     name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger',
   )
 
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
   def initialize(config:, output:)
@@ -31,8 +31,6 @@ class BenchmarkDriver::Runner::Memory
       raise "memory output is not supported for '#{Etc.uname[:sysname]}' for now"
     end
 
-    @output.metrics = [METRIC]
-
     if jobs.any? { |job| job.loop_count.nil? }
       jobs = jobs.map do |job|
         job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))
@@ -46,8 +44,8 @@ class BenchmarkDriver::Runner::Memory
             value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
               run_benchmark(job, exec: exec)
             end
-            @output.with_context(name: exec.name, executable: exec, loop_count: job.loop_count) do
-              @output.report(value: value, metric: METRIC)
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(values: { METRIC => value }, loop_count: job.loop_count)
             end
           end
         end

data/lib/benchmark_driver/runner/once.rb

@@ -7,12 +7,12 @@ require 'shellwords'
 
 # Run only once, for testing
 class BenchmarkDriver::Runner::Once
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
-  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-
-  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
@@ -24,8 +24,6 @@ class BenchmarkDriver::Runner::Once
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
-    @output.metrics = [METRIC]
-
     jobs = jobs.map do |job|
       Job.new(job.to_h.merge(loop_count: 1)) # to show this on output
     end
@@ -35,8 +33,8 @@ class BenchmarkDriver::Runner::Once
         @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
             duration = run_benchmark(job, exec: exec) # no repeat support
-            @output.with_context(name: exec.name, executable: exec, duration: duration, loop_count: 1) do
-              @output.report(value: 1.0 / duration, metric: METRIC)
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(values: { METRIC => 1.0 / duration }, duration: duration, loop_count: 1)
             end
           end
         end

data/lib/benchmark_driver/runner/recorded.rb

@@ -8,20 +8,20 @@ class BenchmarkDriver::Runner::Recorded
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = ::BenchmarkDriver::Struct.new(
     :name,              # @param [String] name - This is mandatory for all runner
+    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
     :warmup_results,    # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
     :benchmark_results, # @param [Hash{ BenchmarkDriver::Context => Array<BenchmarkDriver::Metric> } }]
-    :metrics,           # @param [Array<BenchmarkDriver::Metric>]
   )
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   class << JobParser = Module.new
-    # @param [Hash{ String => Hash{ TrueClass,FalseClass => Hash{ BenchmarkDriver::Context => Hash{ BenchmarkDriver::Metric => Float } } } }] job_warmup_context_metric_value
+    # @param [Hash{ BenchmarkDriver::Job => Hash{ TrueClass,FalseClass => Hash{ BenchmarkDriver::Context => BenchmarkDriver::Result } } }] job_warmup_context_result
     # @param [BenchmarkDriver::Metrics::Type] metrics
-    def parse(job_warmup_context_metric_value:, metrics:)
-      job_warmup_context_metric_value.map do |job_name, warmup_context_values|
+    def parse(job_warmup_context_result:, metrics:)
+      job_warmup_context_result.map do |job, warmup_context_result|
         Job.new(
-          name: job_name,
-          warmup_results: warmup_context_values.fetch(true, {}),
-          benchmark_results: warmup_context_values.fetch(false, {}),
+          name: job.name,
+          warmup_results: warmup_context_result.fetch(true, {}),
+          benchmark_results: warmup_context_result.fetch(false, {}),
           metrics: metrics,
         )
       end
@@ -38,8 +38,6 @@ class BenchmarkDriver::Runner::Recorded
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Runner::Recorded::Job>] record
   def run(records)
-    @output.metrics = records.first.metrics
-
     records.each do |record|
       unless record.warmup_results.empty?
         # TODO:
@@ -49,16 +47,14 @@ class BenchmarkDriver::Runner::Recorded
     @output.with_benchmark do
       records.each do |record|
         @output.with_job(name: record.name) do
-          record.benchmark_results.each do |context, metric_values|
-            @output.with_context(
-              name: context.name,
-              executable: context.executable,
-              duration: context.duration,
-              loop_count: context.loop_count,
-            ) do
-              metric_values.each do |metric, value|
-                @output.report(value: value, metric: metric)
-              end
+          record.benchmark_results.each do |context, result|
+            @output.with_context(name: context.name, executable: context.executable) do
+              @output.report(
+                values: result.values,
+                duration: result.duration,
+                loop_count: result.loop_count,
+                environment: result.environment,
+              )
             end
           end
         end

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -9,9 +9,9 @@ class BenchmarkDriver::Runner::RubyStdout
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = ::BenchmarkDriver::Struct.new(
     :name,                   # @param [String] name - This is mandatory for all runner
+    :metrics,                # @param [Array<BenchmarkDriver::Metric>]
     :command,                # @param [Array<String>]
     :working_directory,      # @param [String,NilClass]
-    :metrics,                # @param [Array<BenchmarkDriver::Metric>]
     :value_from_stdout,      # @param [String]
     :environment_from_stdout # @param [Hash{ String => String }]
   )
@@ -73,7 +73,6 @@ class BenchmarkDriver::Runner::RubyStdout
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
     metric = jobs.first.metrics.first
-    @output.metrics = [metric]
 
     @output.with_benchmark do
       jobs.each do |job|
@@ -92,8 +91,8 @@ class BenchmarkDriver::Runner::RubyStdout
               [script.value, script.environment]
             end
 
-            @output.with_context(name: exec.name, executable: exec, environment: environment) do
-              @output.report(value: value, metric: metric)
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(values: { metric => value }, environment: environment)
             end
           end
         end
@@ -127,20 +126,6 @@ class BenchmarkDriver::Runner::RubyStdout
     stdout
   end
 
-  # Return multiple times and return the best metrics
-  def with_repeat(metric, &block)
-    value_environments = @config.repeat_count.times.map do
-      block.call
-    end
-    value_environments.sort_by do |value, _|
-      if metric.larger_better
-        value
-      else
-        -value
-      end
-    end.last
-  end
-
   StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :value_from_stdout, :environment_from_stdout) do
     def value
       value = eval(value_from_stdout, binding)

data/lib/benchmark_driver/runner/time.rb

@@ -1,12 +1,12 @@
 require 'benchmark_driver/runner/ips'
 
 class BenchmarkDriver::Runner::Time < BenchmarkDriver::Runner::Ips
+  METRIC = BenchmarkDriver::Metric.new(name: 'Execution time', unit: 's', larger_better: false)
+
   # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
-  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-
-  METRIC = BenchmarkDriver::Metric.new(name: 'Execution time', unit: 's', larger_better: false)
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
 
   # Overriding BenchmarkDriver::Runner::Ips#metric
   def metric

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.11.1'
+  VERSION = '0.12.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-30 00:00:00.000000000 Z
+date: 2018-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler