benchmark_driver 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc2f6826606613671a2e6b8d0384421924bd9487
-  data.tar.gz: b310edeb8fa144e6eb6569b670ae7291e5a77a70
+  metadata.gz: 8d80a15877a535548463502efb2a1a39b708c3e7
+  data.tar.gz: 8a569023b7cde0b811f4bfc96c13587e863a7e1d
 SHA512:
-  metadata.gz: a121e71123b95d5dba0f130f2efa65aaeb9adb42c4e9046703c3afcbd3726a698b7f33e870d3c873a5eda7bbaea20bce2061f87a085118a9d8a6b36b183c6e1a
-  data.tar.gz: 942d159581bf537e62c15b46dba3769c56eb981d98d7d7d1055b872b004cdbc9b85713d2447e009b9b61f53357c45d8a116c19aa730af3541e94af1fe4336303
+  metadata.gz: 2d44bf5752e35b615c219916d45828e0e535729b2290ca590d6c3d16b4ed8928afb2c66cdac65609af78f85792374a1715089efad2b6921f1616bcc43c74b824
+  data.tar.gz: a8744d5d19ac34aaf2951a6f7704a2fddc6f4eeeb7a9e079808a59469db16723b03aeff09a9d79b6563f8bda92795e8f99f1539225ffe28bc1b342e554443143

data/.travis.yml

@@ -8,6 +8,9 @@ cache: bundler
 branches:
   only:
     - master
-before_install: gem install bundler -v 1.15.4
+before_install:
+  - gem install bundler -v 1.15.4
+  - gem install haml -v 4.0.7
+  - gem install haml -v 5.0.4
 script:
   - RSPEC_RETRIES=3 VERBOSE=1 bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v0.13.0
+
+- Add support for benchmark with different versions of gems and preludes
+- [breaking change] Runner plugin interface is sightly changed
+
 # v0.12.0
 
 - [breaking change] Plugin interface is changed again

data/lib/benchmark_driver/bulk_output.rb

@@ -18,7 +18,7 @@ module BenchmarkDriver
     end
 
     # The main API you need to override if you make a class inherit `BenchmarkDriver::BulkOutput`.
-    # @param [Hash{ BenchmarkDriver::Job => Hash{ BenchmarkDriver::Context => { BenchmarkDriver::Metric => Float } } }] result
+    # @param [Hash{ BenchmarkDriver::Job => Hash{ BenchmarkDriver::Context => BenchmarkDriver::Result } }] job_context_result
     # @param [Array<BenchmarkDriver::Metric>] metrics
     def bulk_output(job_context_result:, metrics:)
       raise NotImplementedError.new("#{self.class} must override #bulk_output")

data/lib/benchmark_driver/config.rb

@@ -25,7 +25,6 @@ module BenchmarkDriver
 
   # Subset of FullConfig passed to JobRunner
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
-    :executables,   # @param [Array<BenchmarkDriver::Config::Executable>]
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
     :run_duration,  # @param [Float]

data/lib/benchmark_driver/default_job.rb

@@ -4,6 +4,7 @@ module BenchmarkDriver
   DefaultJob = ::BenchmarkDriver::Struct.new(
     :name,       # @param [String] name - This is mandatory for all runner
     :metrics,    # @param [Array<BenchmarkDriver::Metric>] - This is mandatory for all runner too, set by job parser.
+    :contexts,   # @param [Array<BenchmarkDriver::Context>] - This is optional parameter for runners.
     :script,     # @param [String] benchmark
     :prelude,    # @param [String,nil] prelude (optional)
     :teardown,   # @param [String,nil] after (optional)
@@ -11,17 +12,17 @@ module BenchmarkDriver
     :required_ruby_version, # @param [String,nil] required_ruby_version (optional)
     defaults: { prelude: '', teardown: '' },
   ) do
-    def runnable_execs(executables)
+    def runnable_contexts(contexts)
       if required_ruby_version
-        executables.select do |executable|
-          Gem::Version.new(executable.version) >= Gem::Version.new(required_ruby_version)
+        contexts.select do |context|
+          Gem::Version.new(context.executable.version) >= Gem::Version.new(required_ruby_version)
         end.tap do |result|
           if result.empty?
             raise "No Ruby executables conforming required_ruby_version (#{required_ruby_version}) are specified"
           end
         end
       else
-        executables
+        contexts
       end
     end
   end

data/lib/benchmark_driver/default_job_parser.rb

@@ -18,8 +18,9 @@ module BenchmarkDriver
     # @param [Integer] loop_count
     # @param [String] required_ruby_version
     # @return [Array<BenchmarkDriver::Default::Job>]
-    def parse(prelude: nil, benchmark:, teardown: nil, loop_count: nil, required_ruby_version: nil)
+    def parse(contexts: [], prelude: nil, benchmark:, teardown: nil, loop_count: nil, required_ruby_version: nil)
       parse_benchmark(benchmark).each do |job|
+        job.contexts = parse_contexts(contexts)
         job.metrics = job_metrics
         job.prelude.prepend("#{prelude}\n") if prelude
         job.teardown.prepend("#{teardown}\n") if teardown
@@ -66,5 +67,26 @@ module BenchmarkDriver
     def job_class
       raise NotImplementedError # override this
     end
+
+    def parse_contexts(contexts)
+      if contexts.is_a?(Array)
+        contexts.map { |context| parse_context(context) }
+      else
+        raise ArgumentError.new("contexts must be Array, but got: #{contexts.inspect}")
+      end
+    end
+
+    def parse_context(name: nil, prelude: '', gems: {})
+      # Version lock with `Bundler.require`-like behavior (no `require: xxx` support yet)
+      gems.each do |gem, version|
+        prelude = "gem '#{gem}', '#{version}'\nrequire '#{gem}'\n#{prelude}"
+      end
+
+      BenchmarkDriver::Context.new(
+        name: name,
+        gems: gems,
+        prelude: prelude,
+      )
+    end
   end
 end

data/lib/benchmark_driver/metric.rb

@@ -24,7 +24,7 @@ module BenchmarkDriver
     :executable,  # @param [BenchmarkDriver::Config::Executable] - Measured Ruby executable
     :gems,        # @param [Hash{ String => String,nil }] - Gem -> version pairs used for the benchmark
     :prelude,     # @param [String,nil] - Context specific setup script (optional)
-    defaults: { gems: {} },
+    defaults: { prelude: '', gems: {} },
   )
 
   # Everything that can be known after running benchmark

data/lib/benchmark_driver/output.rb

@@ -8,11 +8,11 @@ module BenchmarkDriver
   #   metrics=
   #   with_warmup
   #     with_job(name:)
-  #       with_context(name:, executable:)
+  #       with_context(name:, executable:, gems:)
   #         report(values:, duration: nil, loop_count: nil, environment: {})
   #   with_benchmark
   #     with_job(name:)
-  #       with_context(name:, executable:)
+  #       with_context(name:, executable:, gems:)
   #         report(values:, duration: nil, loop_count: nil, environment: {})
   class Output
     require 'benchmark_driver/output/compare'
@@ -65,10 +65,9 @@ module BenchmarkDriver
 
     # @param [String] name
     # @param [BenchmarkDriver::Config::Executable] executable
-    # @param [Float] duration
-    # @param [Integer] loop_count
-    def with_context(name:, executable:, &block)
-      context = BenchmarkDriver::Context.new(name: name, executable: executable)
+    # @param [Hash{ String => String}] gems
+    def with_context(name:, executable:, gems: {}, &block)
+      context = BenchmarkDriver::Context.new(name: name, executable: executable, gems: gems)
       @output.with_context(context) do
         block.call
       end

data/lib/benchmark_driver/output/compare.rb

@@ -25,7 +25,7 @@ class BenchmarkDriver::Output::Compare
       hash[job] = {}
     end
 
-    without_stdout_buffering do
+    result = without_stdout_buffering do
       $stdout.puts 'Calculating -------------------------------------'
       if @context_names.size > 1
         $stdout.print(' ' * @name_length)
@@ -37,12 +37,12 @@ class BenchmarkDriver::Output::Compare
 
       block.call
     end
-  ensure
     if @context_names.size > 1
       compare_executables
     elsif @job_names.size > 1
       compare_jobs
     end
+    result
   end
 
   # @param [BenchmarkDriver::Job] job
@@ -56,8 +56,7 @@ class BenchmarkDriver::Output::Compare
     @job = name
     @job_results = []
     @job_contexts = []
-    block.call
-  ensure
+    result = block.call
     $stdout.print(@metrics.first.unit)
     loop_count = @job_results.first.loop_count
     if loop_count && @job_results.all? { |r| r.loop_count == loop_count }
@@ -68,6 +67,7 @@ class BenchmarkDriver::Output::Compare
       end
     end
     $stdout.puts
+    result
   end
 
   # @param [BenchmarkDriver::Context] context
@@ -161,9 +161,9 @@ class BenchmarkDriver::Output::Compare
   def compare_jobs
     $stdout.puts "\nComparison:"
     results = @job_context_result.flat_map do |job, context_result|
-      context_result.map { |context, result| Result.new(job: job, value: result.values.values.first, executable: context.executable) }
+      context_result.map { |context, result| Result.new(job: job, value: result.values.values.first, context: context) }
     end
-    show_results(results, show_executable: false)
+    show_results(results, show_context: false)
   end
 
   def compare_executables
@@ -172,15 +172,15 @@ class BenchmarkDriver::Output::Compare
     @job_context_result.each do |job, context_result|
       $stdout.puts("%#{@name_length + 2 + 11}s" % job)
       results = context_result.flat_map do |context, result|
-        result.values.values.map { |value| Result.new(job: job, value: value, executable: context.executable) }
+        result.values.values.map { |value| Result.new(job: job, value: value, context: context) }
       end
-      show_results(results, show_executable: true)
+      show_results(results, show_context: true)
     end
   end
 
   # @param [Array<BenchmarkDriver::Output::Compare::Result>] results
-  # @param [TrueClass,FalseClass] show_executable
-  def show_results(results, show_executable:)
+  # @param [TrueClass,FalseClass] show_context
+  def show_results(results, show_context:)
     results = results.sort_by do |result|
       if @metrics.first.larger_better
         -result.value
@@ -199,8 +199,8 @@ class BenchmarkDriver::Output::Compare
         end
         slower = "- %.2fx  #{@metrics.first.worse_word}" % ratio
       end
-      if show_executable
-        name = result.executable.name
+      if show_context
+        name = result.context.name
       else
         name = result.job
       end
@@ -209,5 +209,5 @@ class BenchmarkDriver::Output::Compare
     $stdout.puts
   end
 
-  Result = ::BenchmarkDriver::Struct.new(:job, :value, :executable)
+  Result = ::BenchmarkDriver::Struct.new(:job, :value, :context)
 end

data/lib/benchmark_driver/output/simple.rb

@@ -1,5 +1,5 @@
 class BenchmarkDriver::Output::Simple
-  NAME_LENGTH = 8
+  NAME_LENGTH = 10
 
   # @param [Array<BenchmarkDriver::Metric>] metrics
   # @param [Array<BenchmarkDriver::Job>] jobs
@@ -28,7 +28,7 @@ class BenchmarkDriver::Output::Simple
       # Show executable names
       if @context_names.size > 1
         $stdout.print("#{' ' * @name_length}  ")
-        @context_name.each do |context_name|
+        @context_names.each do |context_name|
           $stdout.print("%#{NAME_LENGTH}s  " % context_name)
         end
         $stdout.puts
@@ -36,7 +36,7 @@ class BenchmarkDriver::Output::Simple
 
       block.call
     end
-  rescue
+  ensure
     @with_benchmark = false
   end
 

data/lib/benchmark_driver/runner.rb

@@ -21,26 +21,26 @@ module BenchmarkDriver
       end
 
       runner_config = Config::RunnerConfig.new(
-        executables: config.executables,
         repeat_count: config.repeat_count,
         repeat_result: config.repeat_result,
         run_duration: config.run_duration,
         verbose: config.verbose,
       )
 
-      jobs.group_by(&:class).each do |klass, klass_jobs|
-        klass_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
-          runner = runner_for(klass)
-          output = Output.new(
-            type: config.output_type,
-            metrics: metrics,
-            jobs: jobs.map { |job| BenchmarkDriver::Job.new(name: job.name) },
-            contexts: config.executables.map { |exec|
-              BenchmarkDriver::Context.new(name: exec.name, executable: exec)
-            },
-          )
-          with_clean_env do
-            runner.new(config: runner_config, output: output).run(metrics_jobs)
+      jobs.group_by{ |j| j.respond_to?(:contexts) && j.contexts }.each do |contexts, contexts_jobs|
+        contexts_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
+          metrics_jobs.group_by(&:class).each do |klass, klass_jobs|
+            runner = runner_for(klass)
+            contexts = build_contexts(contexts, executables: config.executables)
+            output = Output.new(
+              type: config.output_type,
+              metrics: metrics,
+              jobs: klass_jobs.map { |job| BenchmarkDriver::Job.new(name: job.name) },
+              contexts: contexts,
+            )
+            with_clean_env do
+              runner.new(config: runner_config, output: output, contexts: contexts).run(klass_jobs)
+            end
           end
         end
       end
@@ -48,6 +48,36 @@ module BenchmarkDriver
 
     private
 
+    def build_contexts(contexts, executables:)
+      # If contexts are not specified, just use executables as contexts.
+      if !contexts.is_a?(Array) || contexts.empty?
+        return executables.map { |exec|
+          BenchmarkDriver::Context.new(name: exec.name, executable: exec)
+        }
+      end
+
+      # Create direct product of contexts
+      contexts.product(executables).map do |context, executable|
+        name = context.name
+        if name.nil?
+          # Use the first gem name and version by default
+          name = context.gems.first.join(' ')
+
+          # Append Ruby executable name if it's matrix
+          if executables.size > 1
+            name = "#{name} (#{executable.name})"
+          end
+        end
+
+        BenchmarkDriver::Context.new(
+          name: name,
+          executable: executable,
+          gems: context.gems,
+          prelude: context.prelude,
+        )
+      end
+    end
+
     # Dynamically find class (BenchmarkDriver::*::JobRunner) for plugin support
     # @param [Class] klass - BenchmarkDriver::*::Job
     # @return [Class]

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -47,9 +47,11 @@ class BenchmarkDriver::Runner::CommandStdout
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -60,7 +62,8 @@ class BenchmarkDriver::Runner::CommandStdout
     @output.with_benchmark do
       jobs.each do |job|
         @output.with_job(name: job.name) do
-          @config.executables.each do |exec|
+          @contexts.each do |context|
+            exec = context.executable
             value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: metric.larger_better) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }

data/lib/benchmark_driver/runner/ips.rb

@@ -16,9 +16,11 @@ class BenchmarkDriver::Runner::Ips
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -30,11 +32,11 @@ class BenchmarkDriver::Runner::Ips
           next job if job.loop_count # skip warmup if loop_count is set
 
           @output.with_job(name: job.name) do
-            executable = job.runnable_execs(@config.executables).first
-            duration, loop_count = run_warmup(job, exec: executable)
+            context = job.runnable_contexts(@contexts).first
+            duration, loop_count = run_warmup(job, context: context)
             value, duration = value_duration(duration: duration, loop_count: loop_count)
 
-            @output.with_context(name: executable.name, executable: executable) do
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
               @output.report(values: { metric => value }, duration: duration, loop_count: loop_count)
             end
 
@@ -48,12 +50,12 @@ class BenchmarkDriver::Runner::Ips
     @output.with_benchmark do
       jobs.each do |job|
         @output.with_job(name: job.name) do
-          job.runnable_execs(@config.executables).each do |exec|
+          job.runnable_contexts(@contexts).each do |context|
             repeat_params = { config: @config, larger_better: true, rest_on_average: :average }
             value, duration = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
-              run_benchmark(job, exec: exec)
+              run_benchmark(job, context: context)
             end
-            @output.with_context(name: exec.name, executable: exec) do
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
               @output.report(values: { metric => value }, duration: duration, loop_count: job.loop_count)
             end
           end
@@ -65,10 +67,10 @@ class BenchmarkDriver::Runner::Ips
   private
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is nil
-  # @param [BenchmarkDriver::Config::Executable] exec
-  def run_warmup(job, exec:)
+  # @param [BenchmarkDriver::Context] context
+  def run_warmup(job, context:)
     warmup = WarmupScript.new(
-      prelude:    job.prelude,
+      prelude:    "#{context.prelude}\n#{job.prelude}",
       script:     job.script,
       teardown:   job.teardown,
       loop_count: job.loop_count,
@@ -78,7 +80,7 @@ class BenchmarkDriver::Runner::Ips
 
     duration, loop_count = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(warmup.render(result: f.path)) do |path|
-        execute(*exec.command, path)
+        execute(*context.executable.command, path)
       end
       eval(f.read)
     end
@@ -87,11 +89,11 @@ class BenchmarkDriver::Runner::Ips
   end
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
-  # @param [BenchmarkDriver::Config::Executable] exec
+  # @param [BenchmarkDriver::Context] context
   # @return [BenchmarkDriver::Metrics]
-  def run_benchmark(job, exec:)
+  def run_benchmark(job, context:)
     benchmark = BenchmarkScript.new(
-      prelude:    job.prelude,
+      prelude:    "#{context.prelude}\n#{job.prelude}",
       script:     job.script,
       teardown:   job.teardown,
       loop_count: job.loop_count,
@@ -99,7 +101,7 @@ class BenchmarkDriver::Runner::Ips
 
     duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        execute(*exec.command, path)
+        execute(*context.executable.command, path)
       end
       Float(f.read)
     end

data/lib/benchmark_driver/runner/memory.rb

@@ -18,9 +18,11 @@ class BenchmarkDriver::Runner::Memory
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -40,11 +42,11 @@ class BenchmarkDriver::Runner::Memory
     @output.with_benchmark do
       jobs.each do |job|
         @output.with_job(name: job.name) do
-          job.runnable_execs(@config.executables).each do |exec|
+          job.runnable_contexts(@contexts).each do |context|
             value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
-              run_benchmark(job, exec: exec)
+              run_benchmark(job, context: context)
             end
-            @output.with_context(name: exec.name, executable: exec) do
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
               @output.report(values: { METRIC => value }, loop_count: job.loop_count)
             end
           end
@@ -56,18 +58,18 @@ class BenchmarkDriver::Runner::Memory
   private
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
-  # @param [BenchmarkDriver::Config::Executable] exec
+  # @param [BenchmarkDriver::Context] context
   # @return [BenchmarkDriver::Metrics]
-  def run_benchmark(job, exec:)
+  def run_benchmark(job, context:)
     benchmark = BenchmarkScript.new(
-      prelude:    job.prelude,
+      prelude:    "#{context.prelude}\n#{job.prelude}",
       script:     job.script,
       teardown:   job.teardown,
       loop_count: job.loop_count,
     )
 
     output = with_script(benchmark.render) do |path|
-      execute('/usr/bin/time', *exec.command, path)
+      execute('/usr/bin/time', *context.executable.command, path)
     end
 
     match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(output)

data/lib/benchmark_driver/runner/once.rb

@@ -16,9 +16,11 @@ class BenchmarkDriver::Runner::Once
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -31,9 +33,9 @@ class BenchmarkDriver::Runner::Once
     @output.with_benchmark do
       jobs.each do |job|
         @output.with_job(name: job.name) do
-          job.runnable_execs(@config.executables).each do |exec|
-            duration = run_benchmark(job, exec: exec) # no repeat support
-            @output.with_context(name: exec.name, executable: exec) do
+          job.runnable_contexts(@contexts).each do |context|
+            duration = run_benchmark(job, context: context) # no repeat support
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
               @output.report(values: { METRIC => 1.0 / duration }, duration: duration, loop_count: 1)
             end
           end
@@ -45,11 +47,11 @@ class BenchmarkDriver::Runner::Once
   private
 
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
-  # @param [BenchmarkDriver::Config::Executable] exec
+  # @param [BenchmarkDriver::Context] context
   # @return [Float] duration
-  def run_benchmark(job, exec:)
+  def run_benchmark(job, context:)
     benchmark = BenchmarkScript.new(
-      prelude:    job.prelude,
+      prelude:    "#{context.prelude}\n#{job.prelude}",
       script:     job.script,
       teardown:   job.teardown,
       loop_count: job.loop_count,
@@ -57,7 +59,7 @@ class BenchmarkDriver::Runner::Once
 
     Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        execute(*exec.command, path)
+        execute(*context.executable.command, path)
       end
       Float(f.read)
     end

data/lib/benchmark_driver/runner/recorded.rb

@@ -30,9 +30,11 @@ class BenchmarkDriver::Runner::Recorded
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -48,7 +50,7 @@ class BenchmarkDriver::Runner::Recorded
       records.each do |record|
         @output.with_job(name: record.name) do
           record.benchmark_results.each do |context, result|
-            @output.with_context(name: context.name, executable: context.executable) do
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems) do
               @output.report(
                 values: result.values,
                 duration: result.duration,

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -64,9 +64,11 @@ class BenchmarkDriver::Runner::RubyStdout
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output] output
-  def initialize(config:, output:)
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
     @config = config
     @output = output
+    @contexts = contexts
   end
 
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
@@ -77,7 +79,8 @@ class BenchmarkDriver::Runner::RubyStdout
     @output.with_benchmark do
       jobs.each do |job|
         @output.with_job(name: job.name) do
-          @config.executables.each do |exec|
+          @contexts.each do |context|
+            exec = context.executable
             repeat_params = { config: @config, larger_better: metric.larger_better }
             value, environment = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               stdout = with_chdir(job.working_directory) do

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.12.0'
+  VERSION = '0.13.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-01 00:00:00.000000000 Z
+date: 2018-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler