benchmark_driver 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d35c947c3b1718ea35b688a22546b6fcc28d6494219791ca9b990571a20ec60b
-  data.tar.gz: 67335dcf0ddb2d881539f6ff7c3f565483a4bf4781829ffe75979276869b34ab
+  metadata.gz: 6b444be9753246263a4b3a65f0fe33a07c62f4a538008245fb8a17b2e809f7b6
+  data.tar.gz: e4a43fb02781c343e266e8c5644d8fc4edf1568de93d620e90500f027bb4418a
 SHA512:
-  metadata.gz: 9d78acc25353ed4d06676671bff54a40256a0d6b90ff543ece4659931d9c775a5496445976a7e3d0ce25326c6164394004b8060a8d7d41def62bc75f7c79a06a
-  data.tar.gz: c8b1e02f00655551d8ff4333ccfa7f887da69dae34902e9f525aeaf37a63633bd480556cb3b3a518941c71824afea28d054b736be81e9fb9ed97fb45c3aac377
+  metadata.gz: 4c512a0e7bb65debbb79b2f312db9852db1625cfd3a46cab847b81f0d3f85d09caef043e09b1f47679f6b476ea98d2a913ea7fbd1a694aac4621a2e701256450
+  data.tar.gz: 5fd294728cf04e4524a62aa12916ba4c66b6e9f8dab498cb290a20d324a3aa60b5f53786d6bb150ec1a44d730770a40d77ab8e3f29fcd5c109ed85f5e02c074c

data/.gitignore

@@ -7,3 +7,4 @@
 /spec/reports/
 /tmp/
 /Gemfile.lock
+benchmark_driver.record.yml

data/.travis.yml

@@ -11,5 +11,4 @@ branches:
     - master
 before_install: gem install bundler -v 1.15.4
 script:
-  - bundle exec rake ruby_examples
-  - bundle exec rake yaml_examples
+  - bundle exec rake

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# v0.10.0
+
+- Add "record" output and "recorded" runner
+  - You can record metrics to yaml and change how to output later
+- Change output interface to set metrics\_type
+  - Now runner needs to call `output#metrics_type=`
+
 # v0.9.2
 
 - Add `--verbose` option

data/Rakefile

@@ -1,7 +1,31 @@
 require 'bundler/gem_tasks'
 require 'shellwords'
 
-task :ruby_examples do
+task :test do
+  blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
+  blank_hash = File.expand_path('./examples/yaml/blank_hash.yml', __dir__) # needs warmup
+  {
+    'ips' => 'compare',
+    'time' => 'simple',
+    'memory' => 'simple',
+    'once' => 'markdown',
+  }.each do |runner, output|
+    Bundler.with_clean_env do
+      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', runner, '-o', output].shelljoin
+      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_hash, '-r', runner, '-o', output, '--run-duration', '1'].shelljoin
+    end
+  end
+end
+
+task :test_record do
+  blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
+  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', 'ips', '-o', 'record'].shelljoin
+  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'compare'].shelljoin
+  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'record'].shelljoin
+  sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'simple'].shelljoin
+end
+
+task :test_ruby do
   Dir.glob(File.expand_path('./examples/*.rb', __dir__)).sort.each do |file|
     Bundler.with_clean_env do
       sh ['time', 'bundle', 'exec', 'ruby', file].shelljoin
@@ -10,13 +34,13 @@ task :ruby_examples do
   end
 end
 
-task :yaml_examples do
+task :test_yaml do
   Dir.glob(File.expand_path('./examples/yaml/*.yml', __dir__)).sort.each do |file|
     Bundler.with_clean_env do
-      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', file].shelljoin
+      sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', file, '--run-duration', '1'].shelljoin
     end
     puts
   end
 end
 
-task default: [:ruby_examples, :yaml_examples]
+task default: [:test, :test_record, :test_ruby, :test_yaml]

data/exe/benchmark-driver

@@ -16,7 +16,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
       abort '-r, --runner must take argument but not given' if d.nil?
       c.runner_type = d
     end
-    o.on('-o', '--output [TYPE]', 'Specify output type: compare, simple, markdown (default: compare)') do |out|
+    o.on('-o', '--output [TYPE]', 'Specify output type: compare, simple, markdown, record (default: compare)') do |out|
       abort '-o, --output must take argument but not given' if out.nil?
       c.output_type = out
     end

data/lib/benchmark_driver/default_job.rb

@@ -2,7 +2,7 @@ require 'benchmark_driver/struct'
 
 module BenchmarkDriver
   DefaultJob = ::BenchmarkDriver::Struct.new(
-    :name,       # @param [String] name
+    :name,       # @param [String] name - This is mandatory for all runner
     :script,     # @param [String] benchmark
     :prelude,    # @param [String,nil] prelude (optional)
     :teardown,   # @param [String,nil] after (optional)

data/lib/benchmark_driver/job_parser.rb

@@ -29,7 +29,12 @@ module BenchmarkDriver
       when Hash
         config.dup.tap do |hash|
           hash.keys.each do |key|
-            hash[key.to_sym] = symbolize_keys(hash.delete(key))
+            case key
+            when String, Symbol
+              hash[key.to_sym] = symbolize_keys(hash.delete(key))
+            else # Struct
+              hash[key] = symbolize_keys(hash.delete(key))
+            end
           end
         end
       when Array

data/lib/benchmark_driver/output.rb

@@ -2,6 +2,7 @@ module BenchmarkDriver
   module Output
     require 'benchmark_driver/output/compare'
     require 'benchmark_driver/output/markdown'
+    require 'benchmark_driver/output/record'
     require 'benchmark_driver/output/simple'
   end
 

data/lib/benchmark_driver/output/compare.rb

@@ -2,16 +2,16 @@
 class BenchmarkDriver::Output::Compare
   NAME_LENGTH = 20
 
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  attr_writer :metrics_type
+
   # @param [Array<BenchmarkDriver::*::Job>] jobs
   # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  def initialize(jobs:, executables:, metrics_type:)
+  def initialize(jobs:, executables:)
     @jobs = jobs
     @executables = executables
-    @metrics_type = metrics_type
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_warmup(&block)
     without_stdout_buffering do
       $stdout.puts 'Warming up --------------------------------------'
@@ -20,7 +20,6 @@ class BenchmarkDriver::Output::Compare
     end
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_benchmark(&block)
     @metrics_by_job = Hash.new { |h, k| h[k] = [] }
 

data/lib/benchmark_driver/output/markdown.rb

@@ -1,17 +1,17 @@
 class BenchmarkDriver::Output::Markdown
   NAME_LENGTH = 8
 
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  attr_writer :metrics_type
+
   # @param [Array<BenchmarkDriver::*::Job>] jobs
   # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  def initialize(jobs:, executables:, metrics_type:)
+  def initialize(jobs:, executables:)
     @jobs = jobs
     @executables = executables
-    @metrics_type = metrics_type
     @name_length = jobs.map { |j| j.name.size }.max
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_warmup(&block)
     without_stdout_buffering do
       $stdout.print 'warming up'
@@ -21,7 +21,6 @@ class BenchmarkDriver::Output::Markdown
     $stdout.puts
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_benchmark(&block)
     @with_benchmark = true
     without_stdout_buffering do

data/lib/benchmark_driver/output/record.rb

@@ -0,0 +1,59 @@
+class BenchmarkDriver::Output::Record
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  attr_writer :metrics_type
+
+  # @param [Array<BenchmarkDriver::*::Job>] jobs
+  # @param [Array<BenchmarkDriver::Config::Executable>] executables
+  def initialize(jobs:, executables:)
+    @jobs = jobs
+    @executables = executables
+    @metrics_by_job = Hash.new do |h1, k1|
+      h1[k1] = Hash.new { |h2, k2| h2[k2] = [] }
+    end
+  end
+
+  def with_warmup(&block)
+    $stdout.print 'warming up'
+    block.call
+  ensure
+    $stdout.puts
+  end
+
+  def with_benchmark(&block)
+    @with_benchmark = true
+    $stdout.print 'benchmarking'
+    block.call
+  ensure
+    $stdout.puts
+    @with_benchmark = false
+    save_record
+  end
+
+  # @param [BenchmarkDriver::*::Job] job
+  def with_job(job, &block)
+    @current_job = job
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def report(metrics)
+    $stdout.print '.'
+    if @with_benchmark
+      @metrics_by_job[@current_job][:benchmark] << metrics
+    else
+      @metrics_by_job[@current_job][:warmup] << metrics
+    end
+  end
+
+  private
+
+  def save_record
+    jobs = @benchmark_metrics
+    yaml = {
+      'type' => 'recorded',
+      'metrics_by_job' => @metrics_by_job,
+      'metrics_type' => @metrics_type,
+    }.to_yaml
+    File.write('benchmark_driver.record.yml', yaml)
+  end
+end

data/lib/benchmark_driver/output/simple.rb

@@ -1,17 +1,17 @@
 class BenchmarkDriver::Output::Simple
   NAME_LENGTH = 8
 
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  attr_writer :metrics_type
+
   # @param [Array<BenchmarkDriver::*::Job>] jobs
   # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  def initialize(jobs:, executables:, metrics_type:)
+  def initialize(jobs:, executables:)
     @jobs = jobs
     @executables = executables
-    @metrics_type = metrics_type
     @name_length = jobs.map { |j| j.name.size }.max
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_warmup(&block)
     without_stdout_buffering do
       $stdout.print 'warming up'
@@ -21,7 +21,6 @@ class BenchmarkDriver::Output::Simple
     $stdout.puts
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
   def with_benchmark(&block)
     @with_benchmark = true
     without_stdout_buffering do

data/lib/benchmark_driver/runner.rb

@@ -3,6 +3,7 @@ module BenchmarkDriver
     require 'benchmark_driver/runner/ips'
     require 'benchmark_driver/runner/memory'
     require 'benchmark_driver/runner/once'
+    require 'benchmark_driver/runner/recorded'
     require 'benchmark_driver/runner/time'
   end
 
@@ -29,7 +30,6 @@ module BenchmarkDriver
         output = Output.find(config.output_type).new(
           jobs: jobs,
           executables: config.executables,
-          metrics_type: runner::MetricsType,
         )
         runner.new(config: runner_config, output: output).run(jobs)
       end

data/lib/benchmark_driver/runner/ips.rb

@@ -11,8 +11,8 @@ class BenchmarkDriver::Runner::Ips
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-  # Passed to `output` by `BenchmarkDriver::Runner.run`
-  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
+
+  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output::*] output
@@ -24,6 +24,8 @@ class BenchmarkDriver::Runner::Ips
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
+    set_metrics_type
+
     if jobs.any? { |job| job.loop_count.nil? }
       @output.with_warmup do
         jobs = jobs.map do |job|
@@ -123,6 +125,11 @@ class BenchmarkDriver::Runner::Ips
     )
   end
 
+  # This method is overridden by BenchmarkDriver::Runner::Time
+  def set_metrics_type
+    @output.metrics_type = METRICS_TYPE
+  end
+
   def with_script(script)
     if @config.verbose >= 2
       sep = '-' * 30

data/lib/benchmark_driver/runner/memory.rb

@@ -11,8 +11,8 @@ class BenchmarkDriver::Runner::Memory
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-  # Passed to `output` by `BenchmarkDriver::Runner.run`
-  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 'bytes', larger_better: false, worse_word: 'larger')
+
+  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'bytes', larger_better: false, worse_word: 'larger')
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output::*] output
@@ -29,6 +29,8 @@ class BenchmarkDriver::Runner::Memory
       raise "memory output is not supported for '#{Etc.uname[:sysname]}' for now"
     end
 
+    @output.metrics_type = METRICS_TYPE
+
     if jobs.any? { |job| job.loop_count.nil? }
       jobs = jobs.map do |job|
         job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))

data/lib/benchmark_driver/runner/once.rb

@@ -11,8 +11,8 @@ class BenchmarkDriver::Runner::Once
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-  # Passed to `output` by `BenchmarkDriver::Runner.run`
-  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
+
+  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
 
   # @param [BenchmarkDriver::Config::RunnerConfig] config
   # @param [BenchmarkDriver::Output::*] output
@@ -24,6 +24,8 @@ class BenchmarkDriver::Runner::Once
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
+    @output.metrics_type = METRICS_TYPE
+
     jobs = jobs.map do |job|
       Job.new(job.to_h.merge(loop_count: 1)) # to show this on output
     end

data/lib/benchmark_driver/runner/recorded.rb

@@ -0,0 +1,61 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metrics'
+require 'tempfile'
+require 'shellwords'
+
+# Run only once, for testing
+class BenchmarkDriver::Runner::Recorded
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,              # @param [String] name - This is mandatory for all runner
+    :job,               # @param [BenchmarkDriver::Runner::*::Job]
+    :warmup_metrics,    # @param [Hash]
+    :benchmark_metrics, # @param [Hash]
+    :metrics_type,      # @param [BenchmarkDriver::Metrics::Type]
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [Hash] metrics_by_job
+    # @param [BenchmarkDriver::Metrics::Type] metrics_type
+    def parse(metrics_by_job:, metrics_type:)
+      metrics_by_job.map do |job, metrics_hash|
+        Job.new(
+          name: job.name,
+          job: job,
+          warmup_metrics: metrics_hash.fetch(:warmup, []),
+          benchmark_metrics: metrics_hash.fetch(:benchmark),
+          metrics_type: metrics_type,
+        )
+      end
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output::*] output
+  def initialize(config:, output:)
+    @config = config
+    @output = output
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(records)
+    @output.metrics_type = records.first.metrics_type
+
+    records.each do |record|
+      unless record.warmup_metrics.empty?
+        # TODO:
+      end
+    end
+
+    @output.with_benchmark do
+      records.each do |record|
+        @output.with_job(record.job) do
+          record.benchmark_metrics.each do |metrics|
+            @output.report(metrics)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/benchmark_driver/runner/time.rb

@@ -5,8 +5,13 @@ class BenchmarkDriver::Runner::Time < BenchmarkDriver::Runner::Ips
   Job = Class.new(BenchmarkDriver::DefaultJob)
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
-  # Passed to `output` by `BenchmarkDriver::Runner.run`
-  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 's')
+
+  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 's')
+
+  # Overriding BenchmarkDriver::Runner::Ips#set_metrics_type
+  def set_metrics_type
+    @output.metrics_type = METRICS_TYPE
+  end
 
   # Overriding BenchmarkDriver::Runner::Ips#build_metrics
   def build_metrics(duration:, executable:, loop_count:)

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.9.2'
+  VERSION = '0.10.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-24 00:00:00.000000000 Z
+date: 2018-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -77,6 +77,7 @@ files:
 - lib/benchmark_driver/output.rb
 - lib/benchmark_driver/output/compare.rb
 - lib/benchmark_driver/output/markdown.rb
+- lib/benchmark_driver/output/record.rb
 - lib/benchmark_driver/output/simple.rb
 - lib/benchmark_driver/rbenv.rb
 - lib/benchmark_driver/ruby_interface.rb
@@ -84,6 +85,7 @@ files:
 - lib/benchmark_driver/runner/ips.rb
 - lib/benchmark_driver/runner/memory.rb
 - lib/benchmark_driver/runner/once.rb
+- lib/benchmark_driver/runner/recorded.rb
 - lib/benchmark_driver/runner/time.rb
 - lib/benchmark_driver/struct.rb
 - lib/benchmark_driver/version.rb