benchmark_driver 0.15.9 → 0.15.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e99ed161f437a1c8e08847db6bee7f8ada9773fc527d1be4aaa6436c18ce16d3
4
- data.tar.gz: cc6aa7374390d0451f8d75451676481b032d1d7d8fe4bbdc442e77b4c2fad19a
3
+ metadata.gz: 6d1052380e38062b6a13370022a91959621902cf2d08263d29fc504c9297655f
4
+ data.tar.gz: 32067954ee2170d157f974ead3dce80aa7a2d0dca6f322f2aae053693775a656
5
5
  SHA512:
6
- metadata.gz: 2a89c19b24c61a01b97a6814c6a47185fb19ce81ea8afedbfefa812610aa4774179a128d8e365435c58f967d84faf61ce7628b8672db65e7962a5ee950045db6
7
- data.tar.gz: 71417fee7eadda1c26f6d0cab1c9c7563eb1d8da011f4b4b2c2e1701eaba573bdbf67230fb74034cc08353396bc20e6573ea79cc8acb8ab17f7c6cd01285cb21
6
+ metadata.gz: d8d5417b30e026ea54fe080f434b89fde4c36f8136b3ea082b972d032730ed421d019c5eb2213efbcf5d0e905cc41ced746db31a0af5f1f3c8edfb1ecfb16320
7
+ data.tar.gz: 72da43bfd0d31749296ceb806da6210a86d55ad1046a06a8b2664e023c8ace1f782e1df1f7919205d4e9e95abc736440360fab1a1f8c6824028abc0698ebd2a8
@@ -1,3 +1,23 @@
1
+ # v0.15.14
2
+
3
+ - YAML's `type` key allows a value with `/` to specify a relative path of a runner plugin
4
+
5
+ # v0.15.13
6
+
7
+ - Show a command and stdout on `-vv` for `ips`, `time`, and `block` runner
8
+
9
+ # v0.15.12
10
+
11
+ - Show comparison on `--output=markdown` when `--output-compare` is also specified
12
+
13
+ # v0.15.11
14
+
15
+ - Correctly calculate an average result on `--repeat-result=average`
16
+
17
+ # v0.15.10
18
+
19
+ - Right-justify calculated results in `markdown` output
20
+
1
21
  # v0.15.9
2
22
 
3
23
  - Prefer an exact match in RVM version selection
@@ -124,7 +124,7 @@ jobs = config.paths.flat_map do |path|
124
124
 
125
125
  begin
126
126
  # `working_directory` is YAML-specific special parameter, mainly for "command_stdout"
127
- BenchmarkDriver::JobParser.parse(job, default_params: { working_directory: File.dirname(path) })
127
+ BenchmarkDriver::JobParser.parse(job, working_directory: File.dirname(path))
128
128
  rescue ArgumentError
129
129
  $stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
130
130
  $stderr.puts ' YAML format may be wrong. See error below:'
@@ -3,24 +3,27 @@ require 'benchmark_driver/runner'
3
3
  module BenchmarkDriver
4
4
  class << JobParser = Module.new
5
5
  # @param [Hash] config
6
- # @param [Hash] default_params - Special default values not written in job definition
7
- def parse(config, default_params: {})
6
+ # @param [Hash] working_directory - YAML-specific special parameter for "command_stdout" and a relative path in type
7
+ def parse(config, working_directory: nil)
8
8
  config = symbolize_keys(config)
9
9
  type = config.fetch(:type)
10
10
  if !type.is_a?(String)
11
11
  raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
12
- elsif !type.match(/\A[A-Za-z0-9_]+\z/)
13
- raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
12
+ elsif !type.match(/\A[A-Za-z0-9_\/]+\z/)
13
+ raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_\/])")
14
14
  end
15
15
  config.delete(:type)
16
16
 
17
17
  # Dynamic dispatch for plugin support
18
- require "benchmark_driver/runner/#{type}"
18
+ if type.include?('/')
19
+ require File.join(working_directory || '.', type)
20
+ type = File.basename(type)
21
+ else
22
+ require "benchmark_driver/runner/#{type}"
23
+ end
19
24
  job = ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(**config)
20
- default_params.each do |key, value|
21
- if job.respond_to?(key) && job.respond_to?("#{key}=") && job.public_send(key).nil?
22
- job.public_send("#{key}=", value)
23
- end
25
+ if job.respond_to?(:working_directory) && job.respond_to?(:working_directory=) && job.working_directory.nil?
26
+ job.working_directory = working_directory
24
27
  end
25
28
  job
26
29
  end
@@ -117,17 +117,17 @@ class BenchmarkDriver::Output::Compare
117
117
 
118
118
  def humanize(value, width = 10)
119
119
  if BenchmarkDriver::Result::ERROR.equal?(value)
120
- return " %#{width}s" % 'ERROR'
120
+ return sprintf(" %*s", width, 'ERROR')
121
121
  elsif value == 0.0
122
- return " %#{width}.3f" % 0.0
122
+ return sprintf(" %*.3f", width, 0.0)
123
123
  elsif value < 0
124
124
  raise ArgumentError.new("Negative value: #{value.inspect}")
125
125
  end
126
126
 
127
127
  scale = (Math.log10(value) / 3).to_i
128
- return "%#{width}s" % value.to_s if scale < 0 # like 1.23e-04
128
+ return sprintf("%*s", width, value.to_s) if scale < 0 # like 1.23e-04
129
129
 
130
- prefix = "%#{width}.3f" % (value.to_f / (1000 ** scale))
130
+ prefix = sprintf("%*.3f", width, (value.to_f / (1000 ** scale)))
131
131
  suffix =
132
132
  case scale
133
133
  when 1; 'k'
@@ -173,7 +173,7 @@ class BenchmarkDriver::Output::Compare
173
173
  $stdout.puts "\nComparison:"
174
174
 
175
175
  @job_context_result.each do |job, context_result|
176
- $stdout.puts("%#{@name_length + 2 + 11}s" % job)
176
+ $stdout.printf("%*s\n", @name_length + 2 + 11, job)
177
177
  results = context_result.flat_map do |context, result|
178
178
  result.values.values.map { |value| Result.new(job: job, value: value, context: context) }
179
179
  end
@@ -188,7 +188,7 @@ class BenchmarkDriver::Output::Compare
188
188
 
189
189
  unless BenchmarkDriver::Result::ERROR.equal?(bottom)
190
190
  ratio = top / bottom
191
- "- %.2fx #{@metrics.first.worse_word}" % ratio
191
+ sprintf("- %.2fx %s", ratio, @metrics.first.worse_word)
192
192
  end
193
193
  end
194
194
 
@@ -211,7 +211,7 @@ class BenchmarkDriver::Output::Compare
211
211
  else
212
212
  name = result.job
213
213
  end
214
- $stdout.puts("%#{@name_length}s: %11.1f %s #{slower}" % [name, result.value, @metrics.first.unit])
214
+ $stdout.printf("%*s: %11.1f %s %s\n", @name_length, name, result.value, @metrics.first.unit, slower)
215
215
  end
216
216
  $stdout.puts
217
217
  end
@@ -1,13 +1,19 @@
1
1
  class BenchmarkDriver::Output::Markdown
2
2
  NAME_LENGTH = 8
3
3
 
4
+ OPTIONS = {
5
+ compare: ['--output-compare', 'Show comparison between results'],
6
+ }
7
+
4
8
  # @param [Array<BenchmarkDriver::Metric>] metrics
5
9
  # @param [Array<BenchmarkDriver::Job>] jobs
6
10
  # @param [Array<BenchmarkDriver::Context>] contexts
7
- def initialize(metrics:, jobs:, contexts:)
11
+ def initialize(metrics:, jobs:, contexts:, options:)
8
12
  @metrics = metrics
13
+ @contexts = contexts
9
14
  @context_names = contexts.map(&:name)
10
15
  @name_length = jobs.map(&:name).map(&:size).max
16
+ @compare = options.fetch(:compare, false)
11
17
  end
12
18
 
13
19
  def with_warmup(&block)
@@ -28,14 +34,15 @@ class BenchmarkDriver::Output::Markdown
28
34
  # Show executable names
29
35
  $stdout.print("|#{' ' * @name_length} ")
30
36
  @context_names.each do |context_name|
31
- $stdout.print("|%#{NAME_LENGTH}s" % context_name) # same size as humanize
37
+ $stdout.printf("|%*s", NAME_LENGTH, context_name) # same size as humanize
32
38
  end
33
39
  $stdout.puts('|')
34
40
 
35
41
  # Show header separator
36
42
  $stdout.print("|:#{'-' * (@name_length - 1)}--")
37
43
  @context_names.each do |context_name|
38
- $stdout.print("|:#{'-' * (NAME_LENGTH - 1)}") # same size as humanize
44
+ length = [context_name.length, NAME_LENGTH].max
45
+ $stdout.print("|#{'-' * (length - 1)}:") # same size as humanize
39
46
  end
40
47
  $stdout.puts('|')
41
48
 
@@ -48,24 +55,33 @@ class BenchmarkDriver::Output::Markdown
48
55
  # @param [BenchmarkDriver::Job] job
49
56
  def with_job(job, &block)
50
57
  if @with_benchmark
51
- $stdout.print("|%-#{@name_length}s " % job.name)
58
+ @job_context_result = {} if @context_names.size > 1
59
+
60
+ $stdout.printf("|%-*s ", @name_length, job.name)
52
61
  end
53
62
  block.call
54
63
  ensure
55
64
  if @with_benchmark
56
65
  $stdout.puts('|')
66
+ compare_executables if @compare && @context_names.size > 1
57
67
  end
58
68
  end
59
69
 
60
70
  # @param [BenchmarkDriver::Context] context
61
71
  def with_context(context, &block)
72
+ @context = context
62
73
  block.call
63
74
  end
64
75
 
65
76
  # @param [BenchmarkDriver::Result] result
66
77
  def report(result)
78
+ if defined?(@job_context_result)
79
+ @job_context_result[@context] = result
80
+ end
81
+
67
82
  if @with_benchmark
68
- $stdout.print("|%#{NAME_LENGTH}s" % humanize(result.values.fetch(@metrics.first)))
83
+ length = [NAME_LENGTH, @context.name.length].max
84
+ $stdout.printf("|%*s", length, humanize(result.values.fetch(@metrics.first)))
69
85
  else
70
86
  $stdout.print '.'
71
87
  end
@@ -83,15 +99,15 @@ class BenchmarkDriver::Output::Markdown
83
99
 
84
100
  def humanize(value)
85
101
  if BenchmarkDriver::Result::ERROR.equal?(value)
86
- return "%#{NAME_LENGTH}s" % 'ERROR'
102
+ return sprintf("%*s", NAME_LENGTH, 'ERROR')
87
103
  elsif value == 0.0
88
- return "%#{NAME_LENGTH}.3f" % 0.0
104
+ return sprintf("%*.3f", NAME_LENGTH, 0.0)
89
105
  elsif value < 0
90
106
  raise ArgumentError.new("Negative value: #{value.inspect}")
91
107
  end
92
108
 
93
109
  scale = (Math.log10(value) / 3).to_i
94
- prefix = "%#{NAME_LENGTH - 1}.3f" % (value.to_f / (1000 ** scale))
110
+ prefix = sprintf("%*.3f", NAME_LENGTH - 1, (value.to_f / (1000 ** scale)))
95
111
  suffix =
96
112
  case scale
97
113
  when 1; 'k'
@@ -104,4 +120,30 @@ class BenchmarkDriver::Output::Markdown
104
120
  end
105
121
  "#{prefix}#{suffix}"
106
122
  end
123
+
124
+ def compare_executables
125
+ order = @metrics.first.larger_better ? :min_by : :max_by
126
+ worst, worst_result = @job_context_result.__send__(order) do |_, result|
127
+ result.values.first[1]
128
+ end
129
+ worst_result = worst_result.values.first[1]
130
+ $stdout.print("|", " " * (@name_length + 2))
131
+ @job_context_result.each do |context, result|
132
+ if context == worst
133
+ result = '-'
134
+ else
135
+ result = result.values.first[1]
136
+ if order == :min_by
137
+ result = result.fdiv(worst_result)
138
+ else
139
+ result = best_result.fdiv(worst_result)
140
+ end
141
+ result = sprintf("%.2fx", result)
142
+ end
143
+ length = [context.name.length, NAME_LENGTH].max
144
+ $stdout.printf("|%*s", length, result)
145
+ end
146
+ $stdout.puts('|')
147
+ end
148
+
107
149
  end
@@ -54,7 +54,7 @@ module BenchmarkDriver
54
54
  [values.map { |v| v[0] }.inject(&:+) / values.size.to_f, *rest]
55
55
  when :average
56
56
  values.first.size.times.map do |index|
57
- values.map { |v| v[index] }.inject(&:+) / values.first.size.to_f
57
+ values.map { |v| v[index] }.inject(&:+) / values.size.to_f
58
58
  end
59
59
  else
60
60
  raise "unexpected rest_on_average #{rest_on_average.inspect}"
@@ -107,8 +107,8 @@ class BenchmarkDriver::Runner::Ips
107
107
 
108
108
  duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
109
109
  with_script(benchmark.render(result: f.path)) do |path|
110
- IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
111
- if $?.success? && ((value = Float(f.read)) > 0)
110
+ success = execute(*context.executable.command, path, exception: false)
111
+ if success && ((value = Float(f.read)) > 0)
112
112
  value
113
113
  else
114
114
  BenchmarkDriver::Result::ERROR
@@ -137,10 +137,7 @@ class BenchmarkDriver::Runner::Ips
137
137
  end
138
138
 
139
139
  def with_script(script)
140
- if @config.verbose >= 2
141
- sep = '-' * 30
142
- $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
143
- end
140
+ debug_output('Script', script) if @config.verbose >= 2
144
141
 
145
142
  Tempfile.open(['benchmark_driver-', '.rb']) do |f|
146
143
  f.puts script
@@ -149,11 +146,20 @@ class BenchmarkDriver::Runner::Ips
149
146
  end
150
147
  end
151
148
 
152
- def execute(*args)
153
- IO.popen(args, &:read) # TODO: print stdout if verbose=2
154
- unless $?.success?
155
- raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
149
+ def execute(*args, exception: true)
150
+ $stderr.puts "$ #{args.shelljoin}" if @config.verbose >= 2
151
+
152
+ stdout = IO.popen(args, &:read)
153
+ debug_output('Command output', stdout) if @config.verbose >= 2
154
+ if exception && !$?.success?
155
+ raise "Failed to execute: #{args.shelljoin} (status: #{$?})"
156
156
  end
157
+ $?.success?
158
+ end
159
+
160
+ def debug_output(name, text)
161
+ sep = '-' * 30
162
+ $stdout.puts "\n\n#{sep}[#{name} begin]#{sep}\n#{text}#{sep}[#{name} end]#{sep}\n\n"
157
163
  end
158
164
 
159
165
  WarmupScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count, :first_warmup_duration, :second_warmup_duration) do
@@ -1,3 +1,3 @@
1
1
  module BenchmarkDriver
2
- VERSION = '0.15.9'
2
+ VERSION = '0.15.14'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: benchmark_driver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.9
4
+ version: 0.15.14
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takashi Kokubun
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-01-04 00:00:00.000000000 Z
11
+ date: 2020-06-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
146
146
  - !ruby/object:Gem::Version
147
147
  version: '0'
148
148
  requirements: []
149
- rubygems_version: 3.1.2
149
+ rubygems_version: 3.2.0.pre1
150
150
  signing_key:
151
151
  specification_version: 4
152
152
  summary: Fully-featured accurate benchmark driver for Ruby