benchmark_driver 0.15.9 → 0.15.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/exe/benchmark-driver +1 -1
- data/lib/benchmark_driver/job_parser.rb +12 -9
- data/lib/benchmark_driver/output/compare.rb +7 -7
- data/lib/benchmark_driver/output/markdown.rb +50 -8
- data/lib/benchmark_driver/repeater.rb +1 -1
- data/lib/benchmark_driver/runner/ips.rb +16 -10
- data/lib/benchmark_driver/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6d1052380e38062b6a13370022a91959621902cf2d08263d29fc504c9297655f
|
4
|
+
data.tar.gz: 32067954ee2170d157f974ead3dce80aa7a2d0dca6f322f2aae053693775a656
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d8d5417b30e026ea54fe080f434b89fde4c36f8136b3ea082b972d032730ed421d019c5eb2213efbcf5d0e905cc41ced746db31a0af5f1f3c8edfb1ecfb16320
|
7
|
+
data.tar.gz: 72da43bfd0d31749296ceb806da6210a86d55ad1046a06a8b2664e023c8ace1f782e1df1f7919205d4e9e95abc736440360fab1a1f8c6824028abc0698ebd2a8
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,23 @@
|
|
1
|
+
# v0.15.14
|
2
|
+
|
3
|
+
- YAML's `type` key allows a value with `/` to specify a relative path of a runner plugin
|
4
|
+
|
5
|
+
# v0.15.13
|
6
|
+
|
7
|
+
- Show a command and stdout on `-vv` for `ips`, `time`, and `block` runner
|
8
|
+
|
9
|
+
# v0.15.12
|
10
|
+
|
11
|
+
- Show comparison on `--output=markdown` when `--output-compare` is also specified
|
12
|
+
|
13
|
+
# v0.15.11
|
14
|
+
|
15
|
+
- Correctly calculate an average result on `--repeat-result=average`
|
16
|
+
|
17
|
+
# v0.15.10
|
18
|
+
|
19
|
+
- Right-justify calculated results in `markdown` output
|
20
|
+
|
1
21
|
# v0.15.9
|
2
22
|
|
3
23
|
- Prefer an exact match in RVM version selection
|
data/exe/benchmark-driver
CHANGED
@@ -124,7 +124,7 @@ jobs = config.paths.flat_map do |path|
|
|
124
124
|
|
125
125
|
begin
|
126
126
|
# `working_directory` is YAML-specific special parameter, mainly for "command_stdout"
|
127
|
-
BenchmarkDriver::JobParser.parse(job,
|
127
|
+
BenchmarkDriver::JobParser.parse(job, working_directory: File.dirname(path))
|
128
128
|
rescue ArgumentError
|
129
129
|
$stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
|
130
130
|
$stderr.puts ' YAML format may be wrong. See error below:'
|
@@ -3,24 +3,27 @@ require 'benchmark_driver/runner'
|
|
3
3
|
module BenchmarkDriver
|
4
4
|
class << JobParser = Module.new
|
5
5
|
# @param [Hash] config
|
6
|
-
# @param [Hash]
|
7
|
-
def parse(config,
|
6
|
+
# @param [Hash] working_directory - YAML-specific special parameter for "command_stdout" and a relative path in type
|
7
|
+
def parse(config, working_directory: nil)
|
8
8
|
config = symbolize_keys(config)
|
9
9
|
type = config.fetch(:type)
|
10
10
|
if !type.is_a?(String)
|
11
11
|
raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
|
12
|
-
elsif !type.match(/\A[A-Za-z0-9_]+\z/)
|
13
|
-
raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
|
12
|
+
elsif !type.match(/\A[A-Za-z0-9_\/]+\z/)
|
13
|
+
raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_\/])")
|
14
14
|
end
|
15
15
|
config.delete(:type)
|
16
16
|
|
17
17
|
# Dynamic dispatch for plugin support
|
18
|
-
|
18
|
+
if type.include?('/')
|
19
|
+
require File.join(working_directory || '.', type)
|
20
|
+
type = File.basename(type)
|
21
|
+
else
|
22
|
+
require "benchmark_driver/runner/#{type}"
|
23
|
+
end
|
19
24
|
job = ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(**config)
|
20
|
-
|
21
|
-
|
22
|
-
job.public_send("#{key}=", value)
|
23
|
-
end
|
25
|
+
if job.respond_to?(:working_directory) && job.respond_to?(:working_directory=) && job.working_directory.nil?
|
26
|
+
job.working_directory = working_directory
|
24
27
|
end
|
25
28
|
job
|
26
29
|
end
|
@@ -117,17 +117,17 @@ class BenchmarkDriver::Output::Compare
|
|
117
117
|
|
118
118
|
def humanize(value, width = 10)
|
119
119
|
if BenchmarkDriver::Result::ERROR.equal?(value)
|
120
|
-
return "
|
120
|
+
return sprintf(" %*s", width, 'ERROR')
|
121
121
|
elsif value == 0.0
|
122
|
-
return "
|
122
|
+
return sprintf(" %*.3f", width, 0.0)
|
123
123
|
elsif value < 0
|
124
124
|
raise ArgumentError.new("Negative value: #{value.inspect}")
|
125
125
|
end
|
126
126
|
|
127
127
|
scale = (Math.log10(value) / 3).to_i
|
128
|
-
return "
|
128
|
+
return sprintf("%*s", width, value.to_s) if scale < 0 # like 1.23e-04
|
129
129
|
|
130
|
-
prefix = "
|
130
|
+
prefix = sprintf("%*.3f", width, (value.to_f / (1000 ** scale)))
|
131
131
|
suffix =
|
132
132
|
case scale
|
133
133
|
when 1; 'k'
|
@@ -173,7 +173,7 @@ class BenchmarkDriver::Output::Compare
|
|
173
173
|
$stdout.puts "\nComparison:"
|
174
174
|
|
175
175
|
@job_context_result.each do |job, context_result|
|
176
|
-
$stdout.
|
176
|
+
$stdout.printf("%*s\n", @name_length + 2 + 11, job)
|
177
177
|
results = context_result.flat_map do |context, result|
|
178
178
|
result.values.values.map { |value| Result.new(job: job, value: value, context: context) }
|
179
179
|
end
|
@@ -188,7 +188,7 @@ class BenchmarkDriver::Output::Compare
|
|
188
188
|
|
189
189
|
unless BenchmarkDriver::Result::ERROR.equal?(bottom)
|
190
190
|
ratio = top / bottom
|
191
|
-
"- %.2fx
|
191
|
+
sprintf("- %.2fx %s", ratio, @metrics.first.worse_word)
|
192
192
|
end
|
193
193
|
end
|
194
194
|
|
@@ -211,7 +211,7 @@ class BenchmarkDriver::Output::Compare
|
|
211
211
|
else
|
212
212
|
name = result.job
|
213
213
|
end
|
214
|
-
$stdout.
|
214
|
+
$stdout.printf("%*s: %11.1f %s %s\n", @name_length, name, result.value, @metrics.first.unit, slower)
|
215
215
|
end
|
216
216
|
$stdout.puts
|
217
217
|
end
|
@@ -1,13 +1,19 @@
|
|
1
1
|
class BenchmarkDriver::Output::Markdown
|
2
2
|
NAME_LENGTH = 8
|
3
3
|
|
4
|
+
OPTIONS = {
|
5
|
+
compare: ['--output-compare', 'Show comparison between results'],
|
6
|
+
}
|
7
|
+
|
4
8
|
# @param [Array<BenchmarkDriver::Metric>] metrics
|
5
9
|
# @param [Array<BenchmarkDriver::Job>] jobs
|
6
10
|
# @param [Array<BenchmarkDriver::Context>] contexts
|
7
|
-
def initialize(metrics:, jobs:, contexts:)
|
11
|
+
def initialize(metrics:, jobs:, contexts:, options:)
|
8
12
|
@metrics = metrics
|
13
|
+
@contexts = contexts
|
9
14
|
@context_names = contexts.map(&:name)
|
10
15
|
@name_length = jobs.map(&:name).map(&:size).max
|
16
|
+
@compare = options.fetch(:compare, false)
|
11
17
|
end
|
12
18
|
|
13
19
|
def with_warmup(&block)
|
@@ -28,14 +34,15 @@ class BenchmarkDriver::Output::Markdown
|
|
28
34
|
# Show executable names
|
29
35
|
$stdout.print("|#{' ' * @name_length} ")
|
30
36
|
@context_names.each do |context_name|
|
31
|
-
$stdout.
|
37
|
+
$stdout.printf("|%*s", NAME_LENGTH, context_name) # same size as humanize
|
32
38
|
end
|
33
39
|
$stdout.puts('|')
|
34
40
|
|
35
41
|
# Show header separator
|
36
42
|
$stdout.print("|:#{'-' * (@name_length - 1)}--")
|
37
43
|
@context_names.each do |context_name|
|
38
|
-
|
44
|
+
length = [context_name.length, NAME_LENGTH].max
|
45
|
+
$stdout.print("|#{'-' * (length - 1)}:") # same size as humanize
|
39
46
|
end
|
40
47
|
$stdout.puts('|')
|
41
48
|
|
@@ -48,24 +55,33 @@ class BenchmarkDriver::Output::Markdown
|
|
48
55
|
# @param [BenchmarkDriver::Job] job
|
49
56
|
def with_job(job, &block)
|
50
57
|
if @with_benchmark
|
51
|
-
|
58
|
+
@job_context_result = {} if @context_names.size > 1
|
59
|
+
|
60
|
+
$stdout.printf("|%-*s ", @name_length, job.name)
|
52
61
|
end
|
53
62
|
block.call
|
54
63
|
ensure
|
55
64
|
if @with_benchmark
|
56
65
|
$stdout.puts('|')
|
66
|
+
compare_executables if @compare && @context_names.size > 1
|
57
67
|
end
|
58
68
|
end
|
59
69
|
|
60
70
|
# @param [BenchmarkDriver::Context] context
|
61
71
|
def with_context(context, &block)
|
72
|
+
@context = context
|
62
73
|
block.call
|
63
74
|
end
|
64
75
|
|
65
76
|
# @param [BenchmarkDriver::Result] result
|
66
77
|
def report(result)
|
78
|
+
if defined?(@job_context_result)
|
79
|
+
@job_context_result[@context] = result
|
80
|
+
end
|
81
|
+
|
67
82
|
if @with_benchmark
|
68
|
-
|
83
|
+
length = [NAME_LENGTH, @context.name.length].max
|
84
|
+
$stdout.printf("|%*s", length, humanize(result.values.fetch(@metrics.first)))
|
69
85
|
else
|
70
86
|
$stdout.print '.'
|
71
87
|
end
|
@@ -83,15 +99,15 @@ class BenchmarkDriver::Output::Markdown
|
|
83
99
|
|
84
100
|
def humanize(value)
|
85
101
|
if BenchmarkDriver::Result::ERROR.equal?(value)
|
86
|
-
return "
|
102
|
+
return sprintf("%*s", NAME_LENGTH, 'ERROR')
|
87
103
|
elsif value == 0.0
|
88
|
-
return "
|
104
|
+
return sprintf("%*.3f", NAME_LENGTH, 0.0)
|
89
105
|
elsif value < 0
|
90
106
|
raise ArgumentError.new("Negative value: #{value.inspect}")
|
91
107
|
end
|
92
108
|
|
93
109
|
scale = (Math.log10(value) / 3).to_i
|
94
|
-
prefix = "
|
110
|
+
prefix = sprintf("%*.3f", NAME_LENGTH - 1, (value.to_f / (1000 ** scale)))
|
95
111
|
suffix =
|
96
112
|
case scale
|
97
113
|
when 1; 'k'
|
@@ -104,4 +120,30 @@ class BenchmarkDriver::Output::Markdown
|
|
104
120
|
end
|
105
121
|
"#{prefix}#{suffix}"
|
106
122
|
end
|
123
|
+
|
124
|
+
def compare_executables
|
125
|
+
order = @metrics.first.larger_better ? :min_by : :max_by
|
126
|
+
worst, worst_result = @job_context_result.__send__(order) do |_, result|
|
127
|
+
result.values.first[1]
|
128
|
+
end
|
129
|
+
worst_result = worst_result.values.first[1]
|
130
|
+
$stdout.print("|", " " * (@name_length + 2))
|
131
|
+
@job_context_result.each do |context, result|
|
132
|
+
if context == worst
|
133
|
+
result = '-'
|
134
|
+
else
|
135
|
+
result = result.values.first[1]
|
136
|
+
if order == :min_by
|
137
|
+
result = result.fdiv(worst_result)
|
138
|
+
else
|
139
|
+
result = best_result.fdiv(worst_result)
|
140
|
+
end
|
141
|
+
result = sprintf("%.2fx", result)
|
142
|
+
end
|
143
|
+
length = [context.name.length, NAME_LENGTH].max
|
144
|
+
$stdout.printf("|%*s", length, result)
|
145
|
+
end
|
146
|
+
$stdout.puts('|')
|
147
|
+
end
|
148
|
+
|
107
149
|
end
|
@@ -54,7 +54,7 @@ module BenchmarkDriver
|
|
54
54
|
[values.map { |v| v[0] }.inject(&:+) / values.size.to_f, *rest]
|
55
55
|
when :average
|
56
56
|
values.first.size.times.map do |index|
|
57
|
-
values.map { |v| v[index] }.inject(&:+) / values.
|
57
|
+
values.map { |v| v[index] }.inject(&:+) / values.size.to_f
|
58
58
|
end
|
59
59
|
else
|
60
60
|
raise "unexpected rest_on_average #{rest_on_average.inspect}"
|
@@ -107,8 +107,8 @@ class BenchmarkDriver::Runner::Ips
|
|
107
107
|
|
108
108
|
duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
|
109
109
|
with_script(benchmark.render(result: f.path)) do |path|
|
110
|
-
|
111
|
-
if
|
110
|
+
success = execute(*context.executable.command, path, exception: false)
|
111
|
+
if success && ((value = Float(f.read)) > 0)
|
112
112
|
value
|
113
113
|
else
|
114
114
|
BenchmarkDriver::Result::ERROR
|
@@ -137,10 +137,7 @@ class BenchmarkDriver::Runner::Ips
|
|
137
137
|
end
|
138
138
|
|
139
139
|
def with_script(script)
|
140
|
-
if @config.verbose >= 2
|
141
|
-
sep = '-' * 30
|
142
|
-
$stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
|
143
|
-
end
|
140
|
+
debug_output('Script', script) if @config.verbose >= 2
|
144
141
|
|
145
142
|
Tempfile.open(['benchmark_driver-', '.rb']) do |f|
|
146
143
|
f.puts script
|
@@ -149,11 +146,20 @@ class BenchmarkDriver::Runner::Ips
|
|
149
146
|
end
|
150
147
|
end
|
151
148
|
|
152
|
-
def execute(*args)
|
153
|
-
|
154
|
-
|
155
|
-
|
149
|
+
def execute(*args, exception: true)
|
150
|
+
$stderr.puts "$ #{args.shelljoin}" if @config.verbose >= 2
|
151
|
+
|
152
|
+
stdout = IO.popen(args, &:read)
|
153
|
+
debug_output('Command output', stdout) if @config.verbose >= 2
|
154
|
+
if exception && !$?.success?
|
155
|
+
raise "Failed to execute: #{args.shelljoin} (status: #{$?})"
|
156
156
|
end
|
157
|
+
$?.success?
|
158
|
+
end
|
159
|
+
|
160
|
+
def debug_output(name, text)
|
161
|
+
sep = '-' * 30
|
162
|
+
$stdout.puts "\n\n#{sep}[#{name} begin]#{sep}\n#{text}#{sep}[#{name} end]#{sep}\n\n"
|
157
163
|
end
|
158
164
|
|
159
165
|
WarmupScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count, :first_warmup_duration, :second_warmup_duration) do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark_driver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.15.
|
4
|
+
version: 0.15.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takashi Kokubun
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
146
|
- !ruby/object:Gem::Version
|
147
147
|
version: '0'
|
148
148
|
requirements: []
|
149
|
-
rubygems_version: 3.
|
149
|
+
rubygems_version: 3.2.0.pre1
|
150
150
|
signing_key:
|
151
151
|
specification_version: 4
|
152
152
|
summary: Fully-featured accurate benchmark driver for Ruby
|