benchmark_driver 0.10.0 → 0.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +0 -1
- data/CHANGELOG.md +8 -0
- data/README.md +3 -1
- data/Rakefile +6 -0
- data/benchmark_driver.gemspec +1 -1
- data/exe/benchmark-driver +1 -2
- data/lib/benchmark_driver/output/compare.rb +1 -1
- data/lib/benchmark_driver/runner.rb +1 -0
- data/lib/benchmark_driver/runner/command_stdout.rb +121 -0
- data/lib/benchmark_driver/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d2bcdf477d225e851726d6e8296af8b2ace8542c9b3cf3bc0d0d076dc140c8cb
|
4
|
+
data.tar.gz: 0d13f33d1fe60a936601377686a3121eee98972d587ad2a13e1828fa5a6a6588
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bca45d1c0fd8608aa223aa02ce0e1873f6e9bd2a04edb9efebab5dddff0f68be77be3a318613fda3af152e197e3298d97959e7720daa3b2b4ea5e6fffc8ac7b3
|
7
|
+
data.tar.gz: '0908488a1465b40d808dfa9f3ead01d1b0647106ed8f88328e3aa1dd3a43d13ad90946f40b81094ba5327b6ff480d7f19060c814749bbe63731cbb0871170437'
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
# v0.10.1
|
2
|
+
|
3
|
+
- Add "command\_stdout" runner to plug in existing benchmark
|
4
|
+
- Explicitly bump supported Ruby version to >= 2.2
|
5
|
+
- v0.10.0 actually does not work with 2.1
|
6
|
+
- You can still benchmark Ruby 2.0, 2.1 by --executable, but you need to use newer Ruby for driver
|
7
|
+
|
1
8
|
# v0.10.0
|
2
9
|
|
3
10
|
- Add "record" output and "recorded" runner
|
@@ -24,6 +31,7 @@
|
|
24
31
|
- So Ruby interface can't take Proc
|
25
32
|
- YAML can have arbitrary format depending on the runner
|
26
33
|
- `--compare` option is dropped and changed to `--output compare`
|
34
|
+
- `--dir` option is dropped for now
|
27
35
|
|
28
36
|
# v0.8.6
|
29
37
|
|
data/README.md
CHANGED
data/Rakefile
CHANGED
@@ -12,7 +12,9 @@ task :test do
|
|
12
12
|
}.each do |runner, output|
|
13
13
|
Bundler.with_clean_env do
|
14
14
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', runner, '-o', output].shelljoin
|
15
|
+
puts
|
15
16
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_hash, '-r', runner, '-o', output, '--run-duration', '1'].shelljoin
|
17
|
+
puts
|
16
18
|
end
|
17
19
|
end
|
18
20
|
end
|
@@ -20,9 +22,13 @@ end
|
|
20
22
|
task :test_record do
|
21
23
|
blank_loop = File.expand_path('./examples/yaml/blank_loop.yml', __dir__) # no warmup
|
22
24
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', blank_loop, '-r', 'ips', '-o', 'record'].shelljoin
|
25
|
+
puts
|
23
26
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'compare'].shelljoin
|
27
|
+
puts
|
24
28
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'record'].shelljoin
|
29
|
+
puts
|
25
30
|
sh ['time', 'bundle', 'exec', 'exe/benchmark-driver', 'benchmark_driver.record.yml', '-o', 'simple'].shelljoin
|
31
|
+
puts
|
26
32
|
end
|
27
33
|
|
28
34
|
task :test_ruby do
|
data/benchmark_driver.gemspec
CHANGED
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.bindir = 'exe'
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ['lib']
|
22
|
-
spec.required_ruby_version = '>= 2.
|
22
|
+
spec.required_ruby_version = '>= 2.2.0'
|
23
23
|
|
24
24
|
spec.add_development_dependency 'bundler'
|
25
25
|
spec.add_development_dependency 'rake'
|
data/exe/benchmark-driver
CHANGED
@@ -5,7 +5,6 @@ require 'benchmark_driver'
|
|
5
5
|
require 'optparse'
|
6
6
|
require 'yaml'
|
7
7
|
|
8
|
-
|
9
8
|
# Parse command line options
|
10
9
|
config = BenchmarkDriver::Config.new.tap do |c|
|
11
10
|
executables = []
|
@@ -34,7 +33,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
|
|
34
33
|
r.split(';').each do |spec|
|
35
34
|
version, *args = spec.split(',')
|
36
35
|
executables << BenchmarkDriver::Config::Executable.new(
|
37
|
-
name:
|
36
|
+
name: spec,
|
38
37
|
command: [BenchmarkDriver::Rbenv.ruby_path(version), *args],
|
39
38
|
)
|
40
39
|
end
|
@@ -55,7 +55,7 @@ class BenchmarkDriver::Output::Compare
|
|
55
55
|
block.call
|
56
56
|
ensure
|
57
57
|
$stdout.print(@metrics_type.unit)
|
58
|
-
if job.loop_count
|
58
|
+
if job.respond_to?(:loop_count) && job.loop_count
|
59
59
|
$stdout.print(" - #{humanize(job.loop_count)} times")
|
60
60
|
if @job_metrics.all? { |metrics| metrics.duration }
|
61
61
|
$stdout.print(" in")
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'benchmark_driver/struct'
|
2
|
+
require 'benchmark_driver/metrics'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'shellwords'
|
5
|
+
|
6
|
+
# Run only once, for testing
|
7
|
+
class BenchmarkDriver::Runner::CommandStdout
|
8
|
+
# JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
|
9
|
+
Job = ::BenchmarkDriver::Struct.new(
|
10
|
+
:name, # @param [String] name - This is mandatory for all runner
|
11
|
+
:command, # @param [Array<String>]
|
12
|
+
:working_directory, # @param [String,NilClass]
|
13
|
+
:metrics_type, # @param [BenchmarkDriver::Metrics::Type]
|
14
|
+
:stdout_to_metrics, # @param [String]
|
15
|
+
)
|
16
|
+
# Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
|
17
|
+
class << JobParser = Module.new
|
18
|
+
# @param [String] name
|
19
|
+
# @param [String] command
|
20
|
+
# @param [String,NilClass] working_directory
|
21
|
+
# @param [Hash] metrics_type
|
22
|
+
# @param [String] stdout_to_metrics
|
23
|
+
def parse(name:, command:, working_directory:, metrics_type:, stdout_to_metrics:)
|
24
|
+
Job.new(
|
25
|
+
name: name,
|
26
|
+
command: command.shellsplit,
|
27
|
+
working_directory: working_directory,
|
28
|
+
metrics_type: parse_metrics_type(metrics_type),
|
29
|
+
stdout_to_metrics: stdout_to_metrics,
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def parse_metrics_type(unit:, larger_better: nil, worse_word: nil)
|
36
|
+
BenchmarkDriver::Metrics::Type.new(
|
37
|
+
unit: unit,
|
38
|
+
larger_better: larger_better,
|
39
|
+
worse_word: worse_word,
|
40
|
+
)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# @param [BenchmarkDriver::Config::RunnerConfig] config
|
45
|
+
# @param [BenchmarkDriver::Output::*] output
|
46
|
+
def initialize(config:, output:)
|
47
|
+
@config = config
|
48
|
+
@output = output
|
49
|
+
end
|
50
|
+
|
51
|
+
# This method is dynamically called by `BenchmarkDriver::JobRunner.run`
|
52
|
+
# @param [Array<BenchmarkDriver::Default::Job>] jobs
|
53
|
+
def run(jobs)
|
54
|
+
metrics_type = jobs.first.metrics_type
|
55
|
+
@output.metrics_type = metrics_type
|
56
|
+
|
57
|
+
@output.with_benchmark do
|
58
|
+
jobs.each do |job|
|
59
|
+
@output.with_job(job) do
|
60
|
+
@config.executables.each do |exec|
|
61
|
+
best_value = with_repeat(metrics_type) do
|
62
|
+
stdout = with_chdir(job.working_directory) do
|
63
|
+
execute(*exec.command, *job.command)
|
64
|
+
end
|
65
|
+
StdoutToMetrics.new(
|
66
|
+
stdout: stdout,
|
67
|
+
stdout_to_metrics: job.stdout_to_metrics,
|
68
|
+
).metrics_value
|
69
|
+
end
|
70
|
+
|
71
|
+
@output.report(
|
72
|
+
BenchmarkDriver::Metrics.new(
|
73
|
+
value: best_value,
|
74
|
+
executable: exec,
|
75
|
+
)
|
76
|
+
)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def with_chdir(working_directory, &block)
|
86
|
+
if working_directory
|
87
|
+
Dir.chdir(working_directory) { block.call }
|
88
|
+
else
|
89
|
+
block.call
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def execute(*args)
|
94
|
+
stdout = IO.popen(args, &:read)
|
95
|
+
unless $?.success?
|
96
|
+
raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
|
97
|
+
end
|
98
|
+
stdout
|
99
|
+
end
|
100
|
+
|
101
|
+
# Return multiple times and return the best metrics
|
102
|
+
def with_repeat(metrics_type, &block)
|
103
|
+
values = @config.repeat_count.times.map do
|
104
|
+
block.call
|
105
|
+
end
|
106
|
+
values.sort_by do |value|
|
107
|
+
if metrics_type.larger_better
|
108
|
+
value
|
109
|
+
else
|
110
|
+
-value
|
111
|
+
end
|
112
|
+
end.last
|
113
|
+
end
|
114
|
+
|
115
|
+
StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :stdout_to_metrics) do
|
116
|
+
def metrics_value
|
117
|
+
eval(stdout_to_metrics, binding)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
private_constant :StdoutToMetrics
|
121
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark_driver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takashi Kokubun
|
@@ -82,6 +82,7 @@ files:
|
|
82
82
|
- lib/benchmark_driver/rbenv.rb
|
83
83
|
- lib/benchmark_driver/ruby_interface.rb
|
84
84
|
- lib/benchmark_driver/runner.rb
|
85
|
+
- lib/benchmark_driver/runner/command_stdout.rb
|
85
86
|
- lib/benchmark_driver/runner/ips.rb
|
86
87
|
- lib/benchmark_driver/runner/memory.rb
|
87
88
|
- lib/benchmark_driver/runner/once.rb
|
@@ -101,7 +102,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
101
102
|
requirements:
|
102
103
|
- - ">="
|
103
104
|
- !ruby/object:Gem::Version
|
104
|
-
version: 2.
|
105
|
+
version: 2.2.0
|
105
106
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
107
|
requirements:
|
107
108
|
- - ">="
|