benchmark_driver 0.6.2 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +9 -15
- data/examples/eval_blank.rb +12 -0
- data/examples/{exec_interpolation.rb → eval_interpolation.rb} +3 -3
- data/examples/exec_blank.rb +3 -4
- data/examples/exec_blank_simple.rb +13 -0
- data/lib/benchmark/driver.rb +9 -1
- data/lib/benchmark/driver/ruby_dsl_parser.rb +8 -10
- data/lib/benchmark/driver/version.rb +1 -1
- data/lib/benchmark/runner.rb +1 -0
- data/lib/benchmark/runner/eval.rb +152 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c1f7fc0a8d066c0e6fec248737f0a6a0b62cd84b15ed40121f56817a9c6322a
|
4
|
+
data.tar.gz: 95c7c9a3c4a847ac77592c4cd3dbb0a3f2068e83de442bd8184e999dda9979a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0b0e1be74c8e7a8a7b849828daffd6e586a84c3d2a188c1a84882a40ec83ca7717fc66c62d3d723d53b184e582b01cdc14ca74384c3cdfb93e2b80693593706
|
7
|
+
data.tar.gz: b4343234565046d92eb320eb3b5c9ffb7a73560ea9906cd7d541e7f9d065080fecc73f8d6542555abfe95a57e7438e797109acd5412ae6654b0c92631c3a2034
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
# v0.7.0
|
2
|
+
|
3
|
+
- Change Ruby interface for specifying prelude and script
|
4
|
+
- #prelude= is remaed to #prelude
|
5
|
+
- `script:` is no longer a keyword argument
|
6
|
+
- Add Eval runner and it's made default for Ruby interface when script is String
|
7
|
+
|
1
8
|
# v0.6.2
|
2
9
|
|
3
10
|
- Resurrect support of Ruby interface dropped at v0.6.0
|
data/README.md
CHANGED
@@ -42,9 +42,9 @@ This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's go
|
|
42
42
|
```rb
|
43
43
|
require 'benchmark/driver'
|
44
44
|
require 'active_support/all'
|
45
|
-
array = []
|
46
45
|
|
47
46
|
Benchmark.driver do |x|
|
47
|
+
array = []
|
48
48
|
x.report('blank?') { array.blank? }
|
49
49
|
x.report('empty?') { array.empty? }
|
50
50
|
x.compare!
|
@@ -57,15 +57,12 @@ This interface generates code to profile with low overhead and executes it.
|
|
57
57
|
|
58
58
|
```rb
|
59
59
|
require 'benchmark/driver'
|
60
|
+
require 'active_support/all'
|
60
61
|
|
61
62
|
Benchmark.driver do |x|
|
62
|
-
x.prelude =
|
63
|
-
|
64
|
-
|
65
|
-
RUBY
|
66
|
-
|
67
|
-
x.report('blank?', script: 'array.blank?')
|
68
|
-
x.report('empty?', script: 'array.empty?')
|
63
|
+
x.prelude %{ array = [] }
|
64
|
+
x.report 'blank?', %{ array.blank? }
|
65
|
+
x.report 'empty?', %{ array.empty? }
|
69
66
|
end
|
70
67
|
```
|
71
68
|
|
@@ -73,15 +70,12 @@ or simply:
|
|
73
70
|
|
74
71
|
```rb
|
75
72
|
require 'benchmark/driver'
|
73
|
+
require 'active_support/all'
|
76
74
|
|
77
75
|
Benchmark.driver do |x|
|
78
|
-
x.prelude =
|
79
|
-
|
80
|
-
|
81
|
-
RUBY
|
82
|
-
|
83
|
-
x.report(script: 'array.blank?')
|
84
|
-
x.report(script: 'array.empty?')
|
76
|
+
x.prelude %{ array = [] }
|
77
|
+
x.report %{ array.blank?' }
|
78
|
+
x.report %{ array.empty?' }
|
85
79
|
end
|
86
80
|
```
|
87
81
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'benchmark/driver'
|
2
|
+
|
3
|
+
class Array
|
4
|
+
alias_method :blank?, :empty?
|
5
|
+
end
|
6
|
+
|
7
|
+
Benchmark.driver(runner: :eval) do |x|
|
8
|
+
x.prelude %{ array = [] }
|
9
|
+
x.report 'Array#empty?', %{ array.empty? }
|
10
|
+
x.report 'Array#blank?', %{ array.blank? }
|
11
|
+
x.compare!
|
12
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'benchmark/driver'
|
2
2
|
|
3
3
|
Benchmark.driver do |x|
|
4
|
-
x.prelude
|
4
|
+
x.prelude <<-EOS
|
5
5
|
large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
|
6
6
|
large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
|
7
7
|
|
@@ -9,7 +9,7 @@ Benchmark.driver do |x|
|
|
9
9
|
small_b = "World"
|
10
10
|
EOS
|
11
11
|
|
12
|
-
x.report('large',
|
13
|
-
x.report('small',
|
12
|
+
x.report('large', '"#{large_a}, #{large_b}!"')
|
13
|
+
x.report('small', '"#{small_a}, #{small_b}!"')
|
14
14
|
x.compare!
|
15
15
|
end
|
data/examples/exec_blank.rb
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
require 'benchmark/driver'
|
2
2
|
|
3
3
|
Benchmark.driver(runner: :exec) do |x|
|
4
|
-
x.prelude
|
4
|
+
x.prelude <<-EOS
|
5
5
|
class Array
|
6
6
|
alias_method :blank?, :empty?
|
7
7
|
end
|
8
8
|
array = []
|
9
9
|
EOS
|
10
|
-
|
11
|
-
x.report
|
12
|
-
x.report(script: 'array.blank?')
|
10
|
+
x.report 'Array#empty?', %{ array.empty? }
|
11
|
+
x.report 'Array#blank?', %{ array.blank? }
|
13
12
|
x.compare!
|
14
13
|
end
|
data/lib/benchmark/driver.rb
CHANGED
@@ -57,11 +57,19 @@ module Benchmark
|
|
57
57
|
unless config.jobs.all? { |j| j.script.is_a?(script_class) }
|
58
58
|
raise InvalidConfig.new('Benchmark scripts include both String and Proc. Only either of them should be specified.')
|
59
59
|
end
|
60
|
+
|
61
|
+
# TODO: invalidate prelude for call runner
|
60
62
|
end
|
61
63
|
|
62
64
|
def runner_type_for(config)
|
63
65
|
script_class = config.jobs.first.script.class
|
64
|
-
script_class == Proc
|
66
|
+
if script_class == Proc
|
67
|
+
:call
|
68
|
+
elsif config.runner_options.executables_specified?
|
69
|
+
:exec
|
70
|
+
else
|
71
|
+
:eval
|
72
|
+
end
|
65
73
|
end
|
66
74
|
|
67
75
|
# benchmark_driver ouputs logs ASAP. This enables sync flag for it.
|
@@ -22,25 +22,23 @@ class Benchmark::Driver::RubyDslParser
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
# @param [String]
|
26
|
-
def prelude
|
27
|
-
unless
|
28
|
-
raise ArgumentError.new("prelude must be String but got #{
|
25
|
+
# @param [String] prelude_script - Script required for benchmark whose execution time is not measured.
|
26
|
+
def prelude(prelude_script)
|
27
|
+
unless prelude_script.is_a?(String)
|
28
|
+
raise ArgumentError.new("prelude must be String but got #{prelude_script.inspect}")
|
29
29
|
end
|
30
30
|
unless @prelude.nil?
|
31
31
|
raise ArgumentError.new("prelude is already set:\n#{@prelude}")
|
32
32
|
end
|
33
33
|
|
34
|
-
@prelude =
|
34
|
+
@prelude = prelude_script
|
35
35
|
end
|
36
36
|
|
37
37
|
# @param [String,nil] name - Name shown on result output. This must be provided if block is given.
|
38
38
|
# @param [String,nil] script - Benchmarked script in String. Only either of script or block must be provided.
|
39
39
|
# @param [Proc,nil] block - Benchmarked Proc object.
|
40
|
-
def report(name = nil, script
|
41
|
-
if script.nil? &&
|
42
|
-
raise ArgumentError.new('script or block must be provided')
|
43
|
-
elsif !script.nil? && block_given?
|
40
|
+
def report(name = nil, script = nil, &block)
|
41
|
+
if !script.nil? && block_given?
|
44
42
|
raise ArgumentError.new('script and block cannot be specified at the same time')
|
45
43
|
elsif name.nil? && block_given?
|
46
44
|
raise ArgumentError.new('name must be specified if block is given')
|
@@ -50,7 +48,7 @@ class Benchmark::Driver::RubyDslParser
|
|
50
48
|
raise ArgumentError.new("script must be String but got #{script.inspect}")
|
51
49
|
end
|
52
50
|
|
53
|
-
@jobs << Benchmark::Driver::Configuration::Job.new(name
|
51
|
+
@jobs << Benchmark::Driver::Configuration::Job.new(name, script || block || name)
|
54
52
|
end
|
55
53
|
|
56
54
|
def compare!
|
data/lib/benchmark/runner.rb
CHANGED
@@ -0,0 +1,152 @@
|
|
1
|
+
require 'benchmark/driver/benchmark_result'
|
2
|
+
require 'benchmark/driver/duration_runner'
|
3
|
+
require 'benchmark/driver/repeatable_runner'
|
4
|
+
require 'benchmark/driver/time'
|
5
|
+
|
6
|
+
# Run benchmark by calling compiled script on running ruby.
|
7
|
+
#
|
8
|
+
# Multiple Ruby binaries: x
|
9
|
+
# Memory output: x
|
10
|
+
class Benchmark::Runner::Eval
|
11
|
+
# This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
|
12
|
+
SUPPORTED_FIELDS = [:real]
|
13
|
+
|
14
|
+
WARMUP_DURATION = 2
|
15
|
+
BENCHMARK_DURATION = 5
|
16
|
+
GUESS_TIMES = [1, 1_000, 1_000_000, 10_000_000, 100_000_000]
|
17
|
+
GUESS_THRESHOLD = 0.4 # 400ms
|
18
|
+
|
19
|
+
# @param [Benchmark::Driver::Configuration::RunnerOptions] options
|
20
|
+
# @param [Benchmark::Output::*] output - Object that responds to methods used in this class
|
21
|
+
def initialize(options, output:)
|
22
|
+
@options = options
|
23
|
+
@output = output
|
24
|
+
end
|
25
|
+
|
26
|
+
# @param [Benchmark::Driver::Configuration] config
|
27
|
+
def run(config)
|
28
|
+
validate_config(config)
|
29
|
+
|
30
|
+
if config.jobs.any?(&:warmup_needed?)
|
31
|
+
run_warmup(config.jobs)
|
32
|
+
end
|
33
|
+
|
34
|
+
@output.start_running
|
35
|
+
|
36
|
+
config.jobs.each do |job|
|
37
|
+
@output.running(job.name)
|
38
|
+
|
39
|
+
result = Benchmark::Driver::RepeatableRunner.new(job).run(
|
40
|
+
runner: method(:eval_times),
|
41
|
+
repeat_count: @options.repeat_count,
|
42
|
+
)
|
43
|
+
|
44
|
+
@output.benchmark_stats(result)
|
45
|
+
end
|
46
|
+
|
47
|
+
@output.finish
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def validate_config(config)
|
53
|
+
if config.runner_options.executables_specified?
|
54
|
+
raise ArgumentError.new("#{self.class.name} can't run other Ruby executables")
|
55
|
+
end
|
56
|
+
|
57
|
+
config.jobs.each do |job|
|
58
|
+
unless job.script.is_a?(String)
|
59
|
+
raise NotImplementedError.new(
|
60
|
+
"#{self.class.name} only accepts String, but got #{job.script.inspect}"
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# @param [Array<Benchmark::Driver::Configuration::Job>] jobs
|
67
|
+
# @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
|
68
|
+
def run_warmup(jobs)
|
69
|
+
@output.start_warming
|
70
|
+
|
71
|
+
jobs.each do |job|
|
72
|
+
next if job.loop_count
|
73
|
+
@output.warming(job.name)
|
74
|
+
|
75
|
+
result = Benchmark::Driver::DurationRunner.new(job).run(
|
76
|
+
seconds: WARMUP_DURATION,
|
77
|
+
unit_iters: guess_ip100ms(job),
|
78
|
+
runner: method(:eval_times),
|
79
|
+
)
|
80
|
+
job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
|
81
|
+
|
82
|
+
@output.warmup_stats(result)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# @param [Benchmark::Driver::Configuration::Job] job
|
87
|
+
def guess_ip100ms(job)
|
88
|
+
ip100ms = 0
|
89
|
+
GUESS_TIMES.each do |times|
|
90
|
+
seconds = eval_times(job, times)
|
91
|
+
ip100ms = (times.to_f / (seconds * 10.0)).ceil # ceil for times=1
|
92
|
+
if GUESS_THRESHOLD < seconds
|
93
|
+
return ip100ms
|
94
|
+
end
|
95
|
+
end
|
96
|
+
if ip100ms < 0
|
97
|
+
raise Benchmark::Driver::ExecutionTimeTooShort.new(job, GUESS_TIMES.last)
|
98
|
+
end
|
99
|
+
ip100ms
|
100
|
+
end
|
101
|
+
|
102
|
+
def eval_times(job, times)
|
103
|
+
benchmark = BenchmarkScript.new(job.prelude, job.script)
|
104
|
+
mod = Module.new
|
105
|
+
benchmark.compile_overhead!(mod, times)
|
106
|
+
benchmark.compile_full_script!(mod, times)
|
107
|
+
|
108
|
+
before = Benchmark::Driver::Time.now
|
109
|
+
mod.overhead
|
110
|
+
after = Benchmark::Driver::Time.now
|
111
|
+
overhead_duration = after.to_f - before.to_f
|
112
|
+
|
113
|
+
before = Benchmark::Driver::Time.now
|
114
|
+
mod.full_script
|
115
|
+
after = Benchmark::Driver::Time.now
|
116
|
+
full_script_duration = after.to_f - before.to_f
|
117
|
+
|
118
|
+
full_script_duration - overhead_duration
|
119
|
+
end
|
120
|
+
|
121
|
+
class BenchmarkScript < Struct.new(:prelude, :script)
|
122
|
+
BATCH_SIZE = 1000
|
123
|
+
|
124
|
+
def compile_overhead!(mod, times)
|
125
|
+
raise ArgumentError.new("Negative times: #{times}") if times < 0
|
126
|
+
mod.module_eval(<<-RUBY)
|
127
|
+
def self.overhead
|
128
|
+
#{prelude}
|
129
|
+
__benchmark_driver_i = 0
|
130
|
+
while __benchmark_driver_i < #{times / BATCH_SIZE}
|
131
|
+
__benchmark_driver_i += 1
|
132
|
+
end
|
133
|
+
end
|
134
|
+
RUBY
|
135
|
+
end
|
136
|
+
|
137
|
+
def compile_full_script!(mod, times)
|
138
|
+
raise ArgumentError.new("Negative times: #{times}") if times < 0
|
139
|
+
mod.module_eval(<<-RUBY)
|
140
|
+
def self.full_script
|
141
|
+
#{prelude}
|
142
|
+
__benchmark_driver_i = 0
|
143
|
+
while __benchmark_driver_i < #{times / BATCH_SIZE}
|
144
|
+
__benchmark_driver_i += 1
|
145
|
+
#{"#{script};" * BATCH_SIZE}
|
146
|
+
end
|
147
|
+
#{"#{script};" * (times % BATCH_SIZE)}
|
148
|
+
end
|
149
|
+
RUBY
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark_driver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takashi Kokubun
|
@@ -61,8 +61,10 @@ files:
|
|
61
61
|
- examples/call_blank.rb
|
62
62
|
- examples/call_erb.rb
|
63
63
|
- examples/call_interpolation.rb
|
64
|
+
- examples/eval_blank.rb
|
65
|
+
- examples/eval_interpolation.rb
|
64
66
|
- examples/exec_blank.rb
|
65
|
-
- examples/
|
67
|
+
- examples/exec_blank_simple.rb
|
66
68
|
- examples/yaml/array_duration_time.yml
|
67
69
|
- examples/yaml/array_loop.yml
|
68
70
|
- examples/yaml/blank_hash.yml
|
@@ -91,6 +93,7 @@ files:
|
|
91
93
|
- lib/benchmark/output/time.rb
|
92
94
|
- lib/benchmark/runner.rb
|
93
95
|
- lib/benchmark/runner/call.rb
|
96
|
+
- lib/benchmark/runner/eval.rb
|
94
97
|
- lib/benchmark/runner/exec.rb
|
95
98
|
- lib/benchmark_driver.rb
|
96
99
|
homepage: https://github.com/k0kubun/benchmark_driver
|