benchmark_driver 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 28707c030056cf68bac9e42f5de7acbfe8fa2700a4f23c95ee23983a76aba56b
4
- data.tar.gz: 3734f5f19eabda233d2ad189f898c4214282c1fa7d341fafdabf34b026fe31ae
3
+ metadata.gz: 5c1f7fc0a8d066c0e6fec248737f0a6a0b62cd84b15ed40121f56817a9c6322a
4
+ data.tar.gz: 95c7c9a3c4a847ac77592c4cd3dbb0a3f2068e83de442bd8184e999dda9979a9
5
5
  SHA512:
6
- metadata.gz: 275941867a95295a4cd351009a69d84ad38c88d30b1e066a3835581c5d1b6c7c46fcc503e8980e04dcfeaac9d61e9bdbe22bdba56b6b979f3d16fa727372650e
7
- data.tar.gz: e67add21317516a0dc3fbeadfb6cb01723260fdbac1fcfaddc06d53e25b5e309c71fdf3dabd8cc14aadff6e223fba28b5fbdb1bdfb08e3cc1a2fcc074849dd7d
6
+ metadata.gz: c0b0e1be74c8e7a8a7b849828daffd6e586a84c3d2a188c1a84882a40ec83ca7717fc66c62d3d723d53b184e582b01cdc14ca74384c3cdfb93e2b80693593706
7
+ data.tar.gz: b4343234565046d92eb320eb3b5c9ffb7a73560ea9906cd7d541e7f9d065080fecc73f8d6542555abfe95a57e7438e797109acd5412ae6654b0c92631c3a2034
data/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ # v0.7.0
2
+
3
+ - Change Ruby interface for specifying prelude and script
4
+ - #prelude= is remaed to #prelude
5
+ - `script:` is no longer a keyword argument
6
+ - Add Eval runner and it's made default for Ruby interface when script is String
7
+
1
8
  # v0.6.2
2
9
 
3
10
  - Resurrect support of Ruby interface dropped at v0.6.0
data/README.md CHANGED
@@ -42,9 +42,9 @@ This interface is compatible with `Benchmark.bm` and `Benchmark.ips`, so it's go
42
42
  ```rb
43
43
  require 'benchmark/driver'
44
44
  require 'active_support/all'
45
- array = []
46
45
 
47
46
  Benchmark.driver do |x|
47
+ array = []
48
48
  x.report('blank?') { array.blank? }
49
49
  x.report('empty?') { array.empty? }
50
50
  x.compare!
@@ -57,15 +57,12 @@ This interface generates code to profile with low overhead and executes it.
57
57
 
58
58
  ```rb
59
59
  require 'benchmark/driver'
60
+ require 'active_support/all'
60
61
 
61
62
  Benchmark.driver do |x|
62
- x.prelude = <<~RUBY
63
- require 'active_support/all'
64
- array = []
65
- RUBY
66
-
67
- x.report('blank?', script: 'array.blank?')
68
- x.report('empty?', script: 'array.empty?')
63
+ x.prelude %{ array = [] }
64
+ x.report 'blank?', %{ array.blank? }
65
+ x.report 'empty?', %{ array.empty? }
69
66
  end
70
67
  ```
71
68
 
@@ -73,15 +70,12 @@ or simply:
73
70
 
74
71
  ```rb
75
72
  require 'benchmark/driver'
73
+ require 'active_support/all'
76
74
 
77
75
  Benchmark.driver do |x|
78
- x.prelude = <<~RUBY
79
- require 'active_support/all'
80
- array = []
81
- RUBY
82
-
83
- x.report(script: 'array.blank?')
84
- x.report(script: 'array.empty?')
76
+ x.prelude %{ array = [] }
77
+ x.report %{ array.blank?' }
78
+ x.report %{ array.empty?' }
85
79
  end
86
80
  ```
87
81
 
@@ -0,0 +1,12 @@
1
+ require 'benchmark/driver'
2
+
3
+ class Array
4
+ alias_method :blank?, :empty?
5
+ end
6
+
7
+ Benchmark.driver(runner: :eval) do |x|
8
+ x.prelude %{ array = [] }
9
+ x.report 'Array#empty?', %{ array.empty? }
10
+ x.report 'Array#blank?', %{ array.blank? }
11
+ x.compare!
12
+ end
@@ -1,7 +1,7 @@
1
1
  require 'benchmark/driver'
2
2
 
3
3
  Benchmark.driver do |x|
4
- x.prelude = <<-EOS
4
+ x.prelude <<-EOS
5
5
  large_a = "Hellooooooooooooooooooooooooooooooooooooooooooooooooooo"
6
6
  large_b = "Wooooooooooooooooooooooooooooooooooooooooooooooooooorld"
7
7
 
@@ -9,7 +9,7 @@ Benchmark.driver do |x|
9
9
  small_b = "World"
10
10
  EOS
11
11
 
12
- x.report('large', script: '"#{large_a}, #{large_b}!"')
13
- x.report('small', script: '"#{small_a}, #{small_b}!"')
12
+ x.report('large', '"#{large_a}, #{large_b}!"')
13
+ x.report('small', '"#{small_a}, #{small_b}!"')
14
14
  x.compare!
15
15
  end
@@ -1,14 +1,13 @@
1
1
  require 'benchmark/driver'
2
2
 
3
3
  Benchmark.driver(runner: :exec) do |x|
4
- x.prelude = <<-EOS
4
+ x.prelude <<-EOS
5
5
  class Array
6
6
  alias_method :blank?, :empty?
7
7
  end
8
8
  array = []
9
9
  EOS
10
-
11
- x.report(script: 'array.empty?')
12
- x.report(script: 'array.blank?')
10
+ x.report 'Array#empty?', %{ array.empty? }
11
+ x.report 'Array#blank?', %{ array.blank? }
13
12
  x.compare!
14
13
  end
@@ -0,0 +1,13 @@
1
+ require 'benchmark/driver'
2
+
3
+ Benchmark.driver(runner: :exec) do |x|
4
+ x.prelude <<-EOS
5
+ class Array
6
+ alias_method :blank?, :empty?
7
+ end
8
+ array = []
9
+ EOS
10
+ x.report %{ array.empty? }
11
+ x.report %{ array.blank? }
12
+ x.compare!
13
+ end
@@ -57,11 +57,19 @@ module Benchmark
57
57
  unless config.jobs.all? { |j| j.script.is_a?(script_class) }
58
58
  raise InvalidConfig.new('Benchmark scripts include both String and Proc. Only either of them should be specified.')
59
59
  end
60
+
61
+ # TODO: invalidate prelude for call runner
60
62
  end
61
63
 
62
64
  def runner_type_for(config)
63
65
  script_class = config.jobs.first.script.class
64
- script_class == Proc ? :call : :exec
66
+ if script_class == Proc
67
+ :call
68
+ elsif config.runner_options.executables_specified?
69
+ :exec
70
+ else
71
+ :eval
72
+ end
65
73
  end
66
74
 
67
75
  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
@@ -22,25 +22,23 @@ class Benchmark::Driver::RubyDslParser
22
22
  end
23
23
  end
24
24
 
25
- # @param [String] prelude - Script required for benchmark whose execution time is not measured.
26
- def prelude=(prelude)
27
- unless prelude.is_a?(String)
28
- raise ArgumentError.new("prelude must be String but got #{prelude.inspect}")
25
+ # @param [String] prelude_script - Script required for benchmark whose execution time is not measured.
26
+ def prelude(prelude_script)
27
+ unless prelude_script.is_a?(String)
28
+ raise ArgumentError.new("prelude must be String but got #{prelude_script.inspect}")
29
29
  end
30
30
  unless @prelude.nil?
31
31
  raise ArgumentError.new("prelude is already set:\n#{@prelude}")
32
32
  end
33
33
 
34
- @prelude = prelude
34
+ @prelude = prelude_script
35
35
  end
36
36
 
37
37
  # @param [String,nil] name - Name shown on result output. This must be provided if block is given.
38
38
  # @param [String,nil] script - Benchmarked script in String. Only either of script or block must be provided.
39
39
  # @param [Proc,nil] block - Benchmarked Proc object.
40
- def report(name = nil, script: nil, &block)
41
- if script.nil? && !block_given?
42
- raise ArgumentError.new('script or block must be provided')
43
- elsif !script.nil? && block_given?
40
+ def report(name = nil, script = nil, &block)
41
+ if !script.nil? && block_given?
44
42
  raise ArgumentError.new('script and block cannot be specified at the same time')
45
43
  elsif name.nil? && block_given?
46
44
  raise ArgumentError.new('name must be specified if block is given')
@@ -50,7 +48,7 @@ class Benchmark::Driver::RubyDslParser
50
48
  raise ArgumentError.new("script must be String but got #{script.inspect}")
51
49
  end
52
50
 
53
- @jobs << Benchmark::Driver::Configuration::Job.new(name || script, script || block)
51
+ @jobs << Benchmark::Driver::Configuration::Job.new(name, script || block || name)
54
52
  end
55
53
 
56
54
  def compare!
@@ -1,5 +1,5 @@
1
1
  module Benchmark
2
2
  module Driver
3
- VERSION = '0.6.2'
3
+ VERSION = '0.7.0'
4
4
  end
5
5
  end
@@ -10,4 +10,5 @@ module Benchmark::Runner
10
10
  end
11
11
 
12
12
  require 'benchmark/runner/call'
13
+ require 'benchmark/runner/eval'
13
14
  require 'benchmark/runner/exec'
@@ -0,0 +1,152 @@
1
+ require 'benchmark/driver/benchmark_result'
2
+ require 'benchmark/driver/duration_runner'
3
+ require 'benchmark/driver/repeatable_runner'
4
+ require 'benchmark/driver/time'
5
+
6
+ # Run benchmark by calling compiled script on running ruby.
7
+ #
8
+ # Multiple Ruby binaries: x
9
+ # Memory output: x
10
+ class Benchmark::Runner::Eval
11
+ # This class can provide fields in `Benchmark::Driver::BenchmarkResult` if required by output plugins.
12
+ SUPPORTED_FIELDS = [:real]
13
+
14
+ WARMUP_DURATION = 2
15
+ BENCHMARK_DURATION = 5
16
+ GUESS_TIMES = [1, 1_000, 1_000_000, 10_000_000, 100_000_000]
17
+ GUESS_THRESHOLD = 0.4 # 400ms
18
+
19
+ # @param [Benchmark::Driver::Configuration::RunnerOptions] options
20
+ # @param [Benchmark::Output::*] output - Object that responds to methods used in this class
21
+ def initialize(options, output:)
22
+ @options = options
23
+ @output = output
24
+ end
25
+
26
+ # @param [Benchmark::Driver::Configuration] config
27
+ def run(config)
28
+ validate_config(config)
29
+
30
+ if config.jobs.any?(&:warmup_needed?)
31
+ run_warmup(config.jobs)
32
+ end
33
+
34
+ @output.start_running
35
+
36
+ config.jobs.each do |job|
37
+ @output.running(job.name)
38
+
39
+ result = Benchmark::Driver::RepeatableRunner.new(job).run(
40
+ runner: method(:eval_times),
41
+ repeat_count: @options.repeat_count,
42
+ )
43
+
44
+ @output.benchmark_stats(result)
45
+ end
46
+
47
+ @output.finish
48
+ end
49
+
50
+ private
51
+
52
+ def validate_config(config)
53
+ if config.runner_options.executables_specified?
54
+ raise ArgumentError.new("#{self.class.name} can't run other Ruby executables")
55
+ end
56
+
57
+ config.jobs.each do |job|
58
+ unless job.script.is_a?(String)
59
+ raise NotImplementedError.new(
60
+ "#{self.class.name} only accepts String, but got #{job.script.inspect}"
61
+ )
62
+ end
63
+ end
64
+ end
65
+
66
+ # @param [Array<Benchmark::Driver::Configuration::Job>] jobs
67
+ # @return [Hash{ Benchmark::Driver::Configuration::Job => Integer }] iters_by_job
68
+ def run_warmup(jobs)
69
+ @output.start_warming
70
+
71
+ jobs.each do |job|
72
+ next if job.loop_count
73
+ @output.warming(job.name)
74
+
75
+ result = Benchmark::Driver::DurationRunner.new(job).run(
76
+ seconds: WARMUP_DURATION,
77
+ unit_iters: guess_ip100ms(job),
78
+ runner: method(:eval_times),
79
+ )
80
+ job.guessed_count = (result.ips.to_f * BENCHMARK_DURATION).to_i
81
+
82
+ @output.warmup_stats(result)
83
+ end
84
+ end
85
+
86
+ # @param [Benchmark::Driver::Configuration::Job] job
87
+ def guess_ip100ms(job)
88
+ ip100ms = 0
89
+ GUESS_TIMES.each do |times|
90
+ seconds = eval_times(job, times)
91
+ ip100ms = (times.to_f / (seconds * 10.0)).ceil # ceil for times=1
92
+ if GUESS_THRESHOLD < seconds
93
+ return ip100ms
94
+ end
95
+ end
96
+ if ip100ms < 0
97
+ raise Benchmark::Driver::ExecutionTimeTooShort.new(job, GUESS_TIMES.last)
98
+ end
99
+ ip100ms
100
+ end
101
+
102
+ def eval_times(job, times)
103
+ benchmark = BenchmarkScript.new(job.prelude, job.script)
104
+ mod = Module.new
105
+ benchmark.compile_overhead!(mod, times)
106
+ benchmark.compile_full_script!(mod, times)
107
+
108
+ before = Benchmark::Driver::Time.now
109
+ mod.overhead
110
+ after = Benchmark::Driver::Time.now
111
+ overhead_duration = after.to_f - before.to_f
112
+
113
+ before = Benchmark::Driver::Time.now
114
+ mod.full_script
115
+ after = Benchmark::Driver::Time.now
116
+ full_script_duration = after.to_f - before.to_f
117
+
118
+ full_script_duration - overhead_duration
119
+ end
120
+
121
+ class BenchmarkScript < Struct.new(:prelude, :script)
122
+ BATCH_SIZE = 1000
123
+
124
+ def compile_overhead!(mod, times)
125
+ raise ArgumentError.new("Negative times: #{times}") if times < 0
126
+ mod.module_eval(<<-RUBY)
127
+ def self.overhead
128
+ #{prelude}
129
+ __benchmark_driver_i = 0
130
+ while __benchmark_driver_i < #{times / BATCH_SIZE}
131
+ __benchmark_driver_i += 1
132
+ end
133
+ end
134
+ RUBY
135
+ end
136
+
137
+ def compile_full_script!(mod, times)
138
+ raise ArgumentError.new("Negative times: #{times}") if times < 0
139
+ mod.module_eval(<<-RUBY)
140
+ def self.full_script
141
+ #{prelude}
142
+ __benchmark_driver_i = 0
143
+ while __benchmark_driver_i < #{times / BATCH_SIZE}
144
+ __benchmark_driver_i += 1
145
+ #{"#{script};" * BATCH_SIZE}
146
+ end
147
+ #{"#{script};" * (times % BATCH_SIZE)}
148
+ end
149
+ RUBY
150
+ end
151
+ end
152
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: benchmark_driver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.2
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takashi Kokubun
@@ -61,8 +61,10 @@ files:
61
61
  - examples/call_blank.rb
62
62
  - examples/call_erb.rb
63
63
  - examples/call_interpolation.rb
64
+ - examples/eval_blank.rb
65
+ - examples/eval_interpolation.rb
64
66
  - examples/exec_blank.rb
65
- - examples/exec_interpolation.rb
67
+ - examples/exec_blank_simple.rb
66
68
  - examples/yaml/array_duration_time.yml
67
69
  - examples/yaml/array_loop.yml
68
70
  - examples/yaml/blank_hash.yml
@@ -91,6 +93,7 @@ files:
91
93
  - lib/benchmark/output/time.rb
92
94
  - lib/benchmark/runner.rb
93
95
  - lib/benchmark/runner/call.rb
96
+ - lib/benchmark/runner/eval.rb
94
97
  - lib/benchmark/runner/exec.rb
95
98
  - lib/benchmark_driver.rb
96
99
  homepage: https://github.com/k0kubun/benchmark_driver