benchmark_driver 0.10.13 → 0.10.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 888668b113284dcb442c1d9bfb4fc37563b1b73b26bd93c21d6b823e0dcace4e
-  data.tar.gz: 92413919e8a9959fa88137da71cda69a5941bec2988b71fc365d47f08d254b57
+  metadata.gz: e8842fe9609d09ce46061bf5ad452219ec98f1b24ef34020698500160e9823cd
+  data.tar.gz: 69e3714e9bcc55053d198807b17edcc952e6682da5eed337d9a073df85891d79
 SHA512:
-  metadata.gz: e16d5a6c38b9145778e08f045a7022b8b5d648f9f8b66afafa32158fd8bf52ab9c9f2fe7e895ae4fbbbf7ff977fdb7a57b592e6e4e150de1fc256b6cd8fdc446
-  data.tar.gz: 88055b5b9ef159b75540c62b47eb54bf78316e9174e4f7ef9a3e1768fe0151f6a067814cff4b150cf2fa102d309741793d3c72f9c8e0f7b95c2502052b1f7d30
+  metadata.gz: b3c190db1fe084198706797c8d22cb7dba2a077d68e9d02d9466f9c5d26c206a4c03d3db6824d8c2ee3c09e90caf32cbcdd8c8067e75f254bf7c2078d45054f9
+  data.tar.gz: 3b50ec42bbe545db63347f5379f79f84b5b35b109728c8600725d642f3841098dfde78d820badcdf3a3836f4f26d2a6490e463e671c1c367dcbd3e76eb48e2f9

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v0.10.14
+
+- Fix a bug that large time is shown as better in time runner
+
 # v0.10.13
 
 - Add `x.output` to specify output plugin to Ruby interface

data/README.md

@@ -145,14 +145,16 @@ Comparison:
                2.4.1:   1889805.6 i/s - 1.72x  slower
 ```
 
-## Output options
+## Output Options
 
 By default, there are following output options.
 
-* compare: benchmark-ips's `compare!`-like output (default)
-* simple: ruby's original `benchmark/driver.rb`-like simple output
-* markdown: output in markdown table
-* record: serialize results in `benchmark_driver.record.yml`, to change outputs later as you like
+| Output | Description |
+|:-------|:------------|
+| compare | benchmark-ips's `compare!`-like output (default) |
+| simple | ruby's original `benchmark/driver.rb`-like simple output |
+| markdown | output in markdown table |
+| record | serialize results in `benchmark_driver.record.yml`, to change outputs later as you like |
 
 With `benchmark-driver` CLI, you can specify it with `-o [output]` or `--output [output]`.
 
@@ -166,6 +168,199 @@ Benchmark.driver do |x|
 end
 ```
 
+### compare
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --output compare --rbenv '2.4.2;2.5.0'
+Calculating -------------------------------------
+                          2.4.2       2.5.0
+               empty   195.957M    129.970M i/s -     20.000M times in 0.102063s 0.153882s
+               blank    66.554M     55.630M i/s -     20.000M times in 0.300507s 0.359519s
+
+Comparison:
+                            empty
+               2.4.2: 195957398.5 i/s
+               2.5.0: 129969716.9 i/s - 1.51x  slower
+
+                            blank
+               2.4.2:  66554190.0 i/s
+               2.5.0:  55629883.2 i/s - 1.20x  slower
+```
+
+### simple
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --output simple --rbenv '2.4.2;2.5.0'
+benchmark results (i/s):
+          2.4.2     2.5.0
+empty  184.084M  117.319M
+blank   65.843M   62.093M
+```
+
+### markdown
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --output markdown --rbenv '2.4.2;2.5.0'
+```
+
+benchmark results (i/s)
+
+|       |   2.4.2|   2.5.0|
+|:------|:-------|:-------|
+|empty  |187.296M|117.662M|
+|blank  | 58.895M| 58.852M|
+
+### record
+
+Measure first, output with various formats later.
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --output record --rbenv '2.4.2;2.5.0'
+benchmarking....
+
+$ benchmark-driver benchmark_driver.record.yml --output compare
+Calculating -------------------------------------
+               empty   153.380M    114.228M i/s -     20.000M times in 0.130395s 0.175088s
+               blank    67.834M     64.328M i/s -     20.000M times in 0.294836s 0.310906s
+
+Comparison:
+               empty: 153380113.9 i/s
+               blank:  67834321.4 i/s - 2.26x  slower
+
+$ benchmark-driver benchmark_driver.record.yml --output simple
+benchmark results (i/s):
+empty  153.380M  114.228M
+blank   67.834M   64.328M
+```
+
+### gruff
+
+There is [benchmark\_driver-output-gruff](https://github.com/benchmark-driver/benchmark_driver-output-gruff)
+plugin that renders a graph using gruff.gem.
+
+![](./images/optcarrot.png)
+
+### benchmark-driver.github.io
+
+Benchmark results for https://benchmark-driver.github.io are also stored by [output plugin](https://github.com/benchmark-driver/skybench/tree/master/plugins/benchmark_driver-output-skybench).
+You can integrate any system as you like.
+
+## Runner Options
+
+Runner decides metrics to be collected. There are following default runner options.
+
+ips, time, memory, once
+
+| Runner | Description |
+|:-------|:------------|
+| ips | Iteration per second (default) |
+| time | Elapsed seconds |
+| memory | Max resident set. This is supported only on Linux for now. |
+| once | Forces `loop_count` to 1 for testing |
+| command\_stdout | Special runner to integrate existing benchmarks |
+
+### ips
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --runner ips --rbenv '2.4.3;2.5.0'
+Calculating -------------------------------------
+                          2.4.3       2.5.0
+               empty   228.802M    180.125M i/s -     20.000M times in 0.087412s 0.111034s
+               blank    90.012M     88.853M i/s -     20.000M times in 0.222193s 0.225090s
+
+Comparison:
+                            empty
+               2.4.3: 228801720.5 i/s
+               2.5.0: 180124821.8 i/s - 1.27x  slower
+
+                            blank
+               2.4.3:  90012021.7 i/s
+               2.5.0:  88853269.4 i/s - 1.01x  slower
+```
+
+### time
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --runner time --rbenv '2.4.3;2.5.0'
+Calculating -------------------------------------
+                          2.4.3       2.5.0
+               empty      0.087       0.110 s -     20.000M times
+               blank      0.217       0.219 s -     20.000M times
+
+Comparison:
+                            empty
+               2.5.0:         0.1 s
+               2.4.3:         0.1 s - 1.26x  slower
+
+                            blank
+               2.5.0:         0.2 s
+               2.4.3:         0.2 s - 1.01x  slower
+```
+
+### memory
+
+```
+$ be exe/benchmark-driver examples/yaml/blank_loop.yml --runner memory --rbenv '2.4.3;2.5.0'
+Calculating -------------------------------------
+                          2.4.3       2.5.0
+               empty     9.192M      9.364M bytes -     20.000M times
+               blank     9.080M      9.372M bytes -     20.000M times
+
+Comparison:
+                            empty
+               2.4.3:   9192000.0 bytes
+               2.5.0:   9364000.0 bytes - 1.02x  larger
+
+                            blank
+               2.4.3:   9080000.0 bytes
+               2.5.0:   9372000.0 bytes - 1.03x  larger
+```
+
+### once
+
+Only for testing purpose.
+
+```
+$ benchmark-driver examples/yaml/blank_loop.yml --runner once --rbenv '2.4.3;2.5.0'
+Calculating -------------------------------------
+                          2.4.3       2.5.0
+               empty     1.818M      2.681M i/s -       1.000 times in 0.000001s 0.000000s
+               blank     1.531M      2.421M i/s -       1.000 times in 0.000001s 0.000000s
+
+Comparison:
+                            empty
+               2.5.0:   2680965.1 i/s
+               2.4.3:   1818181.8 i/s - 1.47x  slower
+
+                            blank
+               2.5.0:   2421307.5 i/s
+               2.4.3:   1531393.6 i/s - 1.58x  slower
+```
+
+### command\_stdout
+
+See following examples:
+
+* https://github.com/benchmark-driver/optcarrot
+* https://github.com/benchmark-driver/fluentd-benchmark
+
+If you benchmark can run with `ruby foo bar`, specify `foo bar` to `command:`.
+Then write `stdout_to_metrics:` to convert stdout to metrics. This runner can be used only with YAML interface for now.
+
+```
+$ benchmark-driver benchmark.yml --verbose 1 --rbenv '2.6.0-dev;2.6.0-dev,--jit'
+2.6.0-dev: ruby 2.6.0dev (2018-03-21 trunk 62870) [x86_64-linux]
+2.6.0-dev,--jit: ruby 2.6.0dev (2018-03-21 trunk 62870) +JIT [x86_64-linux]
+Calculating -------------------------------------
+                      2.6.0-dev  2.6.0-dev,--jit
+           optcarrot     51.866           67.445 fps
+
+Comparison:
+                        optcarrot
+     2.6.0-dev,--jit:        67.4 fps
+           2.6.0-dev:        51.9 fps - 1.30x  slower
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/k0kubun/benchmark_driver.

data/lib/benchmark_driver/output/compare.rb

@@ -104,8 +104,8 @@ class BenchmarkDriver::Output::Compare
   end
 
   def humanize(value, width = 10)
-    if value < 0
-      raise ArgumentError.new("Negative value: #{value.inspect}")
+    if value <= 0
+      raise ArgumentError.new("Non positive value: #{value.inspect}")
     end
 
     scale = (Math.log10(value) / 3).to_i

data/lib/benchmark_driver/runner/time.rb

@@ -6,7 +6,7 @@ class BenchmarkDriver::Runner::Time < BenchmarkDriver::Runner::Ips
   # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
   JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
 
-  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 's')
+  METRICS_TYPE = BenchmarkDriver::Metrics::Type.new(unit: 's', larger_better: false)
 
   # Overriding BenchmarkDriver::Runner::Ips#set_metrics_type
   def set_metrics_type

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.10.13'
+  VERSION = '0.10.14'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.10.13
+  version: 0.10.14
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-21 00:00:00.000000000 Z
+date: 2018-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -68,6 +68,7 @@ files:
 - examples/yaml/example_multi.yml
 - examples/yaml/example_single.yml
 - exe/benchmark-driver
+- images/optcarrot.png
 - lib/benchmark_driver.rb
 - lib/benchmark_driver/config.rb
 - lib/benchmark_driver/default_job.rb
@@ -110,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby