benchmark-interface 0.1

data/examples/clamp.rb

@@ -0,0 +1,17 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+def clamp_a(min, value, max)
+  [min, value, max].sort[1]
+end
+
+def clamp_b(min, value, max)
+  return min if value < min
+  return max if value > max
+  value
+end

data/examples/interface.rb

@@ -0,0 +1,13 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require File.expand_path('clamp', File.dirname(__FILE__))
+
+benchmark { clamp_a(10, 40, 90) }
+benchmark('clamp_a') { clamp_a(10, 40, 90) }
+benchmark('clamp_b') { clamp_b(10, 40, 90) }

data/examples/long.rb

@@ -0,0 +1,9 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+benchmark('sleep') { sleep 3 }

data/examples/mri.rb

@@ -0,0 +1,15 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+array = [1, 2, 3]
+
+i = 0
+while i < 30_000_000
+  i += 1
+  array.length
+end

data/examples/perfer.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'perfer'
+
+require File.expand_path('clamp', File.dirname(__FILE__))
+
+Perfer.session 'clamp' do |s|
+  s.iterate 'clamp_a' do |n|
+    n.times do
+      clamp_a(10, 40, 90)
+    end
+  end
+
+  s.iterate 'clamp_b' do |n|
+    n.times do
+      clamp_b(10, 40, 90)
+    end
+  end
+end

data/examples/rbench.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'rbench'
+
+require File.expand_path('clamp', File.dirname(__FILE__))
+
+RBench.run(10_000) do
+  format :width => 65
+
+  group 'clamp1' do
+    report 'clamp_a1' do
+      clamp_a(10, 40, 90)
+    end
+    report 'clamp_b1' do
+      clamp_a(10, 40, 90)
+    end
+  end
+end
+
+RBench.run(10_000) do
+  format :width => 65
+
+  column :times
+  column :one,  :title => "#1"
+  column :two,  :title => "#2"
+  column :diff, :title => "#1/#2", :compare => [:one,:two]
+
+  group 'clamp2' do
+    report 'clamp_a2' do
+      one { clamp_a(10, 40, 90) }
+      two { clamp_a(10, 40, 90) }
+    end
+    report 'clamp_b2' do
+      one { clamp_a(10, 40, 90) }
+      two { clamp_a(10, 40, 90) }
+    end
+    
+    summary 'all methods (totals)'
+  end
+end

data/examples/readme.rb

@@ -0,0 +1,12 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'benchmark-interface'
+
+benchmark('mul') { '14 * 14 * 14' }
+benchmark('pow') { '14 ** 3' }

data/examples/script.rb

@@ -0,0 +1,14 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'benchmark-interface'
+
+require File.expand_path('clamp', File.dirname(__FILE__))
+
+benchmark('clamp_a') { clamp_a(10, 40, 90) }
+benchmark('clamp_b') { clamp_b(10, 40, 90) }

data/lib/benchmark-interface.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'benchmark-interface/version'
+require 'benchmark-interface/benchmark'
+require 'benchmark-interface/benchmark-set'
+require 'benchmark-interface/frontends/mri'
+require 'benchmark-interface/backends/simple'
+require 'benchmark-interface/backends/benchmark'
+require 'benchmark-interface/backends/bips'
+require 'benchmark-interface/backends/bench9000'
+require 'benchmark-interface/require'
+require 'benchmark-interface/run'
+
+module BenchmarkInterface
+  
+  def self.benchmark(name=nil, &block)
+    BenchmarkInterface::BenchmarkSet.current.register name, block
+  end
+  
+end
+
+def benchmark(name=nil, &block)
+  BenchmarkInterface.benchmark name, &block
+end
+
+if $PROGRAM_NAME.split('/').last != 'benchmark'
+  set = BenchmarkInterface::BenchmarkSet.new
+  backend = BenchmarkInterface::Backends::Bips
+
+  at_exit do
+    set.prepare
+    backend.run set, set.benchmarks, {}
+  end
+end

data/lib/benchmark-interface/backends/bench9000.rb

@@ -0,0 +1,87 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module BenchmarkInterface
+  module Backends
+    module Bench9000
+
+      MIN_SAMPLING_TIME = 0.1 # seconds
+
+      def self.run(benchmark_set, names, options)
+        unless names.size == 1
+          abort 'The bench9000 backend only works when you run just one benchmark at a time - specify the name on the command line'
+        end
+
+        unless options['--no-scale']
+          min_time = benchmark_set.benchmarks.map(&:basic_iteration_time).min
+
+          if min_time < MIN_SAMPLING_TIME
+            short_iterations = true
+            samples = (MIN_SAMPLING_TIME / min_time / MIN_SAMPLING_TIME).to_i
+            puts "These are short benchmarks - we're running each #{samples} times so they take about a second and using the micro harness"
+          end
+        end
+
+        benchmark = benchmark_set.benchmark(names.first)
+        block = benchmark.block
+
+        Object.instance_eval do
+          if short_iterations
+            define_method(:micro_harness_input) do
+              nil
+            end
+
+            define_method(:micro_harness_iterations) do
+              samples
+            end
+
+            define_method(:micro_harness_sample) do |input|
+              block.call
+            end
+
+            define_method(:micro_harness_expected) do
+              raise 'not expecting this to be called, as we\'ve patched harness_verify'
+            end
+          else
+            define_method(:harness_input) do
+              nil
+            end
+
+            define_method(:harness_sample) do |input|
+              block.call
+            end
+
+            define_method(:harness_verify) do |output|
+              true
+            end
+          end
+
+        end
+        
+        @loading_real = true
+
+        if short_iterations
+          benchmark_interface_original_require 'bench9000/micro-harness'
+
+          Object.instance_eval do
+            define_method(:harness_verify) do |output|
+              true
+            end
+          end
+        end
+
+        benchmark_interface_original_require 'bench9000/harness'
+      end
+      
+      def self.loading_real?
+        @loading_real
+      end
+
+    end
+  end
+end

data/lib/benchmark-interface/backends/benchmark.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module BenchmarkInterface
+  module Backends
+    module Bm
+
+      MIN_SAMPLING_TIME = 0.1 # seconds
+
+      def self.run(benchmark_set, names, options)
+        # If we don't remove these we'll get a warning when we load the real
+        # implementation.
+          
+        ::Benchmark.send(:remove_const, :CAPTION) if defined?(::Benchmark::CAPTION)
+        ::Benchmark.send(:remove_const, :FORMAT)  if defined?(::Benchmark::FORMAT)
+
+        benchmark_interface_original_require 'benchmark'
+
+        unless options['--no-scale']
+          min_time = benchmark_set.benchmarks.map(&:basic_iteration_time).min
+
+          if min_time < MIN_SAMPLING_TIME
+            short_iterations = true
+            samples = (MIN_SAMPLING_TIME / min_time / MIN_SAMPLING_TIME).to_i
+            puts "These are short benchmarks - we're running each #{samples} times so they take about a second"
+          end
+        end
+
+        label_width = benchmark_set.benchmarks(names).map(&:name).map(&:size).max
+
+        block = Proc.new do |x|
+          benchmark_set.benchmarks(names).each do |benchmark|
+            block = benchmark.block
+            if short_iterations
+              x.report(benchmark.name) do
+                samples.times do
+                  block.call
+                end
+              end
+            else
+              x.report(benchmark.name, &benchmark.block)
+            end
+          end
+        end
+
+        if self == BmBm
+          ::Benchmark.bmbm label_width, &block
+        else
+          ::Benchmark.bm label_width, &block
+        end
+      end
+
+    end
+
+    module BmBm
+
+      def self.run(benchmark_set, names, options)
+        Bm.run benchmark_set, names, options
+      end
+
+    end
+  end
+end

data/lib/benchmark-interface/backends/bips.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module BenchmarkInterface
+  module Backends
+    module Bips
+
+      LONG_ITERATION_THRESHOLD = 0.1 # seconds
+      
+      def self.run(benchmark_set, names, options)
+        Kernel.instance_eval do
+          alias_method :require, :benchmark_interface_original_require
+          require 'rubygems'
+          alias_method :benchmark_interface_original_require, :require
+        end
+
+        benchmark_interface_original_require 'benchmark/ips'
+
+        unless options['--no-scale']
+          if benchmark_set.benchmarks.map(&:basic_iteration_time).max > LONG_ITERATION_THRESHOLD
+            long_iterations = true
+            puts "These are long benchmarks - we're increasing warmup and sample time"
+          end
+        end
+
+        ::Benchmark.ips do |x|
+          x.iterations = 3
+
+          if long_iterations
+            x.time = 10
+            x.warmup = 10
+          end
+
+          benchmark_set.benchmarks(names).each do |benchmark|
+            x.report benchmark.name, &benchmark.block
+          end
+
+          x.compare!
+        end
+      end
+      
+    end
+  end
+end

data/lib/benchmark-interface/backends/simple.rb

@@ -0,0 +1,38 @@
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module BenchmarkInterface
+  module Backends
+    module Simple
+
+      def self.run(benchmark_set, names, options)
+        loop_time = options['--time']
+        
+        benchmark_set.benchmarks(names).each do |benchmark|
+          puts benchmark.name
+          block = benchmark.block
+
+          start_time = Time.now
+
+          while Time.now - start_time < loop_time
+            start_iteration_time = Time.now
+            iterations = 0
+            while Time.now - start_iteration_time < 1
+              block.call
+              iterations += 1
+            end
+            iteration_time = Time.now - start_iteration_time
+            iterations *= benchmark_set.iterations
+            puts iterations / iteration_time
+          end
+        end
+      end
+
+    end
+  end
+end