bench9000 0.1

data/lib/bench9000/commands/score.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+# 
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module Bench
+  module Commands
+
+    class Score < Command
+
+      def result(options, b, i, measurement)
+        if measurement == :failed
+          score = "failed"
+        else
+          score = measurement.score
+        end
+
+        puts "#{b} #{i} #{score}"
+      end
+
+    end
+
+  end
+end

data/lib/bench9000/config.rb

@@ -0,0 +1,88 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module Bench
+
+  class Config
+
+    attr_reader :implementations
+    attr_reader :implementation_groups
+    attr_reader :benchmarks
+    attr_reader :benchmark_groups
+    attr_reader :fail_hard_exclusions
+    attr_reader :commands
+
+    def initialize
+      @implementations = {}
+      @implementation_groups = {}
+      @benchmarks = {}
+      @benchmark_groups = {}
+      @fail_hard_exclusions = []
+      @commands = {}
+    end
+
+    def load(file)
+      @configuration_file_path = file
+      eval File.read(file), binding, file, 1
+    end
+
+    def rbenv(name, version=name, flags="")
+      @implementations[name] = RbenvImplementation.new(name, version, flags)
+    end
+
+    def binary(name, binary, flags="")
+      @implementations[name] = BinaryImplementation.new(name, binary, flags)
+    end
+
+    def implementation_group(name, *implementations)
+      @implementation_groups[name] = Group.new(name, implementations.map { |i|
+        implementation = @implementations[i]
+
+        if implementation.nil?
+          puts "unknown implementation #{i} in group"
+          exit
+        end
+
+        implementation
+      })
+    end
+
+    def default_benchmarks_dir
+      File.expand_path(File.dirname(@configuration_file_path))
+    end
+
+    def benchmark(name, file, flags="")
+      @benchmarks[name] = Benchmark.new(name, file, flags)
+    end
+
+    def benchmark_group(name, *benchmarks)
+      @benchmark_groups[name] = Group.new(name, benchmarks.map { |b|
+        benchmark = @benchmarks[b]
+
+        if benchmark.nil?
+          puts "unknown benchmark #{b} in group"
+          exit
+        end
+
+        benchmark
+      })
+    end
+
+    def fails_hard(implementation, benchmark)
+      @fail_hard_exclusions.push([implementation, benchmark])
+    end
+
+    def command(name, &body)
+      command_class = Class.new(Commands::Command)
+      command_class.class_eval(&body)
+      @commands[name] = command_class
+    end
+
+  end
+
+end

data/lib/bench9000/group.rb

@@ -0,0 +1,23 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+# 
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module Bench
+
+  class Group
+
+    attr_reader :name
+    attr_reader :members
+
+    def initialize(name, members)
+      @name = name
+      @members = members
+    end
+
+  end
+
+end

data/lib/bench9000/harness.rb

@@ -0,0 +1,68 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+# 
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+# This file should be kept as simple as possible to accommodate early
+# implementations of Ruby.
+
+if ARGV[0] == "--loopn"
+  ITERATIONS = ARGV[1].to_i
+  INNER_ITERATIONS = ARGV[2].to_i
+
+  def micro_harness_iterations
+    INNER_ITERATIONS
+  end
+end
+
+begin
+  Process.clock_gettime(Process::CLOCK_MONOTONIC) # test
+  def bench9000_get_time
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+rescue Exception
+  def bench9000_get_time
+    Time.now
+  end
+end
+
+iteration = 0
+
+while true
+  input = harness_input
+
+  start = bench9000_get_time
+  actual_output = harness_sample(input)
+  time = bench9000_get_time - start
+
+  unless harness_verify(actual_output)
+    puts "error"
+    exit
+  end
+
+  puts time
+
+  # Some implementations of Ruby will not flush each line when writing to a pipe
+
+  begin
+    STDOUT.flush
+  rescue Errno::EPIPE
+    # Not the cleanest solution, but if the previous `puts` call reaches the parent and the parent exits before the
+    # `flush` call is encountered, we may end up writing to a closed pipe.  If the parent has closed, there's not much
+    # can do, so we'll just exit and hope for the best.
+    break
+  end
+
+  if ARGV[0] == "--loop"
+    next
+  elsif ARGV[0] == "--loopn"
+    iteration += 1
+    break if iteration == ITERATIONS
+  else
+    command = gets
+    break unless command == "continue" || command == "continue\n"
+  end
+end

data/lib/bench9000/implementation.rb

@@ -0,0 +1,134 @@
+# Copyright (c) 2014, 2015 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module Bench
+
+  class Implementation
+
+    BEFORE_WARMUP_TIME = 30
+    WARMUP_WINDOW_SIZE = 20
+    WARMED_UP_RELATIVE_RANGE = 0.1
+    MAX_WARMUP = 100
+    MAX_WARMUP_TIME = 4 * 60
+    SAMPLES_COUNT = 10
+
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def measure(flags, benchmark)
+      command = "bash -c \"#{command(benchmark)}\""
+
+      puts command if flags.has_key? "--show-commands"
+
+      warming_up = true
+      warmup_window = []
+      warmup_samples = []
+      samples = []
+
+      overall_time = Time.now
+
+      IO.popen command, "r+" do |subprocess|
+        while true
+          line = subprocess.gets
+
+          if line.nil? || line == "error"
+            return :failed
+          end
+
+          unless line.match(/\d+\.\d+/)
+            STDERR.puts line
+            next
+          end
+
+          time = line.to_f
+          puts time if flags.has_key? "--show-samples"
+
+          elapsed_time = Time.now - overall_time
+
+          if elapsed_time < BEFORE_WARMUP_TIME
+            warmup_samples.push time
+
+            subprocess.puts "continue"
+          elsif warming_up
+            warmup_samples.push time
+
+            warmup_window.shift if warmup_window.size == WARMUP_WINDOW_SIZE
+            warmup_window.push time
+            window_relative_range = Stats.range(warmup_window) / Stats.mean(warmup_window)
+
+            if warmup_window.size == WARMUP_WINDOW_SIZE && window_relative_range < WARMED_UP_RELATIVE_RANGE
+              warming_up = false
+              warmup_samples = warmup_samples.reverse.drop(WARMUP_WINDOW_SIZE).reverse
+              samples = warmup_window
+            elsif warmup_samples.size > MAX_WARMUP || elapsed_time > MAX_WARMUP_TIME
+              puts "warning: #{@name} #{benchmark} never warmed up!"
+              warming_up = false
+            end
+
+            subprocess.puts "continue"
+          else
+            samples.push time
+
+            if samples.size < SAMPLES_COUNT
+              subprocess.puts "continue"
+            else
+              subprocess.puts "stop"
+              break
+            end
+          end
+        end
+      end
+
+      raise "not enough warmup samples" if warmup_samples.nil?
+      raise "not enough samples" if samples.nil? || samples.size < SAMPLES_COUNT
+
+      Measurement.new(warmup_samples, samples)
+    end
+
+    def to_s
+      @name
+    end
+
+  end
+
+  HARNESS_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+  class RbenvImplementation < Implementation
+
+    RBENV_ROOT = `rbenv root`.chop
+
+    def initialize(name, version, flags)
+      @name = name
+      @version = version
+      @flags = flags
+    end
+
+    def command(benchmark)
+      "#{RBENV_ROOT}/versions/#{@version}/bin/ruby #{@flags} -I#{HARNESS_DIR} #{benchmark.flags} #{benchmark.file}"
+    end
+
+  end
+
+  class BinaryImplementation < Implementation
+
+    def initialize(name, binary, flags)
+      @name = name
+      @binary = binary
+      @flags = flags
+    end
+
+    def command(benchmark)
+      "#{@binary} #{@flags} -I#{HARNESS_DIR} #{benchmark.flags} #{benchmark.file}"
+    end
+
+  end
+
+end

data/lib/bench9000/json-formatter.rb

@@ -0,0 +1,50 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+# 
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'json'
+
+module Bench
+
+  module JSONFormatter
+
+    def self.format(options, measurements)
+        benchmarks = options.benchmarks
+        implementations = options.implementations
+
+        JSON.pretty_generate({
+          benchmarks: benchmarks,
+          implementations: implementations,
+          measurements: benchmarks.product(implementations).map do |b, i|
+            measurement = measurements[b, i]
+
+            if measurement == :failed
+              {
+                benchmark: b,
+                implementation: i,
+                failed: true
+              }
+            else
+              {
+                benchmark: b,
+                implementation: i,
+                warmup_time: measurement.warmup_time,
+                sample_mean: measurement.sample_mean,
+                sample_error: measurement.sample_error,
+                score: measurement.score,
+                score_error: measurement.score_error,
+                warmup_samples: measurement.warmup_samples,
+                samples: measurement.samples
+              }
+            end
+          end
+        })
+    end
+
+  end
+
+end

data/lib/bench9000/main.rb

@@ -0,0 +1,268 @@
+# Copyright (c) 2014 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+module Bench
+
+  CONFIG_VERSION = 1
+
+  module Main
+
+    def self.main(args)
+      pre_args = args.dup
+
+      until pre_args.empty?
+        arg = pre_args.shift
+
+        if arg.start_with? "--"
+          case arg
+          when "--env"
+            key      = pre_args.shift
+            value    = pre_args.shift
+            ENV[key] = value
+          end
+        end
+      end
+
+      config = Config.new
+
+      # try to find and load default.config.rb file
+      load_configuration_file(config, 'default')
+
+      args = args.dup
+
+      command_name = args.shift
+
+      subjects = []
+      flags    = {}
+
+      until args.empty?
+        arg = args.shift
+
+        if arg.start_with? "--"
+          case arg
+          when "--config"
+            load_configuration_file config, args.shift
+          when "--data"
+            flags[arg] = args.shift
+          when "--baseline"
+            flags[arg] = args.shift
+          when "--notes"
+            flags[arg] = args.shift
+          when "--resume"
+            flags[arg] = true
+          when "--show-commands"
+            flags[arg] = true
+          when "--show-samples"
+            flags[arg] = true
+          when "--ignore-excludes"
+            flags[arg] = true
+          when "--value-per-line"
+            flags[arg] = true
+          when "--benchmark-per-line"
+            flags[arg] = true
+          when "--json"
+            flags[arg] = true
+          when "--test-run"
+            flags[arg] = true
+          when "--env"
+            args.shift
+            args.shift
+          else
+            puts "unknown option #{arg}"
+            exit 1
+          end
+        else
+          subjects.push arg
+        end
+      end
+
+      implementations = []
+      benchmarks      = []
+
+      subjects.each do |s|
+        negated = s.start_with? "^"
+        s       = s[1..-1] if negated
+
+        if s == "all"
+          config.benchmarks.values.each do |b|
+            benchmarks.push b
+          end
+        elsif config.implementations.has_key? s
+          i = config.implementations[s]
+          if negated
+            implementations.delete i
+          else
+            implementations.push i
+          end
+        elsif config.implementation_groups.has_key? s
+          ig = config.implementation_groups[s]
+          if negated
+            implementations.delete_if { |i| ig.members.include? i }
+          else
+            implementations.concat ig.members
+          end
+        elsif config.benchmarks.has_key? s
+          b = config.benchmarks[s]
+          if negated
+            benchmarks.delete b
+          else
+            benchmarks.push b
+          end
+        elsif config.benchmark_groups.has_key? s
+          bg = config.benchmark_groups[s]
+          if negated
+            benchmarks.delete_if { |b| bg.members.include? b }
+          else
+            benchmarks.concat bg.members
+          end
+        else
+          puts "unknown implementation or benchmark or group #{s}"
+          exit 1
+        end
+      end
+
+      options = Bench::Options.new(
+          config,
+          implementations,
+          benchmarks,
+          flags)
+
+      start = Time.now
+
+      existing_measurements = Measurements.new
+
+      if options.flags.has_key?("--data")
+        if File.exist?(options.flags["--data"])
+          file = File.open(options.flags["--data"], "r")
+
+          if file.gets.strip != "version #{CONFIG_VERSION}"
+            puts "the benchmarks have changed since this file was created"
+            exit 1
+          end
+
+          file.each do |line|
+            measurement = JSON.parse(line)
+
+            if measurement["failed"]
+              measurement_object = :failed
+            else
+              measurement_object = Measurement.new(measurement["warmup_samples"], measurement["samples"])
+            end
+
+            existing_measurements[measurement["benchmark"], measurement["implementation"]] = measurement_object
+          end
+
+          file.close
+        end
+      end
+
+      builtin_commands = {
+          "compare"              => Commands::Compare,
+          "compare-reference"    => Commands::CompareReference,
+          "detail"               => Commands::Detail,
+          "list-benchmarks"      => Commands::ListBenchmarks,
+          "list-implementations" => Commands::ListImplementations,
+          "reference"            => Commands::Reference,
+          "report"               => Commands::Report,
+          "remove"               => Commands::Remove,
+          "score"                => Commands::Score
+      }
+
+      commands = builtin_commands.merge(config.commands)
+      command  = commands[command_name]
+
+      if command.nil?
+        puts "unknown command #{command_name}"
+        exit 1
+      end
+
+      command = command.new
+
+      if command.before options, existing_measurements
+        if options.flags.has_key?("--data")
+          file = File.open(options.flags["--data"], "w")
+          file.puts "version #{CONFIG_VERSION}"
+          file.flush
+
+          existing_measurements.measurements.each do |bi, measurement|
+            b, i = bi
+            if measurement == :failed
+              file.puts JSON.generate({ benchmark: b, implementation: i, failed: true })
+            else
+              file.puts JSON.generate({ benchmark: b, implementation: i, warmup_samples: measurement.warmup_samples, samples: measurement.samples })
+            end
+
+            file.flush
+          end
+        end
+
+        measurements = Measurements.new
+
+        options.benchmarks.each do |b|
+          options.implementations.each do |i|
+            measurement = existing_measurements[b.name, i.name]
+
+            if measurement.nil?
+              if config.fail_hard_exclusions.include?([i.name, b.name]) && !options.flags["--ignore-excludes"]
+                measurement = :failed
+              else
+                measurement = i.measure(flags, b)
+              end
+
+              unless file.nil?
+                if measurement == :failed
+                  file.puts JSON.generate({ benchmark: b, implementation: i, failed: true })
+                else
+                  file.puts JSON.generate({ benchmark: b, implementation: i, warmup_samples: measurement.warmup_samples, samples: measurement.samples })
+                end
+
+                file.flush
+              end
+            end
+
+            measurements[b, i] = measurement
+            command.result options, b, i, measurement
+          end
+
+          command.benchmark_complete options, b, measurements
+        end
+
+        command.after options, measurements
+
+        unless file.nil?
+          file.close
+        end
+      end
+
+      puts "took #{Time.now - start}s"
+    end
+
+    def self.load_configuration_file(config, name, fail_on_missing = true)
+      if name.scan('/').any?
+        config.load File.expand_path(name)
+      else
+        full_name = if name =~ /\.config\.rb$/
+                      name
+                    else
+                      name + '.config.rb'
+                    end
+
+        config_files = Dir["{.,bench,benchmark,benchmarks}/#{full_name}"]
+        unless config_files.any?
+          puts "Configuration file #{name} not found."
+          exit 1
+        end
+        config_files.each do |config_file|
+          config.load config_file
+        end
+      end
+    end
+
+  end
+
+end