perfer 0.1.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Benoit Daloze
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# Perfer - A benchmark tool for all rubies!
+
+<!-- usage -->
+```text
+Usage:
+  perfer <command> [options] arguments
+
+Commands:
+  run files+    - run with current ruby
+  report files+ - show the results
+  config reset  - reset the configuration file to the defaults (or create it)
+  help          - show this help
+  results
+    path files+      - show the paths to the result files
+    rm,delete files+ - remove the result files
+
+<files+> are a set of benchmark files
+
+Common options:
+    -t TIME                          Minimal time to run (greater usually improve accuracy)
+    -m N                             Numbers of measurements per job
+    -v                               Verbose
+    -h, --help                       Show this help
+```
+<!-- usage -->
+
+See [The GSoC proposal](http://www.google-melange.com/gsoc/proposal/review/google/gsoc2012/eregon/1) for more details.
+
+And the [Wiki](https://github.com/jruby/perfer/wiki).
+
+## Current status
+
+This is a work in progress, 0.1.0 is a preview release.

data/lib/perfer.rb

@@ -0,0 +1,54 @@
+require 'yaml'
+require 'path'
+require 'optparse'
+require 'hitimes'
+require 'forwardable'
+require 'ffi'
+require 'digest/sha1'
+require 'backports/1.9'
+
+Path.require_tree('perfer', :except => %w[platform/])
+
+module Perfer
+  DIR = Path('~/.perfer')
+  TIMES_FIELDS = [:real, :utime, :stime, :cutime, :cstime].freeze
+
+  class << self
+    attr_reader :sessions, :configuration
+
+    def reset
+      @configuration = Configuration.new
+      @sessions = []
+    end
+
+    def session(name = nil, &block)
+      Session.new(Path.file(caller), name, &block)
+    end
+
+    # Shortcut for Perfer.session { |s| s.iterate ... }
+    def iterate(title = nil, *args, &block)
+      Session.new(Path.file(caller)) { |session|
+        title ||= session.name
+        session.iterate(title, *args, &block)
+      }
+    end
+
+    def measure(&block)
+      times_before = Process.times
+      real = Hitimes::Interval.measure(&block)
+      times = Process.times
+
+      data = { :real => real }
+      times.members.each { |field|
+        # precision of times(3) or getrusage(2) is no more than 1e-6
+        value = (times[field] - times_before[field]).round(6)
+        if value != 0.0 # do not keep these if they measured nothing
+          data[field.to_sym] = value
+        end
+      }
+      data
+    end
+  end
+
+  Perfer.reset
+end

data/lib/perfer/cli.rb

@@ -0,0 +1,139 @@
+module Perfer
+  class CLI
+    COMMANDS = %w[
+      config
+      help
+      report
+      results
+      run
+    ]
+
+    HELP = <<-EOS
+Usage:
+  perfer <command> [options] arguments
+
+Commands:
+  run files+    - run with current ruby
+  report files+ - show the results
+  config reset  - reset the configuration file to the defaults (or create it)
+  help          - show this help
+  results
+    path files+      - show the paths to the result files
+    rm,delete files+ - remove the result files
+
+<files+> are a set of benchmark files
+
+Common options:
+EOS
+
+    def initialize(argv)
+      @argv = argv
+
+      @opts = OptionParser.new do |options|
+        options.banner = HELP
+        common_options(options)
+      end
+    end
+
+    def execute
+      begin
+        @opts.order!(@argv)
+      rescue OptionParser::ParseError => e
+        error e.message
+      end
+
+      @command = @argv.shift
+      error "A command must be given, one of: #{COMMANDS*', '}" unless @command
+      error "Unknown command: #{@command.inspect}" unless COMMANDS.include? @command
+
+      send(@command)
+    end
+
+    def unknown_subcommand(subcommand)
+      if subcommand
+        error "Unknown subcommand for #{@command}: #{subcommand.inspect}"
+      else
+        error "`perfer #{@command}` needs a subcommand"
+      end
+    end
+
+    def error message
+      $stderr.puts message
+      $stderr.puts
+      abort @opts.help
+    end
+
+    def help
+      puts @opts.help
+    end
+
+    def report
+      measurements = (@argv.shift if @argv.first == '--measurements')
+      each_session { |session| session.report_results(:measurements => measurements) }
+    end
+
+    def run
+      # load files
+      files.each do |file|
+        require file.path
+      end
+      Perfer.sessions.each(&:run)
+    end
+
+    def results
+      case subcommand = @argv.shift
+      when "path"
+        each_session { |session| puts session.store.file }
+      when "delete", "rm"
+        each_session { |session| session.store.delete }
+      when "rewrite"
+        each_session { |session| session.store.rewrite }
+      else
+        unknown_subcommand subcommand
+      end
+    end
+
+    def config
+      case subcommand = @argv.shift
+      when "reset"
+        Perfer.configuration.write_defaults
+      else
+        unknown_subcommand subcommand
+      end
+    end
+
+    def common_options(options)
+      options.on('-t TIME', Float, "Minimal time to run (greater usually improve accuracy)") do |t|
+        error "Minimal time must be > 0" if t <= 0
+        Perfer.configuration.minimal_time = t
+      end
+      options.on('-m N', Integer, "Numbers of measurements per job") do |n|
+        error "There must be at least 2 measurements" if n < 2
+        Perfer.configuration.measurements = n
+      end
+      options.on('-v', "Verbose") do
+        Perfer.configuration.verbose = true
+      end
+      options.on('-h', '--help', "Show this help") do
+        puts options.help
+        exit
+      end
+    end
+
+  private
+    def files
+      @argv.map { |file| Path(file).expand }
+    end
+
+    def load_from_files
+      files.each do |file|
+        Session.new(file)
+      end
+    end
+
+    def each_session(&block)
+      load_from_files
+      Perfer.sessions.each(&block)
+    end
+  end
+end

data/lib/perfer/configuration.rb

@@ -0,0 +1,42 @@
+module Perfer
+  class Configuration
+    DEFAULTS = {
+      :minimal_time => 1.0,
+      :measurements => 10,
+      :verbose => false,
+    }.freeze
+
+    PROPERTIES = DEFAULTS.keys
+
+    PROPERTIES.each { |property| attr_accessor property }
+
+    def initialize
+      @config_file = DIR/'config.yml'
+
+      DEFAULTS.each_pair { |property, value|
+        instance_variable_set(:"@#{property}", value)
+      }
+
+      if @config_file.exist? and !@config_file.empty?
+        YAML.load_file(@config_file).each_pair { |property, value|
+          property = property.to_sym
+          if PROPERTIES.include? property
+            instance_variable_set(:"@#{property}", value)
+          else
+            warn "Unknown property in configuration file: #{property}"
+          end
+        }
+      end
+    end
+
+    def write_defaults
+      @config_file.write YAML.dump DEFAULTS
+    end
+
+    def to_hash
+      PROPERTIES.each_with_object({}) { |property, h|
+        h[property] = instance_variable_get(:"@#{property}")
+      }
+    end
+  end
+end

data/lib/perfer/errors.rb

@@ -0,0 +1,11 @@
+module Perfer
+  class Error < RuntimeError
+  end
+
+  module Errors
+    MIX_BENCH_TYPES = "Cannot mix iterations and input size benchmarks in the same session"
+    SAME_JOB_TITLES = "Multiple jobs with the same title are not allowed"
+    WRONG_MEASURE_USE = "Cannot use Session#measure with an iteration job"
+    WORKING_DIR_DIRTY = "The working directory is dirty, commit (or stash) your changes first"
+  end
+end

data/lib/perfer/formatter.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+
+module Perfer
+  module Formatter
+    extend self
+
+    TIME_UNITS = {
+       0 => "s ",
+      -3 => "ms",
+      -6 => "µs",
+      -9 => "ns"
+    }
+
+    def max_length_of(enum)
+      max = 0
+      enum.each { |e|
+        alt = yield(e).to_s.length
+        max = alt if alt > max
+      }
+      max
+    end
+
+    def format_ips(ips)
+      if ips > 100
+        ips.round
+      else
+        "%.3g" % ips
+      end
+    end
+
+    def format_n(n, maxlen)
+      n.to_s.rjust(maxlen)
+    end
+
+    def format_float(f)
+      ('%5.3f' % f)[0...5]
+    end
+
+    def format_error(error, base, scale)
+      "#{format_float(error*10**-scale)} (#{'%4.1f' % (error / base * 100.0)}%)"
+    end
+
+    def format_time(time)
+      time.strftime("%F %T")
+    end
+
+    def float_scale(time)
+      if time == 0 or time > 1.0
+        0
+      elsif time > 0.001
+        -3
+      elsif time > 0.000001
+        -6
+      else
+        -9
+      end
+    end
+
+    # formats a duration with an 8-chars width
+    def format_duration(time, scale = float_scale(time))
+      if time == 0
+        "    0   "
+      else
+        "#{format_float(time*10**-scale)} #{TIME_UNITS[scale]}"
+      end
+    end
+
+    def format_duration_and_error(time, error, after_unit = "")
+      scale = float_scale(time)
+      "#{format_duration(time, scale)}#{after_unit} ± #{format_error(error, time, scale)}"
+    end
+  end
+end

data/lib/perfer/formatter/measurements_formatter.rb

@@ -0,0 +1,22 @@
+module Perfer
+  class MeasurementsFormatter
+    include Formatter
+
+    SEPARATOR = '  '
+
+    def initialize(measurements)
+      @measurements = measurements
+      @fields = Perfer::TIMES_FIELDS.dup
+      @fields.reject! { |field|
+        measurements.none? { |m| m.key? field }
+      }
+    end
+
+    def report
+      puts @fields.map { |field| field.to_s.center(8) }.join(SEPARATOR).rstrip
+      @measurements.each { |m|
+        puts @fields.map { |field| format_duration(m[field] || 0) }.join(SEPARATOR)
+      }
+    end
+  end
+end

data/lib/perfer/formatter/results_formatter.rb

@@ -0,0 +1,43 @@
+module Perfer
+  class ResultsFormatter
+    include Formatter
+
+    # maximal job length is computed from +jobs+ if given,
+    # or deduced from given results
+    def initialize(results, jobs = nil)
+      @results = Array(results)
+      @max_job_length = if jobs
+        max_length_of(jobs, &:title)
+      else
+        max_length_of(@results) { |r| r[:job] }
+      end
+    end
+
+    def job_title(result)
+      result[:job].to_s.ljust(@max_job_length)
+    end
+
+    def max_n_length
+      @max_n_length ||= max_length_of(@results) { |r| r[:n] }
+    end
+
+    def report(options = {})
+      measurements = options[:measurements]
+      @results.each do |result|
+        MeasurementsFormatter.new(result.data).report if measurements
+        r = result
+        stats = r.stats
+        mean = stats.mean
+        error = stats.maximum_absolute_deviation
+        if r[:iterations]
+          time_per_i, ips = mean/r[:iterations], r[:iterations]/mean
+          error /= r[:iterations]
+          puts "#{job_title(r)} #{format_duration_and_error time_per_i, error, '/i'} <=> #{format_ips ips} ips"
+        else
+          n = format_n(r[:n], max_n_length)
+          puts "#{job_title(r)} #{n} in #{format_duration_and_error mean, error}"
+        end
+      end
+    end
+  end
+end

data/lib/perfer/formatter/session_formatter.rb

@@ -0,0 +1,22 @@
+module Perfer
+  class SessionFormatter
+    include Formatter
+
+    def initialize(session)
+      @session = session
+    end
+
+    def report(options = {})
+      return puts "No results available." unless @session.results
+      session_name = @session.results.first[:session]
+      puts session_name
+      @session.results.chunk { |r|
+        r[:run_time]
+      }.each { |run_time, results|
+        puts "Ran at #{format_time run_time} with #{results.first[:ruby]}"
+        ResultsFormatter.new(results).report(options)
+        puts
+      }
+    end
+  end
+end

data/lib/perfer/git.rb

@@ -0,0 +1,36 @@
+module Perfer
+  module Git
+    class << self
+      def git(command)
+        output = `git #{command}`
+        output.chomp!
+        return nil if output.empty?
+        return false if $?.exitstatus != 0
+        output
+      end
+
+      def repository?
+        git "rev-parse --git-dir 2>#{File::NULL}"
+      end
+
+      def current_branch
+        branch = git 'symbolic-ref --quiet HEAD'
+        branch = branch[/[^\/]+$/] if branch
+        branch
+      end
+
+      def current_commit
+        git 'rev-parse --quiet --verify HEAD'
+      end
+
+      def working_directory_clean?
+        git('status --porcelain --untracked-files=no') == nil
+      end
+
+      def goto_commit(commit)
+        raise Error, Errors::WORKING_DIR_DIRTY unless working_directory_clean?
+        git "reset --quiet --hard #{commit}"
+      end
+    end
+  end
+end

data/lib/perfer/job.rb

@@ -0,0 +1,32 @@
+module Perfer
+  class Job
+    attr_reader :session, :title, :metadata
+    def initialize(session, title, &block)
+      @session = session
+      @title = title
+      @block = block
+
+      @metadata = @session.metadata.merge(:job => @title).freeze
+    end
+
+    def results
+      @session.results.select { |result| result[:job] == @title }
+    end
+
+    def minimal_time
+      Perfer.configuration.minimal_time
+    end
+
+    def number_of_measurements
+      Perfer.configuration.measurements
+    end
+
+    def verbose
+      Perfer.configuration.verbose
+    end
+
+    def run
+      @session.current_job = self
+    end
+  end
+end

data/lib/perfer/job/input_size_job.rb

@@ -0,0 +1,44 @@
+module Perfer
+  class InputSizeJob < Job
+    attr_writer :last_measurement
+
+    def start
+      1024
+    end
+
+    def generator(n)
+      n * 2
+    end
+
+    def measure(n)
+      GC.start
+      @block.call(n)
+      @last_measurement
+    end
+
+    def run
+      super
+      n = start
+      # find an appropriate maximal n, acts as warm-up
+      loop do
+        time = measure(n)[:real]
+        break if time > minimal_time
+        n = generator(n)
+      end
+
+      max = n
+      n = start
+      loop do
+        result = Result.new(@metadata)
+        result[:n] = n
+        number_of_measurements.times do
+          result << measure(n)
+        end
+        @session.add_result(result)
+
+        break if n == max
+        n = generator(n)
+      end
+    end
+  end
+end

data/lib/perfer/job/iteration_job.rb

@@ -0,0 +1,156 @@
+module Perfer
+  class IterationJob < Job
+    # This factor ensure some margin,
+    # to avoid endlessly changing the number of iterations
+    CHANGE_ITERATIONS_MARGIN = 0.1
+    UNIQUE_NAME = "a"
+
+    def repeat_eval
+      100
+    end
+
+    def initialize(session, title, code, data, &block)
+      super(session, title, &block)
+      if code and !block
+        @data = data || {}
+        if obj = @data.delete(:self)
+          klass = obj.singleton_class
+          meth = generate_method_name
+        else
+          klass = singleton_class
+          meth = :measure_call_times_code
+        end
+
+        if klass.method_defined?(meth)
+          raise Error, "method #{meth} already defined on #{klass} (#{obj})!"
+        end
+
+        begin
+          klass.class_eval <<-EOR
+          def #{meth}(__n#{@data.keys.map { |k| ", #{k}" }.join})
+            ::Perfer.measure do
+              __i = 0
+              while __i < __n
+                #{"#{code}; " * repeat_eval}
+                __i += 1
+              end
+            end
+          end
+          EOR
+        rescue SyntaxError => e
+          raise Error, "There was an error while eval'ing the code: #{code.inspect}\n#{e}"
+        end
+
+        if obj
+          singleton_class.send(:define_method, :measure_call_times_code) do |*args|
+            obj.send(meth, *args)
+          end
+        end
+      end
+    end
+
+    def generate_method_name
+      :"perfer_eval_#{UNIQUE_NAME.succ!}"
+    end
+
+    def measure_call_times(n)
+      GC.start
+      if !@block
+        if n % repeat_eval != 0
+          raise Error, "Implementation error: #{n} not multiple of #{repeat_eval}"
+        end
+        n /= repeat_eval
+        measure_call_times_code(n, *@data.values)
+      elsif @block.arity == 1
+        # give n, the block must iterate n times
+        Perfer.measure { @block.call(n) }
+      else
+        Perfer.measure { n.times(&@block) }
+      end.tap { |m| p m if verbose }
+    end
+
+    def round_for_eval(iterations)
+      if @block
+        iterations
+      else
+        ((iterations + repeat_eval - 1) / repeat_eval) * repeat_eval
+      end
+    end
+
+    def compute_new_iterations(iterations, time)
+      (minimal_time * (1.0 + CHANGE_ITERATIONS_MARGIN) * iterations / time).ceil
+    end
+
+    def find_number_of_iterations_required(last_iterations = 1, last_time = 0)
+      iterations = last_iterations
+      if last_time > 0
+        iterations = compute_new_iterations(last_iterations, last_time)
+      end
+      iterations = round_for_eval(iterations)
+      puts "Start search for iterations: start=#{iterations}" if verbose
+      loop do
+        puts "iterations: #{iterations}" if verbose
+        time = measure_call_times(iterations)[:real]
+        break if time > minimal_time
+
+        if time <= 0
+          iterations *= 2
+          next
+        end
+
+        new_iterations = compute_new_iterations(iterations, time)
+        # ensure the number of iterations increases
+        if new_iterations <= iterations
+          puts "new_iterations <= iterations: #{new_iterations} <= #{iterations}" if verbose
+          new_iterations = (iterations*1.5).ceil
+        end
+        iterations = round_for_eval(new_iterations)
+      end
+      puts "End search for iterations: iterations=#{iterations}" if verbose
+      iterations
+    end
+
+    # median absolute deviation / median
+    def mad(measurements)
+      stats = Statistics.new(measurements.map { |m| m[:real] })
+      mad = stats.median_absolute_deviation
+      mad /= stats.median
+      puts "mad: #{mad}" if verbose
+      mad
+    end
+
+    def run
+      super
+      measurements = []
+
+      # Run one iteration, so system-level buffers and other OS warm-up can take place
+      # This is usually a very inaccurate measurement, so just discard it
+      measure_call_times(round_for_eval(1))
+
+      iterations = find_number_of_iterations_required
+
+      measurements_taken = 0
+      until measurements.size == number_of_measurements and
+            mad(measurements) < 0.01 * measurements_taken / number_of_measurements
+        time = measure_call_times(iterations)
+        measurements_taken += 1
+        if time[:real] < (1.0 - CHANGE_ITERATIONS_MARGIN) * minimal_time
+          # restart and find a more appropriate number of iterations
+          puts "Restarting, #{time[:real]} < #{minimal_time}" if verbose
+          measurements.clear
+          measurements_taken = 0
+          iterations = find_number_of_iterations_required(iterations, time[:real])
+        else
+          # circular buffer needed!
+          measurements.shift if measurements.size == number_of_measurements
+          measurements << time
+        end
+      end
+
+      result = Result.new(@metadata)
+      result[:iterations] = iterations
+      result.data = measurements
+      @session.add_result(result)
+    end
+  end
+end

data/lib/perfer/platform.rb

@@ -0,0 +1,13 @@
+module Perfer
+  module Platform
+    OS = RbConfig::CONFIG['host_os']
+
+    if /mingw|mswin/ =~ OS
+      require Path.relative('platform/windows')
+      extend Windows
+    else
+      require Path.relative('platform/posix')
+      extend POSIX
+    end
+  end
+end

data/lib/perfer/platform/posix.rb

@@ -0,0 +1,81 @@
+module Perfer::Platform
+  module POSIX
+    class TimeValStruct < FFI::Struct
+      layout  :tv_sec, :time_t,
+              :tv_usec, :suseconds_t
+    end
+
+    class RUsageStruct < FFI::Struct
+      # Rubinius FFI can't handle nested structs
+      layout  :ru_utime_tv_sec, :time_t, # :ru_utime, TimeValStruct,
+              :ru_utime_tv_usec, :suseconds_t,
+              :ru_stime_tv_sec, :time_t, # :ru_stime, TimeValStruct,
+              :ru_stime_tv_usec, :suseconds_t,
+              :ru_maxrss, :long,
+              :ru_ixrss, :long,
+              :ru_idrss, :long,
+              :ru_isrss, :long,
+              :ru_minflt, :long,
+              :ru_majflt, :long,
+              :ru_nswap, :long,
+              :ru_inblock, :long,
+              :ru_oublock, :long,
+              :ru_msgsnd, :long,
+              :ru_msgrcv, :long,
+              :ru_nsignals, :long,
+              :ru_nvcsw, :long,
+              :ru_nivcsw, :long
+    end
+
+    module LibC
+      extend FFI::Library
+      ffi_lib FFI::Library::LIBC
+
+      RUSAGE_SELF = 0
+
+      attach_function :getrusage, [:int, :pointer], :int
+    end
+
+    PID = Process.pid
+
+    def memory_used
+      case OS
+      when /^darwin/, /^linux/, /^solaris/
+        Integer(`ps -o rss= -p #{PID}`) * 1024
+      else
+        warn "Unknown platform for Platform.command_line: #{os.inspect}"
+        nil
+      end
+    end
+
+    def maximum_memory_used
+      rusage = RUsageStruct.new
+      r = LibC.getrusage(LibC::RUSAGE_SELF, rusage)
+      if r != 0
+        warn "Could not retrieve memory information with getrusage(2)"
+        0
+      else
+        m = rusage[:ru_maxrss]
+        warn "Memory information with getrusage(2) is inaccurate, ru_maxrss was 0" if m == 0
+        if /darwin/ =~ OS
+          m # reported in bytes
+        else
+          m * 1024 # reported in KB, as the man page says
+        end
+      end
+    ensure
+      rusage.pointer.free if rusage
+    end
+
+    def command_line
+      return Rubinius::OS_ARGV.join(' ') if defined? Rubinius::OS_ARGV
+      case OS
+      when /^darwin/, /^linux/, /^solaris/
+        `ps -o args= -p #{PID}`.lines.to_a.last.rstrip
+      else
+        warn "Unknown platform for Platform.command_line: #{os.inspect}"
+        nil
+      end
+    end
+  end
+end

data/lib/perfer/platform/windows.rb

@@ -0,0 +1,83 @@
+module Perfer::Platform
+  module Windows
+    module Types
+      BOOL = :bool
+      DWORD = :uint32
+      HANDLE = :pointer
+      SIZE_T = :size_t
+      LPSTR = :string
+    end
+
+    class PProcessMemoryCounters < FFI::Struct
+      include Types
+      layout  :cb, DWORD,
+              :PageFaultCount, DWORD,
+              :PeakWorkingSetSize, SIZE_T,
+              :WorkingSetSize, SIZE_T,
+              :QuotaPeakPagedPoolUsage, SIZE_T,
+              :QuotaPagedPoolUsage, SIZE_T,
+              :QuotaPeakNonPagedPoolUsage, SIZE_T,
+              :QuotaNonPagedPoolUsage, SIZE_T,
+              :PagefileUsage, SIZE_T,
+              :PeakPagefileUsage, SIZE_T
+    end
+
+    module Kernel32
+      include Types
+      extend FFI::Library
+      ffi_lib 'kernel32'
+      ffi_convention :stdcall
+
+      attach_function :GetCurrentProcess, [], HANDLE
+      attach_function :GetCommandLineA, [], LPSTR
+    end
+
+    module PSAPI
+      include Types
+      extend FFI::Library
+      ffi_lib 'psapi'
+      ffi_convention :stdcall
+
+      attach_function :GetProcessMemoryInfo, [HANDLE, PProcessMemoryCounters, DWORD], BOOL
+    end
+
+    def get_process_memory_info
+      process = Kernel32.GetCurrentProcess
+      info = PProcessMemoryCounters.new
+      # See http://msdn.microsoft.com/en-us/library/windows/desktop/ms683219%28v=vs.85%29.aspx
+      r = PSAPI.GetProcessMemoryInfo(process, info, info.size)
+      if !r
+        warn "Could not retrieve memory information with GetProcessMemoryInfo()"
+        nil
+      else
+        yield(info)
+      end
+    ensure
+      info.pointer.free if info
+    end
+    private :get_process_memory_info
+
+    def memory_used
+      get_process_memory_info { |info|
+        # info[:PeakWorkingSetSize] # RAM
+        info[:PeakPagefileUsage] # RAM + SWAP
+      }
+    end
+
+    def maximum_memory_used
+      get_process_memory_info { |info|
+        # info[:WorkingSetSize] # RAM
+        info[:PagefileUsage] # RAM + SWAP
+      }
+    end
+
+    def command_line
+      Kernel32.GetCommandLineA().tap do |command_line|
+        unless command_line
+          warn "Could not get command line via GetCommandLineA()"
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/perfer/result.rb

@@ -0,0 +1,38 @@
+module Perfer
+  # A result for a particular job run
+  class Result
+    extend Forwardable
+
+    attr_reader :metadata
+    attr_accessor :data
+    def initialize(metadata, data = [])
+      @metadata = metadata.dup
+      @data = data
+    end
+
+    def_instance_delegators :@data,
+      :<<, :size, :length, :each
+
+    def_instance_delegators :@metadata, :[], :[]=
+
+    def stats
+      Statistics.new(on(:real))
+    end
+
+    def on(field)
+      @data.map { |result| result[field] }
+    end
+
+    def to_json(*args)
+      {
+        'json_class' => self.class.name,
+        'data'       => @data,
+        'metadata'   => @metadata
+      }.to_json(*args)
+    end
+
+    def self.json_create json
+      new(json['metadata'], json['data'])
+    end
+  end
+end

data/lib/perfer/session.rb

@@ -0,0 +1,114 @@
+module Perfer
+  class Session
+    attr_reader :name, :file, :jobs, :type, :store, :results, :metadata
+    attr_writer :current_job
+    def initialize(file, name = nil, &block)
+      @file = file
+      @name = name || file.base.to_s
+      @store = Store.for_session(self)
+      @results = nil # not an Array, so it errors out if we forgot to load
+
+      setup_for_run(&block) if block_given?
+
+      Perfer.sessions << self
+    end
+
+    def setup_for_run
+      @jobs = []
+      @type = nil # will be decided by API usage (iterate/bench)
+      @results_to_save = []
+
+      @metadata = {
+        :file => @file.path,
+        :session => @name,
+        :ruby => RUBY_DESCRIPTION,
+        :command_line => Platform.command_line,
+        :run_time => Time.now
+      }
+      add_config_metadata
+      add_git_metadata
+      add_bench_file_checksum
+      @metadata.freeze
+
+      yield self
+    end
+
+    def add_config_metadata
+      @metadata.merge!(Perfer.configuration.to_hash)
+    end
+
+    def add_git_metadata
+      if Git.repository?
+        @metadata[:git_branch] = Git.current_branch
+        @metadata[:git_commit] = Git.current_commit
+      end
+    end
+
+    def add_bench_file_checksum
+      checksum = Digest::SHA1.hexdigest(@file.binread)
+      if checksum.respond_to?(:encoding) and checksum.encoding != Encoding::ASCII
+        checksum.force_encoding(Encoding::ASCII)
+      end
+      @metadata[:bench_file_checksum] = checksum
+    end
+
+    def load_results
+      @results = @store.load
+    end
+
+    def add_result(result)
+      @results_to_save << result
+      ResultsFormatter.new(result, @jobs).report
+    end
+
+    def run
+      puts "Session #{@name} with #{@metadata[:ruby]}"
+      print "Taking #{Perfer.configuration.measurements} measurements of"
+      puts " at least #{Perfer.configuration.minimal_time}s"
+      @jobs.each { |job|
+        job.run
+      }
+      @results_to_save.each { |result|
+        @store.append(result)
+      }
+      @results_to_save.clear
+    end
+
+    # not named #report, to avoid any confusion with Benchmark's #report
+    def report_results(options = {})
+      load_results
+      SessionFormatter.new(self).report(options)
+    end
+
+    def iterate(title, code = nil, data = nil, &block)
+      check_benchmark_type(:iterations)
+      check_unique_job_title(title)
+      @jobs << IterationJob.new(self, title, code, data, &block)
+    end
+
+    def bench(title, &block)
+      check_benchmark_type(:input_size)
+      check_unique_job_title(title)
+      @jobs << InputSizeJob.new(self, title, &block)
+    end
+
+    def measure(&block)
+      raise Error, WRONG_MEASURE_USE unless InputSizeJob === @current_job
+      @current_job.last_measurement = Perfer.measure(&block)
+    end
+
+  private
+    def check_benchmark_type(expected)
+      unless !@type or @type == expected
+        raise Error, Errors::MIX_BENCH_TYPES
+      end
+      @type ||= expected
+    end
+
+    def check_unique_job_title(title)
+      if @jobs.any? { |job| job.title == title }
+        raise Error, Errors::SAME_JOB_TITLES
+      end
+    end
+  end
+end

data/lib/perfer/statistics.rb

@@ -0,0 +1,94 @@
+module Perfer
+  class Statistics
+    include Math
+    CONFIDENCE_LEVEL = 0.95
+    ALPHA = 1.0 - CONFIDENCE_LEVEL # significance level
+
+    # Student's t quantiles is used as n is small (= number of measurements)
+    # Indexed by: probability, degrees of freedom
+    T_QUANTILES = {
+      0.975 => [
+        nil,
+        12.71, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, #  1-10
+        2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, # 11-20
+        2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042, # 21-30
+        2.040, 2.037, 2.035, 2.032, 2.030, 2.028, 2.026, 2.024, 2.023, 2.021  # 31-40
+      ]
+    }
+    { 50 => 2.009, 60 => 2.000,  70 => 1.994,
+      80 => 1.990, 90 => 1.987, 100 => 1.984 }.each_pair { |n, value|
+      T_QUANTILES[0.975][n] = value
+    }
+
+    def self.t_quantile(p, degrees_of_freedom)
+      if degrees_of_freedom <= 40
+        T_QUANTILES[p][degrees_of_freedom]
+      elsif degrees_of_freedom <= 100
+        T_QUANTILES[p][degrees_of_freedom.round(-1)]
+      else
+        1.960
+      end
+    end
+
+    def initialize(sample)
+      @sample = sample
+    end
+
+    def size
+      @sample.size
+    end
+
+    def mean
+      @mean ||= @sample.inject(0.0) { |sum, i| sum + i } / size
+    end
+
+    def median
+      @median ||= begin
+        sorted = @sample.sort
+        if size.odd?
+          sorted[size/2]
+        else
+          (sorted[size/2-1] + sorted[size/2]) / 2.0
+        end
+      end
+    end
+
+    def variance
+      mean = mean()
+      @sample.inject(0.0) { |var, i|
+        d = i - mean
+        var + d*d
+      } / (size - 1) # unbiased sample variance
+    end
+
+    def standard_deviation
+      sqrt(variance)
+    end
+
+    def coefficient_of_variation
+      standard_deviation / mean
+    end
+
+    def standard_error
+      standard_deviation / sqrt(size)
+    end
+
+    def mean_absolute_deviation
+      @sample.inject(0.0) { |dev, i| dev + (i - mean).abs } / size
+    end
+
+    def median_absolute_deviation
+      Statistics.new(@sample.map { |i| (i - median).abs }).median
+    end
+
+    # Assumes a standard normal distribution
+    # This is half the width of the confidence interval for the mean
+    def margin_of_error
+      Statistics.t_quantile(1.0 - ALPHA/2, size-1) * standard_error
+    end
+
+    def maximum_absolute_deviation
+      @sample.map { |v| (v - mean).abs }.max
+    end
+  end
+end

data/lib/perfer/store.rb

@@ -0,0 +1,53 @@
+module Perfer
+  class Store
+    attr_reader :file
+    def initialize(file)
+      @file = Path(file)
+    end
+
+    def self.for_session(session)
+      path = DIR/'results'
+      path.mkpath unless path.exist?
+
+      bench_file = session.file
+
+      # get the relative path to root, and relocate in @path
+      names = bench_file.each_filename.to_a
+      # prepend drive letter on Windows
+      names.unshift bench_file.path[0..0].upcase if File.dirname('C:') == 'C:.'
+
+      new path.join(*names).add_ext('.yml')
+    end
+
+    def delete
+      @file.unlink
+    end
+
+    def yaml_load_documents
+      docs = @file.open { |f| YAML.load_stream(f) }
+      docs = docs.documents unless Array === docs
+      docs
+    end
+
+    def load
+      return unless @file.exist?
+      yaml_load_documents
+    end
+
+    def append(result)
+      @file.dir.mkpath unless @file.dir.exist?
+      @file.append YAML.dump(result)
+    end
+
+    def save(results)
+      @file.dir.mkpath unless @file.dir.exist?
+      # ensure results are still ordered by :run_time
+      results.sort_by! { |r| r[:run_time] }
+      @file.write YAML.dump_stream(*results)
+    end
+
+    def rewrite
+      save load
+    end
+  end
+end

data/perfer.gemspec

@@ -0,0 +1,9 @@
+Gem::Specification.new do |s|
+  s.name = 'perfer'
+  s.summary = 'A benchmark tool for all rubies!'
+  s.author = 'eregon'
+  s.email = 'eregontp@gmail.com'
+  s.homepage = 'https://github.com/jruby/perfer'
+  s.files = Dir['lib/**/*.rb'] + %w[README.md LICENSE perfer.gemspec]
+  s.version = '0.1.0'
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: perfer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- eregon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-15 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: eregontp@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/perfer/cli.rb
+- lib/perfer/configuration.rb
+- lib/perfer/errors.rb
+- lib/perfer/formatter/measurements_formatter.rb
+- lib/perfer/formatter/results_formatter.rb
+- lib/perfer/formatter/session_formatter.rb
+- lib/perfer/formatter.rb
+- lib/perfer/git.rb
+- lib/perfer/job/input_size_job.rb
+- lib/perfer/job/iteration_job.rb
+- lib/perfer/job.rb
+- lib/perfer/platform/posix.rb
+- lib/perfer/platform/windows.rb
+- lib/perfer/platform.rb
+- lib/perfer/result.rb
+- lib/perfer/session.rb
+- lib/perfer/statistics.rb
+- lib/perfer/store.rb
+- lib/perfer.rb
+- README.md
+- LICENSE
+- perfer.gemspec
+homepage: https://github.com/jruby/perfer
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A benchmark tool for all rubies!
+test_files: []
+has_rdoc: