fruity 0.1.0

data/.irbrc

@@ -0,0 +1 @@
+require "./lib/fruity"

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  # gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,26 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2.2)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  rspec (~> 2.3.0)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Marc-Andre Lafortune
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,120 @@
+= Fruity
+
+Make sure you're not comparing apples and oranges!
+
+Fruity makes it easy to *accurately* and *quickly* compare the performance of different Ruby algorithms.
+
+No need to decide how many times to run your stuff or to try to gauge how significant a speed difference can be, Fruity does it for you.
+
+= Install
+
+    gem install fruity
+
+= Usage
+
+  require 'fruity'
+  compare do
+    slow      { sleep(0.06) }
+    also_slow { sleep(0.03); sleep(0.03) }
+    quick     { sleep(0.03) }
+    quicker   { sleep(0.01) }
+  end
+
+Prints:
+
+  quicker is faster than quick by 3.0x ± 0.01
+  quick is faster than slow by 1.99x ± 0.01
+  slow is similar to also_slow
+
+= Alternate APIs
+
+There are many ways to specify what to compare.
+
+== Methods
+
+  class Foo
+    def slow
+      delay(0.2)
+    end
+    def faster
+      delay(0.1)
+    end
+  end
+  
+  compare :slow, :faster, :on => Foo.new
+
+  # Also, no need to specify the :on option for global methods:
+  def foo
+    # ...
+  end
+  def bar
+    # ...
+  end
+
+  compare :foo, :bar
+
+== Hash
+
+You can pass a hash with executable values; the keys will be used for to name the results.
+
+  compare(
+    foo: ->{ 2 *  2 },
+    bar: ->{ 2 ** 2 },
+  )
+
+  == List of callables
+
+  You can pass a list of callable objects:
+
+    compare(
+      ->{ "foo".upcase },
+      Proc.new{ "foo".upcase },
+      "foo".method(:upcase),
+    )
+
+  These will be named "Code 1", "Code 2" & "Code 3" respectively.
+
+== Block with parameter
+
+If you prefer, you can pass a block that accepts a parameter:
+
+  compare do |cp|
+    cp.foo { ... }
+    cp.bar { ... }
+  end
+
+== Approach
+
+Benchmarking is not trivial. A well-behaved comparison tool should:
+1) report there is no significant difference for identical blocks
+2) but report a very small difference if it is systematic (e.g between ->{ sleep(1.0) } and ->{ sleep(1.001) } )
+3) report the right performance factor, e.g. a ~2x speed difference for a block that does exactly twice what another is doing, like ->{ 2+2; 2+2 } vs ->{ 2+2 })
+
+I know of no other benchmarking tool that passes these tests.
+
+For example, the most scientifically minded tool I found (better-benchmark) fails 1 & 3; if reports a statistically significant difference of 0.5% for two identical blocks, while reporting a 4% difference for a block doing twice what the previous block is doing.
+
+In addition, all benchmarking tools require the user to either write their own inner loop (e.g. `1000.times{...}` ) or provide a number of inner iterations. Both can be better done automatically by a computer to minimize the time taken to do the test and provide acceptable reliability.
+
+== Algorithm
+
+We first determine the number of inner iterations needed to get a meaningful clock measurement (see sufficient_magnitude).
+When timing an execution, we always compare to a baseline (the time taken by an empty loop of the same magnitude).
+We call the different executables in succession, to minimize the impact on the order of execution.
+We calculate the error by taking into account the standard deviation of the time samples.
+
+== Contributing to fruity
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2012 Marc-Andre Lafortune. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "fruity"
+  gem.homepage = "http://github.com/marcandre/fruity"
+  gem.license = "MIT"
+  gem.summary = %Q{Cool performance comparison tool}
+  gem.description = %Q{Comparing apples with apples}
+  gem.email = "github@marc-andre.ca"
+  gem.authors = ["Marc-Andre Lafortune"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "fruity #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/fruity.rb

@@ -0,0 +1,3 @@
+require_relative "fruity/base"
+
+extend Fruity

data/lib/fruity/base.rb

@@ -0,0 +1,44 @@
+require_relative "baseline"
+require_relative "util"
+require_relative "runner"
+require_relative "comparison_run"
+require_relative "group"
+require_relative "named_block_collector"
+
+Fruity::GLOBAL_SCOPE = self
+
+module Fruity
+  DEFAULT_OPTIONS = {
+    :on         => GLOBAL_SCOPE,
+    :samples    => 20,
+    :disable_gc => false,
+    :filter     => [0, 0.2],
+    :baseline   => :split,
+  }
+
+  OTHER_OPTIONS = [
+    :magnitude,
+    :args,
+    :self,
+    :verbose,
+  ]
+
+  OPTIONS = DEFAULT_OPTIONS.keys + OTHER_OPTIONS
+
+  def report(*stuff, &block)
+    Fruity::Runner.new(Fruity::Group.new(*stuff, &block)).run(&:feedback)
+  end
+
+  def compare(*stuff, &block)
+    puts report(*stuff, &block)
+  end
+
+  def study(*stuff, &block)
+    run = Fruity::Runner.new(Fruity::Group.new(*stuff, &block)).run(:baseline => :single, &:feedback)
+    path = run.export
+    `open "#{path}"`
+    run
+  end
+
+  extend self
+end

data/lib/fruity/baseline.rb

@@ -0,0 +1,79 @@
+module Fruity
+
+  # Utility module for building
+  # baseline equivalents for callables.
+  #
+  module Baseline
+    extend self
+
+    # Returns the baseline for the given callable object
+    # The signature (number of arguments) and type (proc, ...)
+    # will be copied as much as possible.
+    #
+    def [](exec)
+      kind = callable_kind(exec)
+      signature = callable_signature(exec)
+      NOOPs[kind][signature] ||= build_baseline(kind, signature)
+    end
+
+    NOOPs = Hash.new{|h, k| h[k] = {}}
+
+    def callable_kind(exec)
+      if exec.is_a?(Method)
+        exec.source_location ? :method : :builtin_method
+      elsif exec.lambda?
+        :lambda
+      else
+        :proc
+      end
+    end
+
+    def callable_signature(exec)
+      if exec.respond_to?(:parameters)
+        exec.parameters.map(&:first)
+      else
+        # Ruby 1.8 didn't have parameters, so rely on arity
+        opt = exec.arity < 0
+        req = opt ? -1-exec.arity : exec.arity
+        signature = [:req] * req
+        signature << :rest if opt
+        signature
+      end
+    end
+
+    PARAM_MAP = {
+      :req => "%{name}",
+      :opt => "%{name} = nil",
+      :rest => "*%{name}",
+      :block => "&%{name}",
+    }
+
+    def arg_list(signature)
+      signature.map.with_index{|kind, i| PARAM_MAP[kind] % {:name => "p#{i}"}}.join(",")
+    end
+
+    def build_baseline(kind, signature)
+      args = "|#{arg_list(signature)}|"
+      case kind
+      when :lambda, :proc
+        eval("#{kind}{#{args}}")
+      when :builtin_method
+        case signature
+        when []
+          nil.method(:nil?)
+        when [:req]
+          nil.method(:==)
+        else
+          Array.method(:[])
+        end
+      when :method
+        @method_counter ||= 0
+        @method_counter += 1
+        name = "baseline_#{@method_counter}"
+        eval("define_method(:#{name}){#{args}}")
+        method(name)
+      end
+    end
+  end
+end
+  

data/lib/fruity/comparison_run.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+
+module Fruity
+  class ComparisonRun < Struct.new(:group, :timings, :baselines)
+    attr_reader :stats
+
+    # +timings+ must be an array of size `group.size` of arrays of delays
+    # or of arrays of [delay, baseline]
+    #
+    def initialize(group, timings, baselines)
+      raise ArgumentError, "Expected timings to be an array with #{group.size} elements (was #{timings.size})" unless timings.size == group.size
+      super
+
+      filter = group.options.fetch(:filter)
+
+      baseline = Util.filter(baselines, *filter) if baseline_type == :single
+
+      @stats = timings.map.with_index do |series, i|
+        case baseline_type
+        when :split
+          Util.difference(Util.filter(series, *filter), Util.filter(baselines.fetch(i), *filter))
+        when :single
+          Util.difference(Util.filter(series, *filter), baseline)
+        when :none
+          Util.stats(series)
+        end
+      end.freeze
+    end
+
+    def to_s
+      order = (0...group.size).sort_by{|i| @stats[i][:mean] }
+      results = group.elements.map{|n, exec| Util.result_of(exec, group.options) }
+      order.each_cons(2).map do |i, j|
+        cmp = comparison(i, j)
+        s = if cmp[:factor] == 1
+          "%{cur} is similar to %{vs}%{different}"
+        else
+          "%{cur} is faster than %{vs} by %{ratio}%{different}"
+        end
+        s % {
+          :cur => group.elements.keys[i],
+          :vs => group.elements.keys[j],
+          :ratio => format_comparison(cmp),
+          :different => results[i] == results[j] ? "" : " (results differ: #{results[i]} vs #{results[j]})"
+        }
+      end.join("\n")
+    end
+
+    def export(fn = (require "tmpdir"; "#{Dir.tmpdir}/export.csv"))
+      require "csv"
+      CSV.open(fn, "wb") do |csv|
+        head = group.elements.keys
+        case baseline_type
+        when :split
+          head = head.flat_map{|h| [h, "#{head} bl"]}
+          data = timings.zip(baselines).flatten(1).transpose
+        when :single
+          data = (timings + [baselines]).transpose
+          head << "baseline"
+        else
+          data = timings.transpose
+        end
+        csv << head
+        data.each{|vals| csv << vals}
+      end
+      fn
+    end
+
+    def size
+      timings.first.size
+    end
+
+    def factor(cur = 0, vs = 1)
+      comparison(cur, vs)[:factor]
+    end
+
+    def factor_range(cur = 0, vs =1)
+      comparison(cur, vs)[:min]..comparison(cur, vs)[:max]
+    end
+
+    def comparison(cur = 0, vs = 1)
+      Util.compare_stats(@stats[cur], @stats[vs])
+    end
+
+    def format_comparison(cmp)
+      ratio = cmp[:factor]
+      prec = cmp[:precision]
+      if ratio.abs > 1.8
+        "#{ratio}x ± #{prec}"
+      else
+        "#{(ratio - 1)*100}% ± #{prec*100}%"
+      end
+    end
+
+    def baseline_type
+      if baselines.nil?
+        :none
+      elsif baselines.first.is_a?(Array)
+        :split
+      else
+        :single
+      end
+    end
+  end
+end

data/lib/fruity/group.rb

@@ -0,0 +1,102 @@
+module Fruity
+
+  # A group of callable objects
+  #
+  class Group
+    attr_reader :elements
+    attr_reader :options
+
+    # Pass either a list of callable objects, a Hash of names and callable objects
+    # or an Array of methods names with the option :on specifying which object to call
+    # them on (or else the methods are assumed to be global, see the README)
+    # Another possibility is to use a block; if it accepts an argument, 
+    #
+    def initialize(*args, &block)
+      @options = DEFAULT_OPTIONS.dup
+      @elements = {}
+      @counter = 0
+      compare(*args, &block)
+    end
+
+    # Adds things to compare. See +new+ for details on interface
+    #
+    def compare(*args, &block)
+      if args.last.is_a?(Hash) && (args.last.keys - OPTIONS).empty?
+        @options.merge!(args.pop)
+      end
+      case args.first
+        when Hash
+          raise ArgumentError, "Expected only one hash of {value => executable}, got #{args.size-1} extra arguments" unless args.size == 1
+          raise ArgumentError, "Expected values to be executable" unless args.first.values.all?{|v| v.respond_to?(:call)}
+          compare_hash(args.first)
+        when Symbol, String
+          compare_methods(*args)
+        else
+          compare_lambdas(*args)
+      end
+      compare_block(block) if block
+    end
+
+    # Returns the maximal sufficient_magnitude for all elements
+    # See Util.sufficient_magnitude
+    #
+    def sufficient_magnitude
+      elements.map{|name, exec| Util.sufficient_magnitude(exec, options) }.max
+    end
+
+    # Returns the maximal sufficient_magnitude for all elements
+    # and the approximate delay taken for the whole group
+    # See Util.sufficient_magnitude
+    #
+    def sufficient_magnitude_and_delay
+      mags_and_delays = elements.map{|name, exec| Util.sufficient_magnitude_and_delay(exec, options) }
+      mag = mags_and_delays.map(&:first).max
+      delay = mags_and_delays.map{|m, d| d * mag / m}.inject(:+)
+      [mag, delay]
+    end
+
+    def size
+      elements.size
+    end
+
+    def run(options = {})
+      Runner.new(self).run(options)
+    end
+
+  private
+    def compare_hash(h)
+      elements.merge!(h)
+    end
+
+    def compare_methods(*args)
+      on = @options[:on]
+      args.each do |m|
+        elements[m] = on.method(m)
+      end
+    end
+
+    def compare_lambdas(*lambdas)
+      lambdas.flat_map{|o| Array(o)}
+      lambdas.each do |name, callable|
+        name, callable = generate_name(name), name unless callable
+        raise "Excepted a callable object, got #{callable}" unless callable.respond_to?(:call)
+        elements[name] = callable
+      end
+    end
+
+    def compare_block(block)
+      collect = NamedBlockCollector.new(@elements)
+      if block.arity == 0
+        @options[:self] = block.binding.eval("self")
+        collect.instance_eval(&block)
+      else
+        block.call(collect)
+      end
+    end
+
+    def generate_name(callable)
+      "Code #{@counter += 1}"
+    end
+  end
+end
+

data/lib/fruity/named_block_collector.rb

@@ -0,0 +1,12 @@
+module Fruity
+  class NamedBlockCollector
+    def initialize(to_hash)
+      @to_hash = to_hash
+    end
+  
+    def method_missing(method, *args, &block)
+      super unless args.empty?
+      @to_hash[method] = block
+    end
+  end
+end

data/lib/fruity/runner.rb

@@ -0,0 +1,70 @@
+module Fruity
+  class Runner < Struct.new(:group)
+    attr_reader :options, :delay, :timings, :baselines
+
+    def run(options = {})
+      prepare(options)
+      yield self if block_given?
+      sample
+    end
+
+    def feedback
+      mess = "Running each test " << (options[:magnitude] == 1 ? "once." : "#{options[:magnitude]} times.")
+      if d = delay
+        if d > 60
+          d = (d / 60).round
+          unit = "minute"
+        end
+        mess << " Test will take about #{d.ceil} #{unit || 'second'}#{d > 1 ? 's' : ''}."
+      end
+      puts mess
+    end
+
+  private
+    def prepare(opt)
+      @options = group.options.merge(opt)
+      unless options[:magnitude]
+        options[:magnitude], @delay = group.sufficient_magnitude_and_delay
+        @delay *= options.fetch(:samples)
+      end
+    end
+
+    def sample
+      send(:"sample_baseline_#{options.fetch(:baseline)}")
+      ComparisonRun.new(group, timings, baselines)
+    end
+
+    def sample_baseline_split
+      baselines = group.elements.map{|name, exec| Baseline[exec]}
+      exec_and_baselines = group.elements.values.zip(baselines)
+      @baselines, @timings = options.fetch(:samples).times.map do
+        exec_and_baselines.flat_map do |exec, baseline|
+          [
+            Util.real_time(baseline, options),
+            Util.real_time(exec, options),
+          ]
+        end
+      end.transpose.each_slice(2).to_a.transpose
+    end
+
+    def sample_baseline_single
+      baseline = Baseline[group.elements.first.last]
+      @baselines = []
+      @timings = options.fetch(:samples).times.map do
+        baselines << Util.real_time(baseline, options)
+        group.elements.map do |name, exec|
+          Util.real_time(exec, options)
+        end
+      end.transpose
+    end
+
+    def sample_baseline_none
+      @baselines = nil
+      @timings = options.fetch(:samples).times.map do
+        group.elements.map do |name, exec|
+          Util.real_time(exec, options)
+        end
+      end.transpose
+    end
+  end
+end

data/lib/fruity/util.rb

@@ -0,0 +1,192 @@
+require 'benchmark'
+module Fruity
+
+  # Utility module doing most of the maths
+  #
+  module Util
+    extend self
+
+    PROPER_TIME_RELATIVE_ERROR = 0.001
+
+    MEASUREMENTS_BY_REALTIME = 2
+    MEASUREMENTS_BY_PROPER_TIME = 2 * MEASUREMENTS_BY_REALTIME
+
+    # Measures the smallest obtainable delta of two time measurements
+    #
+    def clock_precision
+      @clock_precision ||= 10.times.map do
+        t = Time.now
+        delta = Time.now - t
+        while delta == 0
+          delta = Time.now - t
+        end
+        delta
+      end.min
+    end
+
+
+    APPROX_POWER = 5
+    # Calculates the number +n+ that needs to be passed
+    # to +real_time+ to get a result that is precise
+    # to within +PROPER_TIME_RELATIVE_ERROR+ when compared
+    # to the baseline.
+    #
+    # For example, ->{ sleep(1) } needs only to be run once to get
+    # a measurement that will not be affected by the inherent imprecision
+    # of the time measurement (or of the inner loop), but ->{ 2 + 2 } needs to be called
+    # a huge number of times so that the returned time measurement is not
+    # due in big part to the imprecision of the measurement itself
+    # or the inner loop itself.
+    #
+    def sufficient_magnitude(exec, options = {})
+      mag, delay = sufficient_magnitude_and_delay(exec, options)
+      mag
+    end
+
+    BASELINE_THRESHOLD = 1.02 # Ratio between two identical baselines is typically < 1.02, while {2+2} compared to baseline is typically > 1.02
+
+    def sufficient_magnitude_and_delay(exec, options = {})
+      power = 0
+      min_desired_delta = clock_precision * MEASUREMENTS_BY_PROPER_TIME / PROPER_TIME_RELATIVE_ERROR
+      # First, make a gross approximation with a single sample and no baseline
+      min_approx_delay = min_desired_delta / (1 << APPROX_POWER)
+      while (delay = real_time(exec, options.merge(:magnitude => 1 << power))) < min_approx_delay
+        power += [Math.log(min_approx_delay.div(delay + clock_precision), 2), 1].max.floor
+      end
+
+      # Then take a couple of samples, along with a baseline
+      power += 1 unless delay > 2 * min_approx_delay
+      group = Group.new(exec, Baseline[exec], options.merge(:baseline => :none, :samples => 5, :filter => [0, 0.25], :magnitude => 1 << power))
+      stats = group.run.stats
+      if stats[0][:mean] / stats[1][:mean] < 2
+        # Quite close to baseline, which means we need to be more discriminant
+        power += APPROX_POWER
+        stats = group.run(:samples => 40, :magnitude => 1 << power).stats
+        raise "Given callable can not be reasonably distinguished from an empty block" if stats[0][:mean] / stats[1][:mean] < BASELINE_THRESHOLD
+      end
+      delta = stats[0][:mean] - stats[1][:mean]
+      addl_power = [Math.log(min_desired_delta.div(delta), 2), 0].max.floor
+      [
+        1 << (power + addl_power),
+        stats[0][:mean] * (1 << addl_power),
+      ]
+    end
+
+    # The proper time is the real time taken by calling +exec+
+    # number of times given by +options[:magnitude]+ minus
+    # the real time for calling an empty executable instead.
+    #
+    # If +options[:magnitude]+ is not given, it will be calculated to be meaningful.
+    #
+    def proper_time(exec, options = {})
+      unless options.has_key?(:magnitude)
+        options = {:magnitude => sufficient_magnitude(exec, options)}.merge(options)
+      end
+      real_time(exec, options) - real_time(Baseline[exec], options)
+    end
+
+    # Returns the real time taken by calling +exec+
+    # number of times given by +options[:magnitude]+
+    #
+    def real_time(exec, options = {})
+      GC.start
+      GC.disable if options[:disable_gc]
+      n = options.fetch(:magnitude)
+      if options.has_key?(:self)
+        new_self = options[:self]
+        if args = options[:args] and args.size > 0
+          Benchmark.realtime{ n.times{ new_self.instance_exec(*args, &exec) } }
+        else
+          Benchmark.realtime{ n.times{ new_self.instance_eval(&exec) } }
+        end
+      else
+        if args = options[:args] and args.size > 0
+          Benchmark.realtime{ n.times{ exec.call(*args) } }
+        else
+          Benchmark.realtime{ n.times{ exec.call } }
+        end
+      end
+    ensure
+      GC.enable
+    end
+
+    # Returns the result of calling +exec+
+    #
+    def result_of(exec, options = {})
+      args = (options[:args] || [])
+      if options.has_key?(:self)
+        options[:self].instance_exec(*args, &exec)
+      else
+        exec.call(*args)
+      end
+    end
+
+    # Returns the inherent precision of +proper_time+
+    #
+    def proper_time_precision
+      MEASUREMENTS_BY_PROPER_TIME * clock_precision
+    end
+
+    # Calculates stats on some values: {:min, :max, :mean, :sample_std_dev }
+    #
+    def stats(values)
+      sum = values.inject(0, :+)
+      # See http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
+      q = mean = 0
+      values.each_with_index do |x, k|
+        prev_mean = mean
+        mean += (x - prev_mean) / (k + 1)
+        q += (x - mean) * (x - prev_mean)
+      end
+      sample_std_dev = Math.sqrt( q / (values.size-1) )
+      min, max = values.minmax
+      {
+        :min => min,
+        :max => max,
+        :mean => mean,
+        :sample_std_dev => sample_std_dev
+      }
+    end
+
+    # Calculates the stats of the difference of +values+ and +baseline+
+    # (which can be stats or timings)
+    #
+    def difference(values, baseline)
+      values, baseline = [values, baseline].map{|x| x.is_a?(Hash) ? x : stats(x)}
+      {
+        :min            => values[:min]  - baseline[:max],
+        :max            => values[:max]  - baseline[:min],
+        :mean           => values[:mean] - baseline[:mean],
+        :sample_std_dev => Math.sqrt(values[:sample_std_dev] ** 2 + values[:sample_std_dev] ** 2),
+        # See http://stats.stackexchange.com/questions/6096/correct-way-to-calibrate-means
+      }
+    end
+
+    # Given two stats +cur+ and +vs+, returns a hash with
+    # the ratio between the two, the precision, etc.
+    #
+    def compare_stats(cur, vs)
+      err = (vs[:sample_std_dev] +
+             cur[:sample_std_dev] * vs[:mean] / cur[:mean]
+            ) / cur[:mean]
+
+      rounding = err > 0 ? -Math.log(err, 10) : 666
+      mean = vs[:mean] / cur[:mean]
+      {
+        :mean     => mean,
+        :factor   => (mean).round(rounding),
+        :max      => (vs[:mean] + vs[:sample_std_dev]) / (cur[:mean] - cur[:sample_std_dev]),
+        :min      => (vs[:mean] - vs[:sample_std_dev]) / (cur[:mean] + cur[:sample_std_dev]),
+        :rounding => rounding,
+        :precision => 10.0 **(-rounding.round),
+      }
+    end
+
+    def filter(series, remove_min_ratio, remove_max_ratio = remove_min_ratio)
+      series.sort![
+        (remove_min_ratio * series.size).floor ...
+        ((1-remove_max_ratio) * series.size).ceil
+      ]
+    end
+  end
+end

data/spec/baseline_spec.rb

@@ -0,0 +1,36 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module Fruity
+  describe Baseline do
+    describe "#[]" do
+      it "returns an object of the right type" do
+        b = Baseline[Proc.new{ 42 }]
+        b.class.should == Proc
+        b.lambda?.should == false
+        b.call.should == nil
+
+        b = Baseline[lambda{ 42 }]
+        b.class.should == Proc
+        b.lambda?.should == true
+        b.call.should == nil
+
+        b = Baseline[Fruity.method(:compare)]
+        b.class.should == Method
+        b.call.should == nil
+      end
+
+      it "copies the arity" do
+        b = Baseline[lambda{|a, b| 42}]
+        b.call(1, 2).should == nil
+        lambda{
+          b.call(1)
+        }.should raise_error
+        lambda{
+          b.call(1, 2, 3)
+        }.should raise_error
+        
+        Baseline[lambda{|a, *b| 42}].call(1, 2, 3, 4, 5, 6).should == nil
+      end
+    end
+  end
+end

data/spec/comparison_run_spec.rb

@@ -0,0 +1,20 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module Fruity
+  describe ComparisonRun do
+    let(:group)  { Group.new ->{1}, ->{2} }
+    let(:timings){ ([[1.0, 2.0]] * 10).transpose }
+    subject      { @run = ComparisonRun.new(group, timings, nil) }
+
+    its(:comparison) { should == {
+        :mean=>2.0,
+        :factor=>2.0,
+        :max=>2.0,
+        :min=>2.0,
+        :rounding=>666,
+        :precision=>0.0,
+      }
+    }
+  
+  end
+end

data/spec/find_thresold.rb

@@ -0,0 +1,19 @@
+require_relative "../lib/fruity"
+
+power = 15
+x = 2
+n = 30
+min = 1
+max = 2
+s = 10
+n.times.map do
+  group = Fruity::Group.new(*[->{}] * s, *[->{ 2 + 2 }] * s, :baseline => :none, :samples => 20, :magnitude => 1 << power)
+  means = group.run.stats.map{|s| s[:mean]}
+  noops = means.first(s).sort
+  plus  = means.last(s).sort
+  min = [min, noops.last / noops.first].max
+  max = [max, plus.first / noops.last].min
+  p min, max
+  raise "Damn, #{min} >= #{max}" if min >= max
+end.sort
+puts "Threshold between #{min} and #{max} are ok, suggesting #{Math.sqrt(min * max)}"

data/spec/fruity_spec.rb

@@ -0,0 +1,18 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Fruity do
+  it "is able to report that two identical blocks are identical" do
+    report = Fruity.report(->{ 2 ** 2 }, ->{ 2 ** 2 })
+    report.factor.should == 1.0
+  end
+
+  it "is able to report very small performance differences" do
+    report = Fruity.report(->{ sleep(1.0) }, ->{ sleep(1.001) })
+    report.factor.should == 1.001
+  end
+
+  it "is able to report a 2x difference even for small blocks" do
+    report = Fruity.report(->{ 2 ** 2 }, ->{ 2 ** 2 ; 2 ** 2 })
+    report.factor.should == 2.0
+  end
+end

data/spec/group_spec.rb

@@ -0,0 +1,58 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+TOP_LEVEL = self
+def foo; end
+def bar; end
+
+module Fruity
+  describe Group do
+    lambdas = [->{1},->{2}]
+
+    it "can be built from lambdas" do
+      group = Group.new(*lambdas)
+      group.elements.values.should == lambdas
+    end
+
+    it "can be built from a block with no parameter" do
+      group = Group.new do
+        first(&lambdas.first)
+        last(&lambdas.last)
+      end
+      group.elements.should == {
+        :first => lambdas.first,
+        :last  => lambdas.last,
+      }
+      group.options[:self].object_id.should equal(self.object_id)
+    end
+
+    it "can be built from a block taking a parameter" do
+      group = Group.new do |cmp|
+        cmp.first(&lambdas.first)
+        cmp.last(&lambdas.last)
+        ->{ second{} }.should raise_error(NameError)
+      end
+      group.elements.should == {
+        :first => lambdas.first,
+        :last  => lambdas.last,
+      }
+      group.options[:self].should == nil
+    end
+
+    it "can be built from list of method names and an object" do
+      str = "Hello"
+      group = Group.new(:upcase, :downcase, :on => str)
+      group.elements.should == {
+        :upcase => str.method(:upcase),
+        :downcase => str.method(:downcase),
+      }
+    end
+
+    it "can be built from list of method names and an object" do
+      group = Group.new(:foo, :bar)
+      group.elements.should == {
+        :foo => TOP_LEVEL.method(:foo),
+        :bar => TOP_LEVEL.method(:bar),
+      }
+    end
+  end
+end

data/spec/manual_test.rb

@@ -0,0 +1,14 @@
+require_relative "../lib/fruity"
+
+compare do
+  one       { 42.nil? }
+  two       { 42.nil?; 42.nil? }
+end
+
+compare do
+  two       { 42.nil?; 42.nil? }
+  two_again { 42.nil?; 42.nil? }
+  three     { 42.nil?; 42.nil?; 42.nil? }
+  four      { 42.nil?; 42.nil?; 42.nil?; 42.nil? }
+  five      { 42.nil?; 42.nil?; 42.nil?; 42.nil?; 42.nil? }
+end

data/spec/runner_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module Fruity
+  describe Runner do
+    let(:group) { Group.new(:upcase, :downcase, :on => "Hello") }
+    let(:runner){ Runner.new(group) }
+
+    it "runs from a Group" do
+      run = runner.run(:samples => 42, :magnitude => 100)
+      run.timings.should be_array_of_size(2, 42)
+      run.baselines.should be_array_of_size(2, 42)
+    end
+
+    it "can use a single baseline" do
+      run = runner.run(:samples => 42, :magnitude => 100, :baseline => :single)
+      run.timings.should be_array_of_size(2, 42)
+      run.baselines.should be_array_of_size(42)
+    end
+
+    it "can use no baseline" do
+      run = runner.run(:samples => 42, :magnitude => 100, :baseline => :none)
+      run.timings.should be_array_of_size(2, 42)
+      run.baselines.should == nil
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,47 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'fruity'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end
+
+RSpec::Matchers.define :be_between do |low,high|
+  match do |actual|
+    @low, @high = low, high
+    actual.between? low, high
+  end
+
+  failure_message_for_should do |actual|
+    "expected to be between #{@low} and #{@high}, but was #{actual}"
+  end
+
+  failure_message_for_should_not do |actual|
+    "expected not to be between #{@low} and #{@high}, but was #{actual}"
+  end
+end
+
+RSpec::Matchers.define :be_array_of_size do |*dimensions|
+  match do |actual|
+    @dimensions = dimensions
+    @actual = []
+    while actual.is_a?(Array)
+      @actual << actual.size
+      actual = actual.first
+    end
+    @dimensions == @actual
+  end
+
+  failure_message_for_should do |actual|
+    "expected to be an array of dimension #{@dimensions.join('x')} but was #{@actual.join('x')}"
+  end
+
+  failure_message_for_should_not do |actual|
+    "expected not to be an array of dimension #{@dimensions.join('x')}"
+  end
+end

data/spec/util_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module Fruity
+  describe Util do
+    its(:clock_precision) { should == 1.0e-6 }
+
+    its(:proper_time_precision) { should == 4.0e-06 }
+
+    describe :sufficient_magnitude do
+      it "returns a big value for a quick (but not trivial)" do
+        Util.sufficient_magnitude(->{ 2.nil? }).should > 10000
+      end
+
+      it "return 1 for a sufficiently slow block" do
+        Util.sufficient_magnitude(->{sleep(0.01)}).should == 1
+      end
+
+      it "should raise an error for a trivial block" do
+        ->{
+          Util.sufficient_magnitude(Proc.new{})
+        }.should raise_error
+      end
+    end
+
+    describe :stats do
+      it "returns cools stats on the given values" do
+        Util.stats([0, 4]).should == {
+          :min => 0,
+          :max => 4,
+          :mean => 2,
+          :sample_std_dev => 2.8284271247461903,
+        }
+      end
+    end
+
+    describe :difference do
+      it "returns stats for the difference of two series" do
+        s = Util.stats([0, 4])
+        Util.difference(s, s).should == {
+          :min => -4,
+          :max => 4,
+          :mean => 0,
+          :sample_std_dev => 4,
+        }
+      end
+
+      it "gives similar results when comparing an exec and its baseline from stats on proper_time" do
+        exec = ->{ 2 ** 3 ** 4 }
+        options = {:magnitude => Util.sufficient_magnitude(exec) }
+        n = 100
+        timings  = [exec, ->{}].map do |e|
+          n.times.map { Util.real_time(e, options) }
+        end
+        proper = Util.stats(timings.transpose.map{|e, b| e - b})
+        diff = Util.difference(*timings)
+        diff[:mean].should   be_within(Float::EPSILON).of(proper[:mean])
+        diff[:max].should    be_between(proper[:max], proper[:max]*2)
+        diff[:min].should    <= proper[:min]
+        diff[:sample_std_dev].should be_between(proper[:sample_std_dev], 2 * proper[:sample_std_dev])
+      end
+    end
+
+    describe :filter do
+      it "returns the filtered series" do
+        Util.filter([4, 5, 2, 3, 1], 0.21, 0.39).should == [2, 3, 4]
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: fruity
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Marc-Andre Lafortune
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2170028460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *2170028460
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &2170027780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2170027780
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &2170047220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *2170047220
+description: Comparing apples with apples
+email: github@marc-andre.ca
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .irbrc
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/fruity.rb
+- lib/fruity/base.rb
+- lib/fruity/baseline.rb
+- lib/fruity/comparison_run.rb
+- lib/fruity/group.rb
+- lib/fruity/named_block_collector.rb
+- lib/fruity/runner.rb
+- lib/fruity/util.rb
+- spec/baseline_spec.rb
+- spec/comparison_run_spec.rb
+- spec/find_thresold.rb
+- spec/fruity_spec.rb
+- spec/group_spec.rb
+- spec/manual_test.rb
+- spec/runner_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/util_spec.rb
+homepage: http://github.com/marcandre/fruity
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3031735817276906554
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Cool performance comparison tool
+test_files: []