betatest 0.0.1

data/lib/betatest/autorun.rb

@@ -0,0 +1,12 @@
+begin
+  require "rubygems"
+  gem "betatest"
+rescue Gem::LoadError
+  # do nothing
+end
+
+require "betatest"
+require "betatest/spec"
+require "betatest/mock"
+
+Betatest.autorun

data/lib/betatest/benchmark.rb

@@ -0,0 +1,423 @@
+require 'betatest/unit'
+require 'betatest/spec'
+
+module Betatest
+  ##
+  # Subclass Benchmark to create your own benchmark runs. Methods
+  # starting with "bench_" get executed on a per-class.
+  #
+  # See Betatest::Assertions
+
+  class Benchmark < Test
+    def self.io # :nodoc:
+      @io
+    end
+
+    def io # :nodoc:
+      self.class.io
+    end
+
+    def self.run reporter, options = {} # :nodoc:
+      # NOTE: this is truly horrible... but I don't see a way around this ATM.
+      @io = reporter.reporters.first.io
+      super
+    end
+
+    def self.runnable_methods # :nodoc:
+      methods_matching(/^bench_/)
+    end
+
+    ##
+    # Returns a set of ranges stepped exponentially from +min+ to
+    # +max+ by powers of +base+. Eg:
+    #
+    #   bench_exp(2, 16, 2) # => [2, 4, 8, 16]
+
+    def self.bench_exp min, max, base = 10
+      min = (Math.log10(min) / Math.log10(base)).to_i
+      max = (Math.log10(max) / Math.log10(base)).to_i
+
+      (min..max).map { |m| base ** m }.to_a
+    end
+
+    ##
+    # Returns a set of ranges stepped linearly from +min+ to +max+ by
+    # +step+. Eg:
+    #
+    #   bench_linear(20, 40, 10) # => [20, 30, 40]
+
+    def self.bench_linear min, max, step = 10
+      (min..max).step(step).to_a
+    rescue LocalJumpError # 1.8.6
+      r = []; (min..max).step(step) { |n| r << n }; r
+    end
+
+    ##
+    # Specifies the ranges used for benchmarking for that class.
+    # Defaults to exponential growth from 1 to 10k by powers of 10.
+    # Override if you need different ranges for your benchmarks.
+    #
+    # See also: ::bench_exp and ::bench_linear.
+
+    def self.bench_range
+      bench_exp 1, 10_000
+    end
+
+    ##
+    # Runs the given +work+, gathering the times of each run. Range
+    # and times are then passed to a given +validation+ proc. Outputs
+    # the benchmark name and times in tab-separated format, making it
+    # easy to paste into a spreadsheet for graphing or further
+    # analysis.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     validation = proc { |x, y| ... }
+    #     assert_performance validation do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def assert_performance validation, &work
+      range = self.class.bench_range
+
+      io.print "#{self.name}"
+
+      times = []
+
+      range.each do |x|
+        GC.start
+        t0 = Time.now
+        instance_exec(x, &work)
+        t = Time.now - t0
+
+        io.print "\t%9.6f" % t
+        times << t
+      end
+      io.puts
+
+      validation[range, times]
+    end
+
+    ##
+    # Runs the given +work+ and asserts that the times gathered fit to
+    # match a constant rate (eg, linear slope == 0) within a given
+    # +threshold+. Note: because we're testing for a slope of 0, R^2
+    # is not a good determining factor for the fit, so the threshold
+    # is applied against the slope itself. As such, you probably want
+    # to tighten it from the default.
+    #
+    # See http://www.graphpad.com/curvefit/goodness_of_fit.htm for
+    # more details.
+    #
+    # Fit is calculated by #fit_linear.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     assert_performance_constant 0.9999 do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def assert_performance_constant threshold = 0.99, &work
+      validation = proc do |range, times|
+        a, b, rr = fit_linear range, times
+        assert_in_delta 0, b, 1 - threshold
+        [a, b, rr]
+      end
+
+      assert_performance validation, &work
+    end
+
+    ##
+    # Runs the given +work+ and asserts that the times gathered fit to
+    # match a exponential curve within a given error +threshold+.
+    #
+    # Fit is calculated by #fit_exponential.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     assert_performance_exponential 0.9999 do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def assert_performance_exponential threshold = 0.99, &work
+      assert_performance validation_for_fit(:exponential, threshold), &work
+    end
+
+    ##
+    # Runs the given +work+ and asserts that the times gathered fit to
+    # match a logarithmic curve within a given error +threshold+.
+    #
+    # Fit is calculated by #fit_logarithmic.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     assert_performance_logarithmic 0.9999 do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def assert_performance_logarithmic threshold = 0.99, &work
+      assert_performance validation_for_fit(:logarithmic, threshold), &work
+    end
+
+    ##
+    # Runs the given +work+ and asserts that the times gathered fit to
+    # match a straight line within a given error +threshold+.
+    #
+    # Fit is calculated by #fit_linear.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     assert_performance_linear 0.9999 do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def assert_performance_linear threshold = 0.99, &work
+      assert_performance validation_for_fit(:linear, threshold), &work
+    end
+
+    ##
+    # Runs the given +work+ and asserts that the times gathered curve
+    # fit to match a power curve within a given error +threshold+.
+    #
+    # Fit is calculated by #fit_power.
+    #
+    # Ranges are specified by ::bench_range.
+    #
+    # Eg:
+    #
+    #   def bench_algorithm
+    #     assert_performance_power 0.9999 do |x|
+    #       @obj.algorithm
+    #     end
+    #   end
+
+    def assert_performance_power threshold = 0.99, &work
+      assert_performance validation_for_fit(:power, threshold), &work
+    end
+
+    ##
+    # Takes an array of x/y pairs and calculates the general R^2 value.
+    #
+    # See: http://en.wikipedia.org/wiki/Coefficient_of_determination
+
+    def fit_error xys
+      y_bar  = sigma(xys) { |x, y| y } / xys.size.to_f
+      ss_tot = sigma(xys) { |x, y| (y    - y_bar) ** 2 }
+      ss_err = sigma(xys) { |x, y| (yield(x) - y) ** 2 }
+
+      1 - (ss_err / ss_tot)
+    end
+
+    ##
+    # To fit a functional form: y = ae^(bx).
+    #
+    # Takes x and y values and returns [a, b, r^2].
+    #
+    # See: http://mathworld.wolfram.com/LeastSquaresFittingExponential.html
+
+    def fit_exponential xs, ys
+      n     = xs.size
+      xys   = xs.zip(ys)
+      sxlny = sigma(xys) { |x,y| x * Math.log(y) }
+      slny  = sigma(xys) { |x,y| Math.log(y)     }
+      sx2   = sigma(xys) { |x,y| x * x           }
+      sx    = sigma xs
+
+      c = n * sx2 - sx ** 2
+      a = (slny * sx2 - sx * sxlny) / c
+      b = ( n * sxlny - sx * slny ) / c
+
+      return Math.exp(a), b, fit_error(xys) { |x| Math.exp(a + b * x) }
+    end
+
+    ##
+    # To fit a functional form: y = a + b*ln(x).
+    #
+    # Takes x and y values and returns [a, b, r^2].
+    #
+    # See: http://mathworld.wolfram.com/LeastSquaresFittingLogarithmic.html
+
+    def fit_logarithmic xs, ys
+      n     = xs.size
+      xys   = xs.zip(ys)
+      slnx2 = sigma(xys) { |x,y| Math.log(x) ** 2 }
+      slnx  = sigma(xys) { |x,y| Math.log(x)      }
+      sylnx = sigma(xys) { |x,y| y * Math.log(x)  }
+      sy    = sigma(xys) { |x,y| y                }
+
+      c = n * slnx2 - slnx ** 2
+      b = ( n * sylnx - sy * slnx ) / c
+      a = (sy - b * slnx) / n
+
+      return a, b, fit_error(xys) { |x| a + b * Math.log(x) }
+    end
+
+
+    ##
+    # Fits the functional form: a + bx.
+    #
+    # Takes x and y values and returns [a, b, r^2].
+    #
+    # See: http://mathworld.wolfram.com/LeastSquaresFitting.html
+
+    def fit_linear xs, ys
+      n   = xs.size
+      xys = xs.zip(ys)
+      sx  = sigma xs
+      sy  = sigma ys
+      sx2 = sigma(xs)  { |x|   x ** 2 }
+      sxy = sigma(xys) { |x,y| x * y  }
+
+      c = n * sx2 - sx**2
+      a = (sy * sx2 - sx * sxy) / c
+      b = ( n * sxy - sx * sy ) / c
+
+      return a, b, fit_error(xys) { |x| a + b * x }
+    end
+
+    ##
+    # To fit a functional form: y = ax^b.
+    #
+    # Takes x and y values and returns [a, b, r^2].
+    #
+    # See: http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html
+
+    def fit_power xs, ys
+      n       = xs.size
+      xys     = xs.zip(ys)
+      slnxlny = sigma(xys) { |x, y| Math.log(x) * Math.log(y) }
+      slnx    = sigma(xs)  { |x   | Math.log(x)               }
+      slny    = sigma(ys)  { |   y| Math.log(y)               }
+      slnx2   = sigma(xs)  { |x   | Math.log(x) ** 2          }
+
+      b = (n * slnxlny - slnx * slny) / (n * slnx2 - slnx ** 2);
+      a = (slny - b * slnx) / n
+
+      return Math.exp(a), b, fit_error(xys) { |x| (Math.exp(a) * (x ** b)) }
+    end
+
+    ##
+    # Enumerates over +enum+ mapping +block+ if given, returning the
+    # sum of the result. Eg:
+    #
+    #   sigma([1, 2, 3])                # => 1 + 2 + 3 => 7
+    #   sigma([1, 2, 3]) { |n| n ** 2 } # => 1 + 4 + 9 => 14
+
+    def sigma enum, &block
+      enum = enum.map(&block) if block
+      enum.inject { |sum, n| sum + n }
+    end
+
+    ##
+    # Returns a proc that calls the specified fit method and asserts
+    # that the error is within a tolerable threshold.
+
+    def validation_for_fit msg, threshold
+      proc do |range, times|
+        a, b, rr = send "fit_#{msg}", range, times
+        assert_operator rr, :>=, threshold
+        [a, b, rr]
+      end
+    end
+  end
+end
+
+module Betatest
+  class BenchSpec < Benchmark
+    extend Betatest::Spec::DSL
+
+    ##
+    # This is used to define a new benchmark method. You usually don't
+    # use this directly and is intended for those needing to write new
+    # performance curve fits (eg: you need a specific polynomial fit).
+    #
+    # See ::bench_performance_linear for an example of how to use this.
+
+    def self.bench name, &block
+      define_method "bench_#{name.gsub(/\W+/, '_')}", &block
+    end
+
+    ##
+    # Specifies the ranges used for benchmarking for that class.
+    #
+    #   bench_range do
+    #     bench_exp(2, 16, 2)
+    #   end
+    #
+    # See Betatest::Benchmark#bench_range for more details.
+
+    def self.bench_range &block
+      return super unless block
+
+      meta = (class << self; self; end)
+      meta.send :define_method, "bench_range", &block
+    end
+
+    ##
+    # Create a benchmark that verifies that the performance is linear.
+    #
+    #   describe "my class Bench" do
+    #     bench_performance_linear "fast_algorithm", 0.9999 do |n|
+    #       @obj.fast_algorithm(n)
+    #     end
+    #   end
+
+    def self.bench_performance_linear name, threshold = 0.99, &work
+      bench name do
+        assert_performance_linear threshold, &work
+      end
+    end
+
+    ##
+    # Create a benchmark that verifies that the performance is constant.
+    #
+    #   describe "my class Bench" do
+    #     bench_performance_constant "zoom_algorithm!" do |n|
+    #       @obj.zoom_algorithm!(n)
+    #     end
+    #   end
+
+    def self.bench_performance_constant name, threshold = 0.99, &work
+      bench name do
+        assert_performance_constant threshold, &work
+      end
+    end
+
+    ##
+    # Create a benchmark that verifies that the performance is exponential.
+    #
+    #   describe "my class Bench" do
+    #     bench_performance_exponential "algorithm" do |n|
+    #       @obj.algorithm(n)
+    #     end
+    #   end
+
+    def self.bench_performance_exponential name, threshold = 0.99, &work
+      bench name do
+        assert_performance_exponential threshold, &work
+      end
+    end
+  end
+
+  Betatest::Spec.register_spec_type(/Bench(mark)?$/, Betatest::BenchSpec)
+end

data/lib/betatest/expectations.rb

@@ -0,0 +1,281 @@
+##
+# It's where you hide your "assertions".
+#
+# Please note, because of the way that expectations are implemented,
+# all expectations (eg must_equal) are dependent upon a thread local
+# variable +:current_spec+. If your specs rely on mixing threads into
+# the specs themselves, you're better off using assertions. For
+# example:
+#
+#     it "should still work in threads" do
+#       my_threaded_thingy do
+#         (1+1).must_equal 2  # bad
+#         assert_equal 2, 1+1 # good
+#       end
+#     end
+
+module Betatest::Expectations
+  ##
+  # See Betatest::Assertions#assert_empty.
+  #
+  #    collection.must_be_empty
+  #
+  # :method: must_be_empty
+
+  infect_an_assertion :assert_empty, :must_be_empty, :unary
+
+  ##
+  # See Betatest::Assertions#assert_equal
+  #
+  #    a.must_equal b
+  #
+  # :method: must_equal
+
+  infect_an_assertion :assert_equal, :must_equal
+
+  ##
+  # See Betatest::Assertions#assert_in_delta
+  #
+  #    n.must_be_close_to m [, delta]
+  #
+  # :method: must_be_close_to
+
+  infect_an_assertion :assert_in_delta, :must_be_close_to
+
+  alias :must_be_within_delta :must_be_close_to # :nodoc:
+
+  ##
+  # See Betatest::Assertions#assert_in_epsilon
+  #
+  #    n.must_be_within_epsilon m [, epsilon]
+  #
+  # :method: must_be_within_epsilon
+
+  infect_an_assertion :assert_in_epsilon, :must_be_within_epsilon
+
+  ##
+  # See Betatest::Assertions#assert_includes
+  #
+  #    collection.must_include obj
+  #
+  # :method: must_include
+
+  infect_an_assertion :assert_includes, :must_include, :reverse
+
+  ##
+  # See Betatest::Assertions#assert_instance_of
+  #
+  #    obj.must_be_instance_of klass
+  #
+  # :method: must_be_instance_of
+
+  infect_an_assertion :assert_instance_of, :must_be_instance_of
+
+  ##
+  # See Betatest::Assertions#assert_kind_of
+  #
+  #    obj.must_be_kind_of mod
+  #
+  # :method: must_be_kind_of
+
+  infect_an_assertion :assert_kind_of, :must_be_kind_of
+
+  ##
+  # See Betatest::Assertions#assert_match
+  #
+  #    a.must_match b
+  #
+  # :method: must_match
+
+  infect_an_assertion :assert_match, :must_match
+
+  ##
+  # See Betatest::Assertions#assert_nil
+  #
+  #    obj.must_be_nil
+  #
+  # :method: must_be_nil
+
+  infect_an_assertion :assert_nil, :must_be_nil, :unary
+
+  ##
+  # See Betatest::Assertions#assert_operator
+  #
+  #    n.must_be :<=, 42
+  #
+  # This can also do predicates:
+  #
+  #    str.must_be :empty?
+  #
+  # :method: must_be
+
+  infect_an_assertion :assert_operator, :must_be, :reverse
+
+  ##
+  # See Betatest::Assertions#assert_output
+  #
+  #    proc { ... }.must_output out_or_nil [, err]
+  #
+  # :method: must_output
+
+  infect_an_assertion :assert_output, :must_output
+
+  ##
+  # See Betatest::Assertions#assert_raises
+  #
+  #    proc { ... }.must_raise exception
+  #
+  # :method: must_raise
+
+  infect_an_assertion :assert_raises, :must_raise
+
+  ##
+  # See Betatest::Assertions#assert_respond_to
+  #
+  #    obj.must_respond_to msg
+  #
+  # :method: must_respond_to
+
+  infect_an_assertion :assert_respond_to, :must_respond_to, :reverse
+
+  ##
+  # See Betatest::Assertions#assert_same
+  #
+  #    a.must_be_same_as b
+  #
+  # :method: must_be_same_as
+
+  infect_an_assertion :assert_same, :must_be_same_as
+
+  ##
+  # See Betatest::Assertions#assert_silent
+  #
+  #    proc { ... }.must_be_silent
+  #
+  # :method: must_be_silent
+
+  infect_an_assertion :assert_silent, :must_be_silent
+
+  ##
+  # See Betatest::Assertions#assert_throws
+  #
+  #    proc { ... }.must_throw sym
+  #
+  # :method: must_throw
+
+  infect_an_assertion :assert_throws, :must_throw
+
+  ##
+  # See Betatest::Assertions#refute_empty
+  #
+  #    collection.wont_be_empty
+  #
+  # :method: wont_be_empty
+
+  infect_an_assertion :refute_empty, :wont_be_empty, :unary
+
+  ##
+  # See Betatest::Assertions#refute_equal
+  #
+  #    a.wont_equal b
+  #
+  # :method: wont_equal
+
+  infect_an_assertion :refute_equal, :wont_equal
+
+  ##
+  # See Betatest::Assertions#refute_in_delta
+  #
+  #    n.wont_be_close_to m [, delta]
+  #
+  # :method: wont_be_close_to
+
+  infect_an_assertion :refute_in_delta, :wont_be_close_to
+
+  alias :wont_be_within_delta :wont_be_close_to # :nodoc:
+
+  ##
+  # See Betatest::Assertions#refute_in_epsilon
+  #
+  #    n.wont_be_within_epsilon m [, epsilon]
+  #
+  # :method: wont_be_within_epsilon
+
+  infect_an_assertion :refute_in_epsilon, :wont_be_within_epsilon
+
+  ##
+  # See Betatest::Assertions#refute_includes
+  #
+  #    collection.wont_include obj
+  #
+  # :method: wont_include
+
+  infect_an_assertion :refute_includes, :wont_include, :reverse
+
+  ##
+  # See Betatest::Assertions#refute_instance_of
+  #
+  #    obj.wont_be_instance_of klass
+  #
+  # :method: wont_be_instance_of
+
+  infect_an_assertion :refute_instance_of, :wont_be_instance_of
+
+  ##
+  # See Betatest::Assertions#refute_kind_of
+  #
+  #    obj.wont_be_kind_of mod
+  #
+  # :method: wont_be_kind_of
+
+  infect_an_assertion :refute_kind_of, :wont_be_kind_of
+
+  ##
+  # See Betatest::Assertions#refute_match
+  #
+  #    a.wont_match b
+  #
+  # :method: wont_match
+
+  infect_an_assertion :refute_match, :wont_match
+
+  ##
+  # See Betatest::Assertions#refute_nil
+  #
+  #    obj.wont_be_nil
+  #
+  # :method: wont_be_nil
+
+  infect_an_assertion :refute_nil, :wont_be_nil, :unary
+
+  ##
+  # See Betatest::Assertions#refute_operator
+  #
+  #    n.wont_be :<=, 42
+  #
+  # This can also do predicates:
+  #
+  #    str.wont_be :empty?
+  #
+  # :method: wont_be
+
+  infect_an_assertion :refute_operator, :wont_be, :reverse
+
+  ##
+  # See Betatest::Assertions#refute_respond_to
+  #
+  #    obj.wont_respond_to msg
+  #
+  # :method: wont_respond_to
+
+  infect_an_assertion :refute_respond_to, :wont_respond_to, :reverse
+
+  ##
+  # See Betatest::Assertions#refute_same
+  #
+  #    a.wont_be_same_as b
+  #
+  # :method: wont_be_same_as
+
+  infect_an_assertion :refute_same, :wont_be_same_as
+end