minitest-check 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA1: 
+  metadata.gz: 055592ca4baf16a6dffc2ebdedc7527af28732f8
+  data.tar.gz: 49082898a189979ed651d9ae292fe63654fb8fe2
+SHA512: 
+  metadata.gz: c600fa8a481f693bb5191c0447159536897521b2b342a012051111c445679fcfe93238173bcce7c377f7f3601a672b8bd6036c6075619a22a088ecbc7e736a12
+  data.tar.gz: 61933fbd71e0d460db24612f8de1339f0ae6a9824205aeaa3c97ad295fbf2e5205a63a93dfd2bf4511fb481a82bf522272fecc3b6b6214ae385baee05bf252a6

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+.DS_Store
+*.swp
+*.swo

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in minitest-check.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andrew O'Brien
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Minitest::Check
+
+Writing tests is easy in Ruby—there are so many tools and TDD is ingrained in the culture. However, with all of that emphasis on testing, isn't it a shame that we spend so much time testing against the same data? All we end up proving is that our code runs well against a certain class of hand-picked inputs which may or may not resemble our real-world data.
+
+What we need is a way to express invariants over a domain: given a class of inputs, our code should produce a certain class of outputs.
+
+This library provides three things:
+
+1. A means to parameterize tests so that the same test can be run with different inputs and those inputs are clear to other developers.
+2. A means to seed the test suite with the data to run. This could be from a generator that returns a certain type of objects for unit testing, or from an external data source if you wish to validate integration data. 
+3. A means to collect data from inside your tests. In the future, you will be able to make assertions on these to test invariants over the entire suite (e.g.: author.books.length should follow a power distribution.)
+
+For now, see the examples for more documentation.
+
+## Future Directions
+
+The initial versions of this library are meant to work with the version of minitest found in Ruby 1.9.3-p327. More current versions of minitest are a good deal easier to extend. My goal is to have a stable version of this that can run against the stock minitest, with future development using a later one. When I make this switch, there will be a major version bump.
+
+As mentioned above, I'm planning on adding the ability to add assertions on collectors. Fortunately, minitest/benchmark has already done most of the statistics work for me there.
+
+Lastly, I'm looking into modifying the Minitest runner to run using an Enumerator of callables, rather than it's current instance method based implementation. This would allow the test runner to operate continuously (with the right data generator).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'minitest-check'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install minitest-check
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/examples/simple_spec.rb

@@ -0,0 +1,40 @@
+$:.unshift("lib")
+require "minitest/autorun"
+require "minitest-check"
+
+class MyClass
+  def add(x, y)
+    if x && y
+      x + y
+    else
+      nil
+    end
+  end
+end
+
+#SimpleSpec = describe "MyClass" do
+class SimpleSpec < Minitest::Spec
+  check "add" do |a, b|
+    collect(:input, [a, b])
+    #puts "checking with #{a}, #{b}"
+    assert_equal(collect(:output, MyClass.new.add(a, b)), a + b)
+  end
+
+  check "maybe add" do |b, c|
+    actual = MyClass.new.add(c, b)
+    if c
+      assert_equal(actual, c + b)
+    else
+      assert_equal(actual, nil)
+    end
+  end
+end
+
+# Some general tests
+SimpleSpec.seed(100) do |i|
+  {a: rand(i), b: rand(i * 2), c: rand(i * 3) }
+end
+
+# Make sure we test with c as nil at least once
+SimpleSpec.seed_value(a: 1, b: 2, c: nil)
+

data/examples/simple_test.rb

@@ -0,0 +1,38 @@
+$:.unshift("lib")
+require "minitest/autorun"
+require "minitest-check"
+
+class MyClass
+  def add(x, y)
+    if x && y
+      x + y
+    else
+      nil
+    end
+  end
+end
+
+class SimpleTest < MiniTest::Unit::TestCase
+  def check_add(a, b)
+    collect(:input, [a, b])
+    #puts "checking with #{a}, #{b}"
+    assert_equal(collect(:output, MyClass.new.add(a, b)), a + b)
+  end
+
+  def check_maybe_add(b, c)
+    actual = MyClass.new.add(c, b)
+    if c
+      assert_equal(actual, c + b)
+    else
+      assert_equal(actual, nil)
+    end
+  end
+end
+
+# Some general tests
+SimpleTest.seed(100) do |i|
+  {a: rand(i), b: rand(i * 2), c: rand(i * 3) }
+end
+
+# Make sure we test with c as nil at least once
+SimpleTest.seed_value(a: 1, b: 2, c: nil)

data/lib/minitest-check/version.rb

@@ -0,0 +1,5 @@
+module Minitest
+  module Check
+    VERSION = "0.0.1"
+  end
+end

data/lib/minitest-check.rb

@@ -0,0 +1,199 @@
+require "minitest-check/version"
+require "delegate"
+require "ostruct"
+require "minitest/unit"
+require "observer"
+require "set"
+
+module MiniTest
+  class Unit
+    attr_reader :collector
+    def run_checks 
+      @collector = Check::Collector.new
+      _run_anything :check
+      @collector.report
+    end
+
+    class TestCase
+      def self.check_suites
+        TestCase.test_suites.reject {
+          |s| s.check_methods.empty?
+        }.map {|s|
+          s.generate_suites
+        }.flatten
+      end
+
+      def self.check_methods # :nodoc:
+        public_instance_methods(true).grep(/^check_/).map { |m| m.to_s }.sort
+      end
+
+      class << self
+        def contexts
+          @contexts or superclass.respond_to?(:contexts) ? superclass.contexts : []
+        end
+
+        def check_with(generator)
+          @contexts ||= []
+          # It would be nice if Minitest lazily iterated through its tests, calling one-by-one.
+          # Then we could feed new tests into the generator as the system run or depending on
+          # external data sources. It's a departure from the unit testing that Minitest is built for,
+          # but it is a use-case I'm trying to cover here.
+          @contexts += generator.to_a
+        end
+
+        # Convenience method to run a block a certain number of times
+        def seed(num = 1, &blk)
+          check_with(Enumerator.new {|contexts|
+            num.times {|i| contexts << blk.call(i)}
+          })
+        end
+
+        def seed_value(hash_or_object)
+          check_with([hash_or_object])
+        end
+
+        def generate_suites
+          contexts.map {|c| Check::SuiteWrapper.new(self, c) }
+        end
+      end
+    end
+  end
+
+
+  module Check
+    class SuiteWrapper < SimpleDelegator
+      def initialize(suite, context)
+        @context = context.kind_of?(Hash) ? OpenStruct.new(context) : context
+        super(suite)
+
+        @test_wrapper = Class.new(suite) do
+          include Observable
+          attr_reader :context
+          def initialize(name, _context)
+            # Getting a little gnarly here...
+            method = self.class.superclass.instance_method(name)
+            params = method.parameters.map {|p| p[1]}
+            @context = Hash[params.map {|p| [p, _context.send(p)] }]
+
+            super(name)
+          end
+          
+          def run(runner)
+            add_observer(runner.collector)
+            super(runner).tap do
+              runner.report[-1] += " Context: #{@context.inspect}" unless @passed
+            end
+          end
+
+          private
+          def collect(stat_name, stat_value)
+            changed
+            notify_observers("#{self.class.superclass.name}##{self.__name__}:#{stat_name}", stat_value)#, @context) Waiting until I know how we want to display contexts
+            stat_value
+          end
+        end
+        check_methods.each do |name|
+          # TODO: fewer horrible metaprogramming hacks
+          @test_wrapper.send(:define_method, name) do
+            super(*@context.values)
+          end
+        end
+      end
+      
+      def new(name)
+        @test_wrapper.new(name, @context)
+      end
+
+      def check_suite_header(suite)
+        puts "Checking with context: #{@context.inspect}"
+      end
+    end
+
+    class Collector
+      Record = Struct.new(:count, :contexts) do
+        def initialize
+          super(0, Set.new)
+        end
+      end
+
+      def initialize
+        # TODO: better storage mechanism
+        # Creates a hash like:
+        #
+        #  {
+        #   "stat_foo" => {
+        #     "value_1" => Record.new(count, Set of contexts producing this value)
+        #   }
+        # }
+        @store = Hash.new {|s, n|
+          s[n] = Hash.new {|n, v|
+            n[v] = Record.new
+          }
+        }
+      end
+
+      def update(name, value, context = nil)
+        @store[name][value].count += 1
+        @store[name][value].contexts << context
+      end
+
+      def report(io = STDOUT)
+        return if @store.length == 0
+
+        io.puts
+        io.puts "Collected data for #{@store.length} probes during checks:"
+        @store.each do |name, data|
+          io.puts
+          io.puts name
+          io.puts
+          draw_graph(
+            data.map {|(value, record)| [value.to_s, record.count] },
+            io
+          )
+        end
+      end
+
+      private
+      def draw_graph(pairs, io, max_width = 80)
+        if pairs
+          data_groups = pairs.group_by {|d| d[1] == 1 ? :singles : :multis }
+          data = data_groups[:multis].to_a.sort {|a,b| b[1] <=> a[1]}
+          largest_value_width = data.map {|d| d[0].length}.max
+          largest_num = data[0][1]
+          scale = if largest_num > 0
+            (max_width - largest_value_width - 3).to_f / largest_num.to_f
+          else
+            0
+          end
+
+          data.each do |(value, num)|
+            num_string = num.to_s
+            num_length = num_string.length
+            bar_length = (num.to_f * scale).to_i
+            fill_length = bar_length - num_length
+            if fill_length > 0
+              io.puts "#{value.to_s.ljust(largest_value_width)} | #{num_string}#{'#' * fill_length}"
+            else
+              io.puts "#{value.to_s.ljust(largest_value_width)} | #{'#' * bar_length} (#{num_string})"
+            end
+          end
+
+          if singles = data_groups[:singles]
+            io.puts
+            io.puts "#{singles.length} singles: #{singles.map {|d| d[0].inspect}.join(", ")}"
+          end
+        end
+      end
+    end
+  end
+
+  class Spec
+    class << self
+      def check name, &block
+        define_method "check_#{name.gsub(/\W+/, '_')}", &block
+      end
+      # Make the Haskell people happy. :)
+      alias_method :prop, :check
+    end
+  end
+end

data/minitest-check.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'minitest-check/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "minitest-check"
+  gem.version       = Minitest::Check::VERSION
+  gem.authors       = ["Andrew O'Brien"]
+  gem.email         = ["obrien.andrew@gmail.com"]
+  gem.description   = %q{Generative testing for Minitest.}
+  gem.summary       = %q{Testing with fixed data causes false negatives. Testing with random values leads to spaghetti. Run tests across entire domains with this library.}
+  gem.homepage      = "https://github.com/AndrewO/minitest-check"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.license = 'MIT'
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: minitest-check
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Andrew O'Brien
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-06-10 00:00:00 Z
+dependencies: []
+
+description: Generative testing for Minitest.
+email: 
+- obrien.andrew@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/simple_spec.rb
+- examples/simple_test.rb
+- lib/minitest-check.rb
+- lib/minitest-check/version.rb
+- minitest-check.gemspec
+homepage: https://github.com/AndrewO/minitest-check
+licenses: 
+- MIT
+metadata: {}
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - &id001 
+    - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - *id001
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Testing with fixed data causes false negatives. Testing with random values leads to spaghetti. Run tests across entire domains with this library.
+test_files: []
+
+has_rdoc: