greybox 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6403c4bd8f174968087fb1add29ea027ffb2e502
+  data.tar.gz: eaf7e713f00cc65d609a5c80be3f94479da6d42a
+SHA512:
+  metadata.gz: c8ca2cc43f5b623785081aff0918db238fd10e052fe2ac23363dd97539d509afec441fc84b7ddf2c55dd04e7aaf596da0ec21c0c007f60944e0d65f2c6f103c0
+  data.tar.gz: 39f8f4521871ee01435b2a1a7829f8037352aaf42d56414e5f6755bcf63942f3e985d3620202dab9c3a06224680bcf8a7fdbda4fe78c031216d2db08407bf739

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "1.9.3"
+  - "2.0.0"
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Justin Jaffray
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+[![Build Status](https://travis-ci.org/justinj/greybox.png?branch=master)](https://travis-ci.org/justinj/greybox)
+[![Code Climate](https://codeclimate.com/github/justinj/greybox.png)](https://codeclimate.com/github/justinj/greybox)
+
+# Greybox
+
+Test a CLI program against a black box.
+Intended for school projects.
+
+Often when working on a school project, the class will provide a black box to test your programs against.
+Greybox provides a simple testing harness to easily add new test cases whose expected output is automatically generated by this black box.
+Since these might be slow, or even over an ssh connection, it will store the results so it only has to actually run the black box the first time.
+
+## Installation
+
+Only Ruby 1.9.3 and 2.0.0 are supported.
+
+    $ gem install greybox
+
+## Usage
+
+In your project directory, create a file called `Greyfile`.
+A `Greyfile` might look like this:
+
+```ruby
+Greybox.setup do |c|
+  c.test_command = "./my_test_command < %"
+  c.blackbox_command = "./blackbox_command < %"
+  c.input = "test/*.input"
+end
+```
+
+This defines a testing setup in which the command
+    
+    $ ./blackbox_command < INPUT_FILE.input
+
+is used as the reference for correctness.
+`%` denotes where the input file will be placed.
+
+    $ ./my_test_command < INPUT_FILE.input
+
+is verified to match the blackbox output every time
+
+    $ greybox
+
+is run.
+
+The expected results will be stored in `INPUT_FILE.output`.
+This is the default behaviour but can be changed.
+
+As a more complete example, here is a Greyfile I used for a school project:
+
+```ruby
+Greybox.setup do |c|
+  c.test_command = "racket wlppscan.ss < % 2>&1"
+  c.blackbox_command = "ssh school 'java cs241.WLPPScan' < % 2>&1"
+  c.input = "test/*.input"
+
+  # if the expected ERRORs, all we care about is that
+  # the actual ERRORs as well.
+  c.comparison = ->(actual, expected) do
+    if expected =~ /ERROR/
+      actual =~ /ERROR/
+    else
+      expected == actual
+    end
+  end
+end
+```
+
+The `test_command` specifies how to run _my_ version of the program, a racket program.
+`blackbox_command` specifies how to generate the expected output for a specific test case.
+
+`comparison` specifies how the output of `test_command` should be compared to the black box output.
+In this case, if part of the output contains `ERROR`, all we care about is that the actual output contains `ERROR` as well.
+Otherwise, they must be equal.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = ["spec/spec_helper"] + FileList['spec/**/*_spec.rb']
+end
+
+task :gem do
+  sh "yes | gem uninstall greybox; true"
+  sh "gem build greybox.gemspec"
+  sh "gem install *.gem"
+end

data/bin/greybox

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require "greybox"
+
+begin
+  load Dir.pwd + "/Greyfile"
+rescue LoadError
+  puts "Could not load Greyfile"
+end

data/example/Greyfile

@@ -0,0 +1,5 @@
+Greybox.setup do |c|
+  c.input "test/*.input" 
+  c.blackbox_command "cat < %"
+  c.test_command "cat < %"
+end

data/example/test/file1.input

@@ -0,0 +1 @@
+Hello

data/example/test/file1.output

@@ -0,0 +1 @@
+Hello

data/greybox.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'greybox/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "greybox"
+  spec.version       = Greybox::VERSION
+  spec.authors       = ["Justin Jaffray"]
+  spec.email         = ["justin.jaffray@gmail.com"]
+  spec.description   = %q{Test against a black box.}
+  spec.summary       = %q{Easily test school assignments or other projects where you have a reference}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "diffy"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "fakefs"
+  spec.add_development_dependency "mocha"
+end

data/lib/greybox/configurable.rb

@@ -0,0 +1,66 @@
+module Greybox
+  module Configurable
+    attr_accessor :properties
+
+    module ClassMethods
+      attr_accessor :defaults
+      attr_accessor :required_properties
+      def def_property(name, args = {})
+        setter = "#{name}=".to_sym
+        getter = name
+        (@defaults ||= {})[name] = args[:default] if args.has_key? :default
+        (@required_properties ||= Set.new) << name if args[:required]
+
+        # Stolen from RubyTapas episodes 27 and 28
+        mod = if const_defined?(:Properties, false)
+                const_get(:Properties)
+              else
+                const_set(:Properties, new_blank_properties_module)
+              end
+
+        mod.module_eval %Q{
+          def #{setter}(value)
+            @properties ||= {}
+            @properties[:#{name}] = value
+          end
+
+          def #{getter}
+            get_prop(:#{name})
+          end
+        }
+
+        include mod
+      end
+
+      private
+      def new_blank_properties_module
+        Module.new do
+          class << self
+            def to_s
+              "Properties(#{properties.join(", ")})"
+            end
+            alias_method :inspect, :to_s
+
+            def properties
+              instance_methods(false).reject { |m| m.to_s.end_with? "=" }
+            end
+          end
+        end
+      end
+    end
+
+    def get_prop(prop)
+      (@properties || {}).fetch(prop, self.class.defaults[prop]) 
+    end
+
+    def verify
+      self.class.required_properties.each do |prop|
+        raise "Property #{prop} is required." unless get_prop(prop)
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+  end
+end

data/lib/greybox/configuration.rb

@@ -0,0 +1,12 @@
+module Greybox
+  class Configuration
+    include Configurable
+
+    def_property :input, required: true
+    def_property :test_command, required: true
+    def_property :expected, default: ->(input) { input.gsub(/\.input$/, ".output") }
+    def_property :comparison, default: ->(actual, expected) { actual == expected }
+    def_property :blackbox_command, required: true
+
+  end
+end

data/lib/greybox/runner.rb

@@ -0,0 +1,48 @@
+module Greybox
+  class Runner
+    attr_reader :failures
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+      @failures = []
+    end
+    def files
+      Dir.glob(config.input)
+    end
+
+    def expected_filename(input_filename)
+      expected_filename = config.expected.call(input_filename)
+      raise "Output filename for #{input_filename} was same as input!" if expected_filename == input_filename
+      expected_filename
+    end
+
+    def expected_for(filename)
+      file = expected_filename(filename)
+      File.open(file, 'w') do |f|
+        f.write run_command(config.blackbox_command, filename)
+      end
+      File.read(file)
+    end
+
+    def run
+      files.each do |input_file|
+        expected = expected_for(input_file)
+        actual = run_command(config.test_command, input_file)
+        report_failure(input_file, expected, actual) unless config.comparison.call(actual, expected)
+      end
+    end
+
+    def run_command(command, file)
+      `#{insert_filename(command, file)}`
+    end
+
+    def insert_filename(command, filename)
+      command.gsub("%", filename)
+    end
+
+    def report_failure(filename, expected, actual)
+      @failures << { filename: filename, expected: expected, actual: actual }
+    end
+  end
+end

data/lib/greybox/version.rb

@@ -0,0 +1,3 @@
+module Greybox
+  VERSION = "0.0.1"
+end

data/lib/greybox.rb

@@ -0,0 +1,38 @@
+require "set"
+require "diffy"
+require "greybox/version"
+require "greybox/configurable"
+require "greybox/runner"
+require "greybox/configuration"
+
+module Greybox
+  class << self
+    attr_reader :failures
+    attr_reader :configuration
+    attr_reader :runner
+    def setup(&blk)
+      config(&blk)
+      run
+      check
+    end
+
+    def run
+      @runner = Runner.new(configuration)
+      runner.run
+    end
+
+    def check
+      runner.failures.each do |failure|
+        puts "FAILED:"
+        puts "filename: #{failure[:filename]}"
+        puts Diffy::Diff.new(failure[:expected], failure[:actual])
+      end
+    end
+
+    def config
+      @configuration = Configuration.new
+      yield configuration
+      configuration.verify
+    end
+  end
+end

data/spec/greybox_spec.rb

@@ -0,0 +1,12 @@
+module Greybox
+  describe Greybox do
+    describe "errors" do
+      it "complains if a missing property is gone before being run" do
+        -> do
+          Greybox.config do |c|
+          end
+        end.must_raise RuntimeError
+      end
+    end
+  end
+end

data/spec/runner_spec.rb

@@ -0,0 +1,108 @@
+module Greybox
+  describe Runner do
+    before do
+      FakeFS.activate!
+
+      FileUtils.touch "inputfile.input"
+    end
+
+    after do
+      FakeFS.activate!
+      Dir.glob("*").each { |f| FileUtils.rm f }
+      FakeFS.deactivate!
+    end
+
+    # it should be noted these are just the defaults
+    # for these tests to reduce duplication,
+    # the real-world defaults are different
+    def defaults
+      {
+        input: "*.input",
+        expected: ->(input) { "output_file" },
+        blackbox_command: "echo BLACKBOX_COMMAND",
+        comparison: ->(actual, expected) { actual == expected },
+      }
+    end
+
+    attr_reader :runner
+
+    def with_config(config = {})
+      @runner = Runner.new(stub(defaults.merge(config)))
+    end
+
+    describe "finding files" do
+      it "uses the glob to find files" do
+        with_config
+        runner.files.must_equal ["inputfile.input"]
+      end
+
+      it "uses the 'expected' value to determine the expected name" do
+        with_config(expected: ->(input) {
+          input.gsub(/\.input$/, ".output")
+        })
+        runner.expected_filename("hello.input").must_equal "hello.output"
+      end
+
+      it "complains if the output is the same as the input" do
+        with_config(expected: ->(input){ input })
+        ->() { runner.expected_filename("input.input") }.must_raise RuntimeError
+      end
+    end
+
+    describe "running commands" do
+      it "uses the provided test command" do
+        with_config(test_command: "echo TEST_COMMAND")
+        runner.expects(:`).with("echo TEST_COMMAND")
+        runner.expects(:`).with("echo BLACKBOX_COMMAND")
+        runner.run
+      end
+
+      it "inserts filenames for %'s" do
+        with_config(test_command: "echo hi %")
+        runner.expects(:`).with "echo hi inputfile.input"
+        runner.expects(:`).with "echo BLACKBOX_COMMAND"
+        runner.run
+      end
+
+      it "uses the provided blackbox command to find the expected value for cases that don't have one yet" do
+        with_config(test_command: "echo hi",
+                    blackbox_command: "echo output")
+        runner.expects(:`).with "echo hi"
+        runner.expects(:`).with "echo output"
+        runner.run
+      end
+
+      it "inserts output filenames for %'s in the blackbox command" do
+        with_config(test_command: "echo hi",
+                    blackbox_command: "echo output < %")
+        runner.expects(:`).with "echo hi"
+        runner.expects(:`).with "echo output < inputfile.input"
+        runner.run
+      end
+    end
+
+    describe "asserting output" do
+      it "compares for equality by default" do
+        with_config(test_command: "echo hi",
+                    blackbox_command: "echo hi")
+        runner.run
+        runner.failures.must_be_empty
+      end
+
+      it "has a failure if a test fails" do
+        with_config(test_command: "echo hi",
+                    blackbox_command: "echo hello")
+        runner.run
+        runner.failures.wont_be_empty
+      end
+
+      it "compares by the provided comparison" do
+        with_config(comparison: ->(actual, expected) { actual == "hi\n" },
+                    test_command: "echo hi",
+                    blackbox_command: "echo hello")
+        runner.run
+        runner.failures.must_be_empty
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require "minitest/autorun"
+require "mocha/setup"
+require "fakefs"
+require "fileutils"
+require_relative "../lib/greybox"

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: greybox
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Justin Jaffray
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: diffy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Test against a black box.
+email:
+- justin.jaffray@gmail.com
+executables:
+- greybox
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/greybox
+- example/Greyfile
+- example/test/file1.input
+- example/test/file1.output
+- greybox.gemspec
+- lib/greybox.rb
+- lib/greybox/configurable.rb
+- lib/greybox/configuration.rb
+- lib/greybox/runner.rb
+- lib/greybox/version.rb
+- spec/greybox_spec.rb
+- spec/runner_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Easily test school assignments or other projects where you have a reference
+test_files:
+- spec/greybox_spec.rb
+- spec/runner_spec.rb
+- spec/spec_helper.rb