rcomp 0.2.0 → 0.2.1

data/{CHANGELOG.rdoc → CHANGELOG.md}

@@ -1,4 +1,11 @@
-== 0.2.0, released 2012-11-25
+# 0.2.1, released 2012-11-28
+
+* Overhaul default output
+  * Add output highlighting
+  * More descriptive output for test failure
+  * Explanations: why tests were skipped by test or generate
+
+# 0.2.0, released 2012-11-25
 
 * Add "ignore" configuration option
 * Add "timeout" configuration option
@@ -6,11 +13,11 @@
 * Remove "set-directory"
 * Upgrade "init" to handle command setting when neccessary
 
-== 0.1.1, released 2012-11-20
+# 0.1.1, released 2012-11-20
 
 * Ignore dotfiles in tests directory
 * Clean up generate output, remove remnant debugging output
 
-== 0.1.0, released 2012-11-20
+# 0.1.0, released 2012-11-20
 
 * Initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rcomp (0.2.0)
+    rcomp (0.2.1)
       childprocess (~> 0.3.6)
       thor (~> 0.16.0)
 

data/README.md

@@ -54,7 +54,7 @@ Managed by RComp. Stores the results of your most recent test suite run.
 
 ---
 
-A simple RComp suite might look something like:
+A simple RComp suite might look like:
 
 ```
 .

data/features/generate.feature

@@ -62,7 +62,9 @@ Feature: Generate
 
       """
     When I run `rcomp generate`
-    Then the output should contain "1 file (1 skipped)"
+    Then the output should contain "Skipped 1 file"
+    And the output should contain "Run rcomp generate -O"
+    And the output should contain "1 file (1 skipped)"
     And the file "rcomp/expected/test1.out" should contain "DEF"
     And the exit status should be 0
 
@@ -324,7 +326,7 @@ Feature: Generate
       """
     When I run `rcomp generate` for up to 6 seconds
     Then the output should contain "1 file (1 failed)"
-    And the output should contain "timeout : /test1"
+    And the output should contain "timeout : test1.test"
     And the exit status should be 1
 
   @loop-conf
@@ -341,5 +343,5 @@ Feature: Generate
       """
     When I run `rcomp generate` for up to 2 seconds
     Then the output should contain "1 file (1 failed)"
-    And the output should contain "timeout : /test1"
+    And the output should contain "timeout : test1.test"
     And the exit status should be 1

data/features/test.feature

@@ -61,7 +61,9 @@ Feature: Test
 
       """
     When I run `rcomp test`
-    Then the output should contain "1 test (1 skipped)"
+    Then the output should contain "Skipped 1 test"
+    And the output should contain "Run rcomp generate"
+    And the output should contain "1 test (1 skipped)"
     And the exit status should be 0
 
   @basic-conf
@@ -77,7 +79,8 @@ Feature: Test
 
       """
     When I run `rcomp test`
-    Then the output should contain "1 test (1 failed)"
+    Then the output should contain "failed : test1.test (out)"
+    And the output should contain "1 test (1 failed)"
     And the exit status should be 1
 
   # stdout multiple tests
@@ -118,7 +121,8 @@ Feature: Test
 
       """
     When I run `rcomp test`
-    Then the output should contain "1 test (1 failed)"
+    Then the output should contain "failed : test1.test (err)"
+    And the output should contain "1 test (1 failed)"
     And the exit status should be 1
 
   # stderr multiple tests
@@ -129,6 +133,29 @@ Feature: Test
     Then the output should contain "3 tests (1 failed, 1 skipped, 1 passed)"
     And the exit status should be 1
 
+  # stderr + stdout tests
+  @basic-conf
+  Scenario: Single test, failing both err and out
+    Given a file named "rcomp/tests/test1.test" with:
+      """
+      ABC
+
+      """
+    And a file named "rcomp/expected/test1.out" with:
+      """
+      XYZ
+
+      """
+    And a file named "rcomp/expected/test1.err" with:
+      """
+      XYZ
+
+      """
+    When I run `rcomp test`
+    Then the output should contain "failed : test1.test (out, err)"
+    And the output should contain "1 test (1 failed)"
+    And the exit status should be 1
+
   # alias
   @basic-conf
   @load-assorted-tests
@@ -274,7 +301,7 @@ Feature: Test
       """
     When I run `rcomp test` for up to 6 seconds
     Then the output should contain "1 test (1 failed)"
-    And the output should contain "timeout : /test1"
+    And the output should contain "timeout : test1.test"
     And the exit status should be 1
 
   @loop-conf
@@ -296,5 +323,5 @@ Feature: Test
       """
     When I run `rcomp test` for up to 2 seconds
     Then the output should contain "1 test (1 failed)"
-    And the output should contain "timeout : /test1"
+    And the output should contain "timeout : test1.test"
     And the exit status should be 1

data/lib/rcomp.rb

@@ -7,5 +7,5 @@ require 'rcomp/runner'
 require 'rcomp/suite'
 require 'rcomp/test'
 require 'rcomp/version'
-require 'rcomp/initializer'
+require 'rcomp/env'
 require 'rcomp/process'

data/lib/rcomp/actions.rb

@@ -33,9 +33,7 @@ module RComp
     end
 
     def touch(path)
-      unless File.exist? path
-        FileUtils.touch path
-      end
+      FileUtils.touch path
     end
   end
 end

data/lib/rcomp/cli.rb

@@ -4,9 +4,6 @@ module RComp
   class CLI < Thor
 
     include Thor::Actions
-    include RComp::Runner
-    include RComp::Suite
-    include RComp::Initializer
 
     def initialize(args=[], options={}, config={})
       super
@@ -20,13 +17,13 @@ module RComp
     
     desc "init", "Setup rcomp test directory"
     def init
-      guard_initialized
+      Env.guard_initialized
 
-      unless command_exists?
+      unless Env.command_exists?
         @conf.set_command(ask("Enter the command you want to test:"))
       end
 
-      initialize_directories
+      Env.initialize_directories
       puts "RComp successfully initialized"
     end
 
@@ -44,11 +41,11 @@ module RComp
       :type => :string,
       :desc => "Only test files that match pattern"
     def test
-      guard_uninitialized
+      Env.guard_uninitialized
       if @options[:grep]
-        run_suite(load_suite(@options[:grep]), :test)
+        Runner.run(Suite.load(@options[:grep]), :test)
       else
-        run_suite(load_suite, :test)
+        Runner.run(Suite.load, :test)
       end
     end
 
@@ -65,7 +62,7 @@ module RComp
       :aliases => "-O",
       :desc => "Overwrite expected output file for test if present"
     def generate
-      guard_uninitialized
+      Env.guard_uninitialized
 
       # Display confirmation dialouge when -O is passed without filter
       if !@options[:grep] && options.overwrite
@@ -76,9 +73,9 @@ module RComp
       end
 
       if @options[:grep]
-        run_suite(load_suite(@options[:grep]), :generate, @options)
+        Runner.run(Suite.load(@options[:grep]), :generate, @options)
       else
-        run_suite(load_suite, :generate, @options)
+        Runner.run(Suite.load, :generate, @options)
       end
     end
 

data/lib/rcomp/conf.rb

@@ -34,8 +34,7 @@ module RComp
 
       # Load configuration values into attributes
       @command = @conf['command']
-      @ignore = @conf['ignore']
-      @ignore ||= []
+      @ignore = @conf['ignore'] || []
       @timeout = @conf['timeout']
       @root = @conf['directory']
       @test_root = @root + '/tests'
@@ -58,7 +57,7 @@ module RComp
     #
     # Returns nothing
     def write_conf_file
-      touch CONF_PATH unless File.exists?(CONF_PATH)
+      touch CONF_PATH
       conf_file = File.open(CONF_PATH, 'w')
       conf_file.puts YAML.dump @custom
     end

data/lib/rcomp/{initializer.rb → env.rb}

@@ -1,7 +1,8 @@
 module RComp
-  module Initializer
+  module Env
 
-    include RComp::Actions
+    extend self
+    extend Actions
 
     @@conf = Conf.instance
 
@@ -33,9 +34,7 @@ module RComp
       mkdir @@conf.expected_root
       mkdir @@conf.result_root
     end
-
-    private
-
+    
     # Checks for the existance of a command to test with
     #
     # Returns a boolean
@@ -43,6 +42,8 @@ module RComp
       @@conf.command
     end
 
+    private
+    
     # Checks for the existance of RComp's root directory
     #
     # Returns a boolean
@@ -57,7 +58,7 @@ module RComp
     def root_subdirs_exist?
       File.exists?(@@conf.test_root) && 
       File.exists?(@@conf.result_root) && 
-      File.exists?(@conf.expected_root)
+      File.exists?(@@conf.expected_root)
     end
 
     # Checks to see if RComp is fully initialized

data/lib/rcomp/path.rb

@@ -1,23 +1,8 @@
 module RComp
   module Path
+
     def rel_path(test_path)
       test_path.gsub(Conf.instance.test_root, '')
     end
-
-    def result_path(test_path, type)
-      cmpnts = []
-      cmpnts << Conf.instance.result_root
-      cmpnts << rel_path(File.dirname(test_path))
-      cmpnts << File.basename(test_path, ".*") + (type == :out ? '.out' : '.err')
-      File.join(cmpnts)
-    end
-
-    def expected_path(test_path, type)
-      cmpnts = []
-      cmpnts << Conf.instance.expected_root
-      cmpnts << rel_path(File.dirname(test_path))
-      cmpnts << File.basename(test_path, ".*") + (type == :out ? '.out' : '.err')
-      File.join(cmpnts)
-    end
   end
 end

data/lib/rcomp/process.rb

@@ -22,15 +22,17 @@ module RComp
     #
     # Returns nothing
     def run
-      begin @process.start
+      begin 
+        @process.start
       rescue ChildProcess::LaunchError => e
         raise StandardError.new(e.message)
       end
 
-      begin @process.poll_for_exit(@timeout)
+      begin 
+        @process.poll_for_exit(@timeout)
       rescue ChildProcess::TimeoutError
         @timedout = true
-        @process.stop
+        @process.stop(@timeout)
       end
     end
 

data/lib/rcomp/reporter.rb

@@ -1,8 +1,15 @@
+require 'thor'
+
 module RComp
   class Reporter
 
+    include Thor::Shell
     include RComp::Helper
 
+    PADDING = 4
+    GEN_JUSTIFY = 12 + PADDING
+    TEST_JUSTIFY = 10 + PADDING
+
     # Initialize a new Reporter
     #
     # type - The type (Symbol) of the suite
@@ -15,64 +22,162 @@ module RComp
       @failed = 0
     end
 
+    # Main interface for reporting
+    # Reports the result of a single test or generation
+    #
+    # test - A test object that has been run
+    #
+    # Returns nothing
     def report(test)
       case test.result
-      # success
       when :success
         if @type == :test
-          puts "\tpassed : #{test.relative_path}"
+          print_test_success(test)
         else
-          puts "\tgenerated : #{test.relative_path}"
+          print_generate_success(test)
         end
         @success += 1
 
-      # skipped
       when :skipped
-        puts "\tskipped : #{test.relative_path}"
+        if @type == :test
+          print_test_skipped(test)
+        else
+          print_generate_skipped(test)
+        end
         @skipped += 1
 
-      # failed
+      # Generate can't fail directly
       when :failed
-        puts "\tfailed : #{test.relative_path}"
+        print_test_failed(test)
         @failed += 1
 
-      # timedout
       when :timedout
-        puts "\ttimeout : #{test.relative_path}"
+        if @type == :test
+          print_test_timeout(test)
+        else
+          print_generate_timeout(test)
+        end
         @failed += 1
       end
     end
 
-    def summary
-      case @type
-      when :test
-        print_test_summary
-      when :generate
-        print_generate_summary
+    # Prints header for test suite report
+    #
+    # Returns nothing
+    def header
+      # The suite is a test suite
+      if @type == :test
+        puts "RComp: running test suite..."
+
+      # The suite is a generation suite
+      else
+        puts "RComp: generating expected output..."
       end
+      puts
+    end
+
+    # Prints summary of test suite report
+    #
+    # Returns nothing
+    def summary
+      print_summary
       exit 1 if @failed > 0
     end
 
     private
 
-    def print_test_summary
-      desc = []
-      summary = "#{plural((@failed + @skipped + @success), 'test')} ("
-      desc << "#{@failed} failed" unless @failed == 0
-      desc << "#{@skipped} skipped" unless @skipped == 0
-      desc << "#{@success} passed" unless @success == 0
-      summary += desc.join(", ") + ")"
-      puts summary
+    def print_test_success(test)
+      say "passed : ".rjust(TEST_JUSTIFY) + test.formatted_path, :green
+    end
+
+    def print_generate_success(test)
+      say "generated : ".rjust(GEN_JUSTIFY) + test.formatted_path, :green
+    end
+
+    def print_test_skipped(test)
+      say "skipped : ".rjust(TEST_JUSTIFY) + test.formatted_path, :yellow
+    end
+
+    def print_generate_skipped(test)
+      say "skipped : ".rjust(GEN_JUSTIFY) + test.formatted_path, :yellow
+    end
+
+    def print_test_failed(test)
+      msg = "failed : ".rjust(TEST_JUSTIFY) + test.formatted_path
+
+      # both out and err present
+      if test.expected_out_exists? && test.expected_err_exists?
+        # failed both out and err cmp
+        if !test.out_result && !test.err_result
+          msg += " (out, err)"
+
+        # failed out cmp
+        elsif !test.out_result
+          msg += " (out)"
+
+        # failed err cmp
+        else
+          msg += " (err)"
+        end
+
+      # out present
+      elsif test.expected_out_exists?
+        msg += " (out)"
+
+      # err present
+      else
+        msg += " (err)"
+      end
+
+      say msg, :red
+    end
+
+    def print_test_timeout(test)
+      say "timeout : ".rjust(TEST_JUSTIFY) + test.formatted_path, :red
     end
 
-    def print_generate_summary
+    def print_generate_timeout(test)
+      say "timeout : ".rjust(GEN_JUSTIFY) + test.formatted_path, :red
+    end
+
+    def print_summary
+      # print skipped explanation if anything was skipped
+      if @skipped > 0
+        if @type == :test
+          print_skipped_test_explanation
+        else
+          print_skipped_generate_explanation
+        end
+      end
+
+      # construct and print output summary
       desc = []
-      summary = "#{plural((@failed + @skipped + @success), 'file')} ("
-      desc << "#{@failed} failed" unless @failed == 0
-      desc << "#{@skipped} skipped" unless @skipped == 0
-      desc << "#{@success} generated" unless @success == 0
+      tests = @failed + @skipped + @success
+
+      summary = "#{plural(tests, @type == :test ? 'test' : 'file')} ("
+
+      desc << set_color("#{@failed} failed", :red) unless @failed == 0
+      desc << set_color("#{@skipped} skipped", :yellow) unless @skipped == 0
+      unless @success == 0
+        if @type == :test
+          desc << set_color("#{@success} passed", :green)
+        else
+          desc << set_color("#{@success} generated", :green)
+        end
+      end
+
       summary += desc.join(", ") + ")"
-      puts summary
+      puts "\n" + summary
+    end
+
+    def print_skipped_test_explanation
+      say "\nSkipped #{plural(@skipped, 'test')} due to missing expected output"
+      say "Run rcomp generate or manually create expected output"
+    end
+
+    def print_skipped_generate_explanation
+      say "\nSkipped #{plural(@skipped, 'file')} due to existing expected output"
+      say "Run rcomp generate -O to overwrite"
     end
   end
 end

data/lib/rcomp/runner.rb

@@ -3,7 +3,8 @@ require 'fileutils'
 module RComp
   module Runner
 
-    include RComp::Actions
+    extend self
+    extend Actions
     
     # Run a suite of tests
     #
@@ -12,20 +13,22 @@ module RComp
     # options - A Hash of runner options
     #
     # Returns nothing
-    def run_suite(suite, type, options={})
+    def run(suite, type, options={})
       @conf = Conf.instance
       reporter = Reporter.new(type)
 
+      reporter.header
+
       suite.each do |test|
         case type
         when :test
-          run(test) if expected_exists?(test)
+          run_test(test) if expected_exists?(test)
 
         when :generate
           if expected_exists?(test)
-            run(test, true) if options[:overwrite]
+            run_test(test, true) if options[:overwrite]
           else
-            run(test, true)
+            run_test(test, true)
           end
         end
 
@@ -44,8 +47,7 @@ module RComp
     #
     # Returns a boolean
     def expected_exists?(test)
-      File.exists?(test.expected_out_path) ||
-        File.exists?(test.expected_err_path)
+      test.expected_out_exists? || test.expected_err_exists?
     end
 
     # Test or generate output for a specified test
@@ -54,11 +56,13 @@ module RComp
     # generate - Flag for running generate. Runs test otherwise.
     #
     # Returns nothing
-    def run(test, generate=false)
+    def run_test(test, generate=false)
+      # Create output path if it doens't exist
       generate ? mkpath_to(test.expected_out_path) :
         mkpath_to(test.result_out_path)
 
       # Create process and run
+      # Handle the difference in path between a test and generate process
       cmd = [@conf.command, test.test_path]
       out = generate ? test.expected_out_path : test.result_out_path
       err = generate ? test.expected_err_path : test.result_err_path
@@ -67,36 +71,59 @@ module RComp
 
       if process.timedout?
         test.result = :timedout
-      else
-        test.result = generate ? :success : compare_output(test) 
+        return
       end
+
+      test.result = generate ? :success : cmp_output(test) 
     end
 
     # Compare the result and expected output of a test that has been run
     #
     # test - A Test object that has been run
-    # precondition: expected_exists?(test) is true
+    # precondition :: expected_exists?(test) is true
     #
-    # Returns a Symbol, :success or :failure, conditionally if the test passed
-    def compare_output(test)
-      exp_out_exists = File.exists?(test.expected_out_path)
-      exp_err_exists = File.exists?(test.expected_err_path)
-
-      if exp_out_exists && exp_err_exists # test out and err
-        if FileUtils.identical?(test.expected_out_path, test.result_out_path) &&
-          FileUtils.identical?(test.expected_err_path, test.result_err_path)
-          return :success
-        end
-      elsif exp_out_exists # test only out
-        if FileUtils.identical?(test.expected_out_path, test.result_out_path)
-          return :success
-        end
-      else # test only err
-        if FileUtils.identical?(test.expected_err_path, test.result_err_path)
-          return :success
-        end
+    # Returns success or failure as a symbol
+    def cmp_output(test)
+      # test out and err
+      if test.expected_out_exists? && test.expected_err_exists?
+        cmp_out(test)
+        cmp_err(test)
+        return :success if (test.out_result && test.err_result)
+
+      # test only out
+      elsif test.expected_out_exists?
+        cmp_out(test)
+        return :success if test.out_result
+
+      # test only err
+      else
+        cmp_err(test)
+        return :success if test.err_result
       end
+
       return :failed
     end
+
+    # Compare a tests expected and result stdout
+    # Sets the result of the comparison to out_result in the test
+    #
+    # test - A test object that has been run
+    #
+    # Returns nothing
+    def cmp_out(test)
+      test.out_result = FileUtils.cmp(test.expected_out_path, 
+                                      test.result_out_path)
+    end
+
+    # Compare a tests expected and result stderr
+    # Sets the result of the comparison to err_result in the test
+    #
+    # test - A test object that has been run
+    #
+    # Returns nothing
+    def cmp_err(test)
+      test.err_result = FileUtils.cmp(test.expected_err_path,
+                                      test.result_err_path)
+    end
   end
 end

data/lib/rcomp/suite.rb

@@ -3,7 +3,8 @@ require 'find'
 module RComp
   module Suite
 
-    include RComp::Path
+    extend self
+    extend Path
 
     @@conf = Conf.instance
 
@@ -12,7 +13,7 @@ module RComp
     # pattern - A pattern to filter the tests that are added to the suite
     #
     # Returns an Array of Test objects
-    def load_suite(pattern='')
+    def load(pattern=nil)
       tests = []
 
       # Find all tests in the tests directory
@@ -21,7 +22,7 @@ module RComp
         next if File.directory? path
 
         # filter tests by pattern if present
-        unless pattern.empty?
+        if pattern
           next unless rel_path(path).match(pattern)
         end
 
@@ -37,6 +38,8 @@ module RComp
       return tests
     end
 
+    private
+
     # Checks all ignore patterns against a given relative path
     #
     # path - A relative path of a test

data/lib/rcomp/test.rb

@@ -4,9 +4,10 @@ module RComp
     include RComp::Path
 
     attr_reader :test_path, :result_out_path, :result_err_path,
-                :expected_out_path, :expected_err_path, :relative_path
+                :expected_out_path, :expected_err_path, :relative_path,
+                :formatted_path
 
-    attr_accessor :result
+    attr_accessor :result, :out_result, :err_result
 
     # Initialize a new Test
     #
@@ -18,10 +19,42 @@ module RComp
       @result = :skipped
       @relative_path = rel_path(path)
       @test_path = path
-      @result_out_path = result_path(path, :out)
-      @result_err_path = result_path(path, :err)
-      @expected_out_path = expected_path(path, :out)
-      @expected_err_path = expected_path(path, :err)
+      @result_out_path = get_path(:result, path, '.out')
+      @result_err_path = get_path(:result, path, '.err')
+      @expected_out_path = get_path(:expected, path, '.out')
+      @expected_err_path = get_path(:expected, path, '.err')
+      @formatted_path = format_path(@relative_path)
+    end
+
+    def expected_out_exists?
+      @expected_out_exists ||= File.exists?(@expected_out_path)
+    end
+
+    def expected_err_exists?
+      @expected_err_exists ||= File.exists?(@expected_err_path)
+    end
+
+    private
+
+    def get_path(type, test_path, extension)
+      cmpnts = []
+      if type == :result
+        cmpnts << Conf.instance.result_root
+      else
+        cmpnts << Conf.instance.expected_root
+      end
+      cmpnts << rel_path(File.dirname(test_path))
+      cmpnts << File.basename(test_path, ".*") + extension
+      File.join(cmpnts)
+    end
+
+    # Formats relative path for user output
+    #
+    # path - A relative test path
+    #
+    # Returns formatted path
+    def format_path(path)
+      path[1..-1]
     end
   end
 end

data/lib/rcomp/version.rb

@@ -1,3 +1,3 @@
 module RComp
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

data/rcomp.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |s|
   s.authors     = ["Chris Knadler"]
   s.email       = ["takeshi91k@gmail.com"]
   s.homepage    = "https://github.com/cknadler/rcomp"
-  s.summary     = "A simple framework for testing command line application output."
-  s.description = "RComp tests by passing a specified command tests (files) by argument and comparing the result with expected output."
+  s.summary     = "A simple framework for testing command line application output"
+  s.description = "Command line output testing framework with dead simple tests"
 
   s.required_ruby_version     = ">= 1.9.3"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rcomp
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-25 00:00:00.000000000 Z
+date: 2012-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -91,8 +91,7 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.0
-description: RComp tests by passing a specified command tests (files) by argument
-  and comparing the result with expected output.
+description: Command line output testing framework with dead simple tests
 email:
 - takeshi91k@gmail.com
 executables:
@@ -102,7 +101,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
-- CHANGELOG.rdoc
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -121,8 +120,8 @@ files:
 - lib/rcomp/actions.rb
 - lib/rcomp/cli.rb
 - lib/rcomp/conf.rb
+- lib/rcomp/env.rb
 - lib/rcomp/helper.rb
-- lib/rcomp/initializer.rb
 - lib/rcomp/path.rb
 - lib/rcomp/process.rb
 - lib/rcomp/reporter.rb
@@ -155,7 +154,7 @@ rubyforge_project: rcomp
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: A simple framework for testing command line application output.
+summary: A simple framework for testing command line application output
 test_files:
 - features/generate.feature
 - features/init.feature