test-parser 0.8.0

data/spec/example_data/junit4.rb

@@ -0,0 +1,17 @@
+#not sure if this is junit4
+#stolen from http://www.samspublishing.com/articles/article.asp?p=32052&seqNum=6&rl=1
+[
+  ["% java com.networksByteDesign.eMarket.inventory.salesItemTest
+  .F.
+  Time: 0.033
+  There was 1 failure:
+  1) testConstructor(com.networksByteDesign.eMarket.inventory.salesItemTest)
+    \"Expected exception was not thrown\"
+
+  FAILURES!!!
+  Tests run: 2, Failures: 1, Errors: 0",
+  {:failure_count => 1, :success_count => 1, :test_count => 2,
+#    :failures => [{:test => 'testConstructor',
+#                   :message => "Expected exception was not thrown"}]
+  }],
+]

data/spec/example_data/pyunit.rb

@@ -0,0 +1,143 @@
+[
+#baseline
+  [
+  "...................
+----------------------------------------------------------------------
+Ran 19 tests in 2.181s
+
+OK
+  ",
+  {:success_count => 19, :failure_count => 0}
+  ],
+  
+#verbose mode...
+  [
+  "Should perform the same as reading, then parsing the header ... ok
+testHeaderParsing (__main__.TestHTTPProtocol) ... ok
+Attempting to parse a header that doesn't end in \\r\\n\\r\\n should raise a ValueError ... ok
+A valid buffer read in shouldn't cause parse_header to complain ... ok
+Grabbing the next header should leave all other data after the header ... ok
+If two headers are in the buffer reading twice in a row should produce one header, then the next. ... ok
+
+----------------------------------------------------------------------
+Ran 6 tests in 0.003s
+
+OK",
+  {:failure_count => 0, :success_count => 6, 
+
+#YAGNI has stripped this functionality right out...
+#     :test_names => ["Should perform the same as reading, then parsing the header",
+#                     "testHeaderParsing (__main__.TestHTTPProtocol)",
+#                     "Attempting to parse a header that doesn't end in \\r\\n\\r\\n should raise a ValueError",
+#                     "A valid buffer read in shouldn't cause parse_header to complain",
+#                     "Grabbing the next header should leave all other data after the header",
+#                     "If two headers are in the buffer reading twice in a row should produce one header, then the next."]}
+  }],
+#testing for an ERROR
+  [
+  "...E..
+======================================================================
+ERROR: A valid buffer read in shouldn't cause parse_header to complain
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File \"test_http.py\", line 24, in testReadHeader
+    header = self.http.read_header()
+  File \"/Users/peter/Documents/Class Notes/2007s/Networking/labs/software/2/http/http.py\", line 124, in read_header
+    match = find_header.match(self.buffer)
+TypeError: expected string or buffer
+
+----------------------------------------------------------------------
+Ran 6 tests in 0.022s
+
+FAILED (errors=1)",
+  {
+    :failure_count => 1,
+    :success_count => 5,
+    :failures => [{
+      :file => "test_http.py",
+      :line => 24,
+      :error_type => "TypeError",
+      :message => "expected string or buffer",
+      :method => "testReadHeader",
+      :test => "A valid buffer read in shouldn't cause parse_header to complain",  
+    }],
+  }],
+#testing for FAIL
+  [
+  "....F.
+======================================================================
+FAIL: Grabbing the next header should leave all other data after the header
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File \"test_http.py\", line 32, in testReadHeadersHandlesExtraDataProperly
+    self.assertEqual(self.http.buffer,3)#data)
+AssertionError: 'Lieutenant Commander' != 3
+
+----------------------------------------------------------------------
+Ran 6 tests in 0.003s
+
+FAILED (failures=1)
+",
+  {
+    :failure_count => 1,
+    :success_count => 5,
+    :failures => [{
+      :file => "test_http.py",
+      :line => 32,
+      :error_type => "AssertionError",
+      :message => "'Lieutenant Commander' != 3",
+      :test => "Grabbing the next header should leave all other data after the header",
+      :method => "testReadHeadersHandlesExtraDataProperly",
+    }],
+  },
+  ],
+
+#test multiple failures
+  [
+   "...EF.
+======================================================================
+ERROR: A valid buffer read in shouldn't cause parse_header to complain
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File \"test_http.py\", line 24, in testReadHeader
+    header = self.http.read_header()
+  File \"/Users/peter/Documents/Class Notes/2007s/Networking/labs/software/2/http/http.py\", line 124, in read_header
+    match = find_header.match(self.buffer)
+TypeError: expected string or buffer
+
+======================================================================
+FAIL: Grabbing the next header should leave all other data after the header
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File \"test_http.py\", line 32, in testReadHeadersHandlesExtraDataProperly
+    self.assertEqual(self.http.buffer,3)#data)
+AssertionError: 'Lieutenant Commander' != 3
+
+----------------------------------------------------------------------
+Ran 6 tests in 0.003s
+
+FAILED (failures=1, errors=1)",
+   {
+     :success_count => 4,
+     :failure_count => 2,
+     :failures => [
+       {
+         :file => "test_http.py",
+         :line => 32,
+         :error_type => "AssertionError",
+         :message => "'Lieutenant Commander' != 3",
+         :test => "Grabbing the next header should leave all other data after the header",
+         :method => "testReadHeadersHandlesExtraDataProperly",
+       },
+       {
+         :file => "test_http.py",
+         :line => 24,
+         :error_type => "TypeError",
+         :message => "expected string or buffer",
+         :method => "testReadHeader",
+         :test => "A valid buffer read in shouldn't cause parse_header to complain",  
+       }
+       ]
+   }
+  ]
+]

data/spec/example_data/quickCheck.rb

@@ -0,0 +1,61 @@
+[
+[
+'    ___         ___ _
+  / _ \ /\  /\/ __(_)
+ / /_\// /_/ / /  | |      GHC Interactive, version 6.5.20060608, for Haskell 98.
+/ /_\\\\/ __  / /___| |      http://www.haskell.org/ghc/
+\____/\/ /_/\____/|_|      Type :? for help.
+
+Loading package base-1.0 ... linking ... done.
+Prelude> [1 of 2] Compiling QuickCheck       ( QuickCheck.hs, interpreted )
+[2 of 2] Compiling Main             ( primes.hs, interpreted )
+Ok, modules loaded: QuickCheck, Main.
+*Main> Loading package haskell98-1.0 ... linking ... done.
+OK, passed 100 tests.
+*Main> Leaving GHCi.
+',
+ {:failure_count => 0, :success_count => 1, :test_count => 1}
+],
+[
+'  ___         ___ _
+  / _ \ /\  /\/ __(_)
+ / /_\// /_/ / /  | |      GHC Interactive, version 6.5.20060608, for Haskell 98.
+/ /_\\/ __  / /___| |      http://www.haskell.org/ghc/
+\____/\/ /_/\____/|_|      Type :? for help.
+
+Loading package base-1.0 ... linking ... done.
+Prelude> [1 of 2] Compiling QuickCheck       ( QuickCheck.hs, interpreted )
+[2 of 2] Compiling Main             ( primes.hs, interpreted )
+Ok, modules loaded: QuickCheck, Main.
+*Main> Loading package haskell98-1.0 ... linking ... done.
+prop_PrimesArePrime: Falsifiable, after 0 tests:
+5696
+*Main> Leaving GHCi.
+',
+ {:failure_count => 1, :success_count => 0, :test_count => 1,
+  :failures => [{:test => "prop_PrimesArePrime", :message => "Falsifiable, after 0 tests:\n5696"}]
+ }
+],
+
+["   ___         ___ _
+  / _ \\ /\\  /\\/ __(_)
+ / /_\\// /_/ / /  | |      GHC Interactive, version 6.5.20060608, for Haskell 98.
+/ /_\\\\/ __  / /___| |      http://www.haskell.org/ghc/
+\\____/\\/ /_/\\____/|_|      Type :? for help.
+
+Loading package base-1.0 ... linking ... done.
+Prelude> [1 of 2] Compiling QuickCheck       ( QuickCheck.hs, interpreted )
+[2 of 2] Compiling Main             ( primes.hs, interpreted )
+Ok, modules loaded: QuickCheck, Main.
+*Main> Loading package haskell98-1.0 ... linking ... done.
+prop_PrimesArePrime: OK, passed 100 tests.
+*Main> prop_PrimesAreOdd: Falsifiable, after 65 tests:
+2
+*Main> Leaving GHCi.
+",
+ {:failure_count => 1, :success_count => 1, :test_count => 2,
+  :failures => [{:test => "prop_PrimesAreOdd", :message => "Falsifiable, after 65 tests:\n2"}]}
+],
+
+
+]

data/spec/example_data/rspec.rb

@@ -0,0 +1,105 @@
+#This is a big array of example data.
+
+#It consists of pairs of values.
+#The first is the output to parse
+#The second is a hash of values that should be extracted by the parser.
+[
+  ["
+
+Finished in 0.007427 seconds
+
+1 example, 0 failures", 
+{:success_count => 1, :failure_count => 0, :test_count => 1 }],
+
+  ["This is a bunch of text.
+- perhaps we can throw off the parser...
+
+Finished in 0.007427 seconds
+
+2 examples, 0 failures", 
+{:success_count => 2, :failure_count => 0, :test_count => 2}],
+
+  ["[17, 18, 18, 17]
+\"The 'internet'\"
+
+Finished in 0.007427 seconds
+
+2 specifications, 0 failures", 
+{:success_count => 2, :failure_count => 0, :test_count => 2}],
+
+
+  ["the PyUnitParser
+- should be able to count successes and failures in some examples (ERROR - 1)
+
+the rspec parser
+- should agree with the listed examples
+- should be able to pick out a stack trace of an error
+
+1)
+NameError in 'the PyUnitParser should be able to count successes and failures in some examples'
+undefined local variable or method `results' for PyUnitParser:Module
+./spec/../lib/test-parser/parsers/pyunit.rb:8:in `parse'
+./spec/test_pyunit_parser_spec.rb:13:
+./spec/test_pyunit_parser_spec.rb:12:
+
+Finished in 0.01853 seconds
+
+3 examples, 1 failure
+",
+    {:success_count => 2, :failure_count => 1, :test_count => 3,
+       :failures => [{:error_type => "NameError", 
+                    :file => './spec/../lib/test-parser/parsers/pyunit.rb', 
+                    :test => 'the PyUnitParser should be able to count successes and failures in some examples',
+                    :line => 8,
+                    :method => 'parse',
+                    :message => "undefined local variable or method `results' for PyUnitParser:Module" }]
+    }
+  ],
+  ["1)
+  'RSpecParser should agree with the listed examples' FAILED
+expected 1, got 2 (using ==)
+./spec/test_rspec_parser_spec.rb:15:
+
+Finished in 0.08832 seconds
+
+1 examples, 1 failure
+",{:failure_count => 1,
+   :failures => [{:test => 'RSpecParser should agree with the listed examples',
+                    :message => 'expected 1, got 2 (using ==)',
+                    :file => './spec/test_rspec_parser_spec.rb',
+                    :line => 15
+    }]}],
+  ["RSpecParser
+- should agree with the listed examples
+
+PyUnitParser
+- should agree with the listed examples (ERROR - 1)
+
+1)
+SyntaxError in 'PyUnitParser should agree with the listed examples'
+(eval):30: compile error
+(eval):25: parse error, unexpected '=', expecting tASSOC
+    :test_names = [\"Should perform the same as reading, then parsing the header\",
+                 ^
+(eval):30: parse error, unexpected '}', expecting $
+./spec/test_parsers.rb:21:
+./spec/test_parsers.rb:21:
+
+Finished in 0.01552 seconds
+
+2 examples, 1 failure
+",
+    {:failure_count => 1, :success_count => 1,
+     :failures => [
+       {:file => "(eval)",
+        :line => 25,
+        :test => "PyUnitParser should agree with the listed examples",
+        :error_type => "SyntaxError",
+        :message => "(eval):30: compile error",
+        }
+       ]
+    }
+  ],
+  
+  
+]

data/spec/example_data/rubyunit.rb

@@ -0,0 +1,77 @@
+[
+
+["Loaded suite -e
+Started
+.
+Finished in 0.00045 seconds.
+
+1 tests, 1 assertions, 0 failures, 0 errors
+",
+ {:failure_count => 0, :success_count => 1, :test_count => 1}
+],
+
+["Loaded suite test/test_gtest
+Started
+F
+Finished in 0.049108 seconds.
+
+  1) Failure:
+test_truth(TestGtest) [test/test_gtest.rb:9]:
+<false> is not true.
+
+1 tests, 1 assertions, 1 failures, 0 errors
+
+",
+ {:failure_count => 1, :success_count => 0, :test_count => 1,
+  :failures => [{:test => "test_truth(TestGtest)",
+                 :file => "test/test_gtest.rb",
+                 :line => 9,
+                 :message => "<false> is not true."}]}
+ 
+],
+
+["Started
+E
+Finished in 0.00042 seconds.
+
+  1) Error:
+test_truth(TestGtest):
+RuntimeError: What
+    test/test_gtest.rb:9:in `test_truth'
+
+1 tests, 0 assertions, 0 failures, 1 errors
+",
+ {:failure_count => 1, :success_count => 0, :test_count => 1,
+  :failures => [{:test => "test_truth(TestGtest)",
+                 :error_type => "RuntimeError",
+                 :message => "What",
+                 :file => "test/test_gtest.rb",
+                 :line => 9,
+                 :method => "test_truth"}]}
+],
+
+["snarl_notifier is not available
+Loaded suite test/popup_notifier_test
+Started
+E
+Finished in 0.010711 seconds.
+
+  1) Error:
+test_growl(PopupNotifierTest):
+NoMethodError: undefined method `LoadError' for #<PopupNotifier:0x112210c>
+    ./test/../lib/popup_notifier.rb:10:in `initialize'
+    test/popup_notifier_test.rb:14:in `test_growl'
+    /opt/local/lib/ruby/gems/1.8/gems/mocha-0.5.1/lib/mocha/test_case_adapter.rb:19:in `run'
+
+1 tests, 0 assertions, 0 failures, 1 errors
+",
+ {:failure_count => 1, :success_count => 0, :test_count => 1,
+  :failures => [{:test => "test_growl(PopupNotifierTest)",
+                 :error_type => "NoMethodError",
+                 :message => "undefined method `LoadError' for #<PopupNotifier:0x112210c>",
+                 :file => "./test/../lib/popup_notifier.rb",
+                 :line => 10,
+                 :method => "initialize"}]}
+],
+
+]

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'spec'
+$: << File.dirname(__FILE__) + '/../lib/'
+require 'test-parser'

data/spec/test_parsers_spec.rb

@@ -0,0 +1,101 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+def have_key? ex, keys
+  keys = [keys] if !keys.respond_to?(:each)
+  ex.any? {|(_,c,_)| 
+    keys.all? {|key| c[key]}
+  }
+end
+
+def hash_match ex, key
+  ex.each do |test_results, correct_info, parsed_info|
+    next if !correct_info[key] # don't test things that aren't in the data file
+    parsed_info[key].should == correct_info[key]
+  end
+end
+
+
+
+TestParser.parsers.each_value do |parser|
+
+  begin
+    data_filename = File.dirname(__FILE__) + "/example_data/" + parser.name.downcase.sub("testparser::","") + ".rb"
+    examples = eval(File.read(data_filename))
+  rescue Exception => err
+    STDERR.puts "Syntax error in #{data_filename}:\n\n"
+    raise err
+  end
+  
+
+  describe parser do
+    
+    before :all do
+      examples.each do |example|
+        test_results, correct_info = example
+        example << parser.parse(test_results)
+      end
+    end
+    
+    #if there are any examples at all, they should...
+    if !examples.empty?
+      it "should give accurate failure_counts" do
+        hash_match examples, :failure_count
+      end
+    end
+    
+    #if any of the examples include info on any of these properties, ensure that the
+    #parser aligns with that property
+    [:success_count, :test_count].each do |property|
+      if have_key? examples, property
+        it "should give accurate #{property}s" do
+          hash_match examples, property
+        end
+      end
+    end
+    
+    #if any of the examples include detailed info on failures...
+    if have_key? examples, :failures
+
+      it "should give details on failures" do
+        examples.each do |test_results, correct_info, parsed_info|
+
+          next if !correct_info[:failures]
+
+          correct_info[:failures].each do |correct_hash|
+
+            #find the relevent failure in the parsed_info by the filename/line number this may not work for some systems...
+            
+            parsed_hash = parsed_info[:failures].find {|p_h| p_h[:file] == correct_hash[:file] && p_h[:line] == correct_hash[:line] }
+
+            if !parsed_hash
+              fail "didn't find the error in '#{correct_hash[:file]}' line #{correct_hash[:line]}"
+            else
+              #anything in the correct_info needs to match what was parsed
+              #this enables the parsed data to include more info than we're testing for
+              correct_hash.each do |(key,val)|
+                parsed_hash[key].should == val
+              end
+            end
+          end
+        end
+      end
+      
+      #the number of detailed failure accounts should match the number of reported failures
+      it "should give the same number of failure details as there were failures" do
+        examples.each do |test_results, correct_info, parsed_info|
+          parsed_info[:failures].length.should == parsed_info[:failure_count]          
+        end
+      end
+    end
+
+    if have_key? examples, [:test_count,:success_count]
+      it "should give a test_count that's equal to the sum of failure_count and success_count" do
+        examples.each do |_tr,_ci, parsed_info|
+          next if (!parsed_info[:test_count]) || (!parsed_info[:success_count])
+          parsed_info[:test_count].should == parsed_info[:failure_count] + parsed_info[:success_count]
+        end
+      end
+    end
+    
+  end
+end