rltk 2.2.0 → 2.2.1

data/README.md

@@ -312,28 +312,29 @@ Calls to {RLTK::Parser.parse} may raise one of four exceptions:
 
 **Warning: this is the lest tested feature of RLTK.  If you encounter any problems while using it, please let me know so I can fix any bugs as soon as possible.**
 
-When an RLTK parser encounters a token for which there are no more valid actions (and it is on the last parse stack / possible parse-tree path) it will enter error handling mode.  In this mode the parser pops states and input off of the parse stack (the parser is a pushdown automaton after all) until it finds a state that has a shift action for the `ERROR` terminal.  A dummy `ERROR` terminal is then placed onto the parse stack and the shift action is taken.  This error token will have the position information of the token that caused the parser to enter error handling mode.
+When an RLTK parser encounters a token for which there are no more valid actions (and it is on the last parse stack / possible parse-tree path) it will enter error handling mode.  In this mode the parser pops states and input off of the parse stack (the parser is a pushdown automaton after all) until it finds a state that has a shift action for the `ERROR` terminal.  A dummy `ERROR` terminal is then placed onto the parse stack and the shift action is taken.  This error token will have the position information of the token that caused the parser to enter error handling mode.  Additional tokens may have been discarded after this token.
 
-If the input (including the `ERROR` token) can be reduced immediately the associated error handling proc is evaluated and we continue parsing.  If the parser can't immediately reduce it will begin shifting tokens onto the input stack.  This may cause the parser to enter a state in which it again has no valid actions for an input.  When this happens it enters error handling mode again and pops states and input off of the stack until it reaches an error state again.  In this way it searches for the first substring after the error occurred for which it can resume parsing.
+If the input (including the `ERROR` token) can be reduced immediately the associated error handling proc is evaluated and we continue parsing.  If no shift or reduce action is available the parser will being shifting tokens off of the input stack until a token appears with a valid action in the current state, in which case parsing resumes as normal.
 
-The example below for the unit tests shows a very basic usage of error productions:
+The value of an `ERROR` non-terminal will be an array containing all of the tokens that were discarded while the parser was searching for a valid action.
 
-	class AfterPlsError < StandardError; end
-	class AfterSubError < StandardError; end
+The example below, based on one of the unit tests, shows a very basic usage of error productions:
 
 	class ErrorCalc < RLTK::Parser
+		left :ERROR
+		right :PLS, :SUB, :MUL, :DIV, :NUM
+		
 		production(:e) do
-			clause('NUM') { |n| n }
-			
+			clause('NUM') {|n| n}
+		
 			clause('e PLS e') { |e0, _, e1| e0 + e1 }
 			clause('e SUB e') { |e0, _, e1| e0 - e1 }
 			clause('e MUL e') { |e0, _, e1| e0 * e1 }
 			clause('e DIV e') { |e0, _, e1| e0 / e1 }
-			
-			clause('e PLS ERROR') { |_, _, _| raise AfterPlsError }
-			clause('e SUB ERROR') { |_, _, _| raise AfterSubError }
-		end
 		
+			clause('e PLS ERROR e') { |e0, _, err, e1| error("#{err.len} tokens skipped."); e0 + e1 }
+		end
+	
 		finalize
 	end
 

data/lib/rltk/parser.rb

@@ -835,6 +835,13 @@ module RLTK # :nodoc:
 							# If we are already in error mode and there
 							# are no actions we skip this token.
 							if error_mode
+								v.puts("Discarding token: #{token.type}#{if token.value then "(#{token.value})" end}") if v
+								
+								# Add the current token to the array
+								# that corresponds to the output value
+								# for the ERROR token.
+								stack.output_stack.last << token
+								
 								moving_on << stack
 								next
 							end
@@ -842,6 +849,16 @@ module RLTK # :nodoc:
 							# We would be dropping the last stack so we
 							# are going to go into error mode.
 							if accepted.empty? and moving_on.empty? and processing.empty?
+								
+								if v
+									v.puts
+									v.puts('Current stack:')
+									v.puts("\tID: #{stack.id}")
+									v.puts("\tState stack:\t#{stack.state_stack.inspect}")
+									v.puts("\tOutput Stack:\t#{stack.output_stack.inspect}")
+									v.puts
+								end
+								
 								# Try and find a valid error state.
 								while stack.state
 									if (actions = @states[stack.state].on?(:ERROR)).empty?
@@ -850,7 +867,7 @@ module RLTK # :nodoc:
 										stack.pop
 									else
 										# Enter the found error state.
-										stack.push(actions.first.id, nil, :ERROR, token.position)
+										stack.push(actions.first.id, [token], :ERROR, token.position)
 										
 										break
 									end
@@ -860,9 +877,12 @@ module RLTK # :nodoc:
 									# We found a valid error state.
 									error_mode = reduction_guard = true
 									opts[:env].he = true
-									processing << stack
+									moving_on << stack
 									
-									v.puts('Invalid input encountered.  Entering error handling mode.') if v
+									if v
+										v.puts('Invalid input encountered.  Entering error handling mode.')
+										v.puts("Discarding token: #{token.type}#{if token.value then "(#{token.value})" end}")
+									end
 								else
 									# No valid error states could be
 									# found.  Time to print a message

data/lib/rltk/version.rb

@@ -5,7 +5,7 @@
 
 module RLTK # :nodoc:
 	# The version number of the RLTK library.
-	VERSION			= '2.2.0'
+	VERSION			= '2.2.1'
 	# The version of LLVM targeted by RLTK.
 	LLVM_TARGET_VERSION	= '3.0'
 end

data/test/tc_parser.rb

@@ -127,6 +127,9 @@ class ParserTester < Test::Unit::TestCase
 	class DummyError2 < StandardError; end
 	
 	class ErrorCalc < RLTK::Parser
+		left :ERROR
+		right :PLS, :SUB, :MUL, :DIV, :NUM
+		
 		production(:e) do
 			clause('NUM') {|n| n}
 		
@@ -135,8 +138,7 @@ class ParserTester < Test::Unit::TestCase
 			clause('e MUL e') { |e0, _, e1| e0 * e1 }
 			clause('e DIV e') { |e0, _, e1| e0 / e1 }
 		
-			clause('e PLS ERROR') { |_, _, _| raise DummyError1 }
-			clause('e SUB ERROR') { |_, _, _| raise DummyError2 }
+			clause('e PLS ERROR e') { |e0, _, ts, e1| error(ts); e0 + e1 }
 		end
 	
 		finalize
@@ -159,7 +161,7 @@ class ParserTester < Test::Unit::TestCase
 			clause('NEWLINE') { |_| nil }
 		
 			clause('WORD+ SEMI NEWLINE')	{ |w, _, _| w }
-			clause('WORD+ ERROR NEWLINE')	{ |w, e, _| error(pos(1).line_number); w }
+			clause('WORD+ ERROR')		{ |w, e| error(pos(1).line_number); w }
 		end
 	
 		finalize
@@ -277,8 +279,8 @@ class ParserTester < Test::Unit::TestCase
 	end
 	
 	def test_error_productions
-		assert_raise(DummyError1) { ErrorCalc.parse(RLTK::Lexers::Calculator.lex('1 + +')) }
-		assert_raise(DummyError2) { ErrorCalc.parse(RLTK::Lexers::Calculator.lex('1 - +')) }
+		
+		# Test to see if error reporting is working correctly.
 		
 		test_string  = "first line;\n"
 		test_string += "second line\n"
@@ -289,8 +291,30 @@ class ParserTester < Test::Unit::TestCase
 		
 		begin
 			ErrorLine.parse(ELLexer.lex(test_string))
+			
+		rescue RLTK::HandledError => e
+			assert_equal([2,4], e.errors)
+		end
+		
+		# Test to see if we can continue parsing after errors are encounterd.
+		
+		begin
+			ErrorCalc.parse(RLTK::Lexers::Calculator.lex('1 + + 1'))
+			
+		rescue RLTK::HandledError => e
+			assert_equal(1, e.errors.first.length)
+			assert_equal(2, e.result)
+		end
+		
+		# Test to see if we pop tokens correctly after an error is
+		# encountered.
+		
+		begin
+			ErrorCalc.parse(RLTK::Lexers::Calculator.lex('1 + + + + + + 1'))
+			
 		rescue RLTK::HandledError => e
-			assert_equal(e.errors, [2,4])
+			assert_equal(5, e.errors.first.length)
+			assert_equal(2, e.result)
 		end
 	end
 	

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rltk
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-09 00:00:00.000000000 Z
+date: 2012-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi