ebnf 0.2.3 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDg4MzMwZTA0Yzg3MzY5Y2UyZDMwMWI2ZWRiODNjZWZjZmU5NGRlZQ==
+    ZDI4YWE4ZjExOGI5N2NkZDVmYzA4ZDMwMzM0ZGRhZThhNWU0ZGI2MQ==
   data.tar.gz: !binary |-
-    NzA2ODg3ZTI1NzIwNjVjNmE1YWJjMDYzNTEyNjI1NGIzZWI5OTg1ZQ==
+    MWZmMjVkYjMwYzY1NmNlMDc0ZDM5MzUxOWU1ZTUwNGU2NDQ2YzFiYQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    N2FjYTRhNzI5NDFkZWI3OTNiOWFkNTNmYWY5NjUwMjA4YTM0ZmNiZTQ5NDYy
-    MjFjODYxODc1MzRjMTZjNGM0N2U4NDk2NTM2ZjM0MmY0ZGI3ZDE3OTg3OWFi
-    NjQwYTdjNzcxYTc3ZWJmYzcyMzlmOWJiZDlmZjc3YTJiOWVkODA=
+    M2E2MTcyM2MyZWYyNTg5Y2E3ZDVkNWI2ODgwYjdhZjhlYmQ1YTUzZGE5Nzc3
+    M2M3MDFiYzllMzE5YjNmMDA2Zjc3NjQ5YzViMzhmMTNmZTBkYWZhMDk0M2Rl
+    MjlkNWMxNzY3ODM4ZDYxZjBiZWI2NzA4ZGRjYzQ2MGY2MTBiZDI=
   data.tar.gz: !binary |-
-    ODVmZDZlMjVkMjY0ZDkwYWFiZGY4OGE3ZTRhNDQzZWRiZDlkZTQzODQ4ZWMw
-    MTZkZGMzNWYzZTMxNDc0MDc3YWFkNmU2NjExNWM2ZTkzOTJlZWE0MDQ1Yzdm
-    N2RmOTU5NGFhZTVlZDEwYzhlMjJjMDNhYmIwNDkyOTc3ZGVkNDQ=
+    OGIxZGZjNzNhZTlkZDAwZTM0ZmExYzMzMzUxMjQ4ZmNmYjBmNTM4OGJlMmM2
+    NmFmODM4MjMwODYyY2E1YjhjNjNlY2E1YjI0MTg1NjFlYWI0MzI4ZThhMjA2
+    MmMyMjBiZjA5NGQ2MjlhYzA0Yzg1YzYzNzhjNjNkZmQ2ZTg5N2I=

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.3.0

data/lib/ebnf/base.rb

@@ -103,12 +103,12 @@ module EBNF
     include Parser
 
     # Abstract syntax tree from parse
-    # @!attribute [r] ast
+    #
     # @return [Array<Rule>]
     attr_reader :ast
 
     # Grammar errors, or errors found genering parse tables
-    # @!attribute [r] errors
+    #
     # @return [Array<String>]
     attr_accessor :errors
 

data/lib/ebnf/ll1.rb

@@ -6,29 +6,29 @@ module EBNF
 
     # Branch table, represented as a recursive hash.
     # The table is indexed by rule symbol, which in-turn references a hash of terminals (which are the first terminals of the production), which in turn reference the sequence of rules that follow, given that terminal as input
-    # @!attribute [r] branch
+    #
     # @return [Hash{Symbol => Hash{String, Symbol => Array<Symbol>}}]
     attr_reader :branch
 
     # First table
-    # @!attribute [r] first
+    #
     # @return [Hash{Symbol, String => Symbol}]
     attr_reader :first
 
     # Follow table
-    # @!attribute [r] first
+    #
     # @return [Hash{Symbol, String => Symbol}]
     attr_reader :follow
 
     # Terminal table
     # The list of terminals used in the grammar.
-    # @!attribute [r] terminals
+    #
     # @return [Array<String, Symbol>]
     attr_reader :terminals
 
     # Start symbol
     # The rule which starts the grammar
-    # @!attribute[r] start
+    #
     # @return [Symbol]
     attr_reader :start
 

data/lib/ebnf/ll1/lexer.rb

@@ -53,12 +53,10 @@ module EBNF::LL1
     ML_START            = /\'\'\'|\"\"\"/.freeze              # Beginning of terminals that may span lines
 
     ##
-    # @!attribute whitespace
     # @return [Regexp] defines whitespace, defaults to WS
     attr_reader :whitespace
 
     ##
-    # @!attribute comment
     # @return [Regexp] defines single-line comment, defaults to COMMENT
     attr_reader :comment
 
@@ -233,15 +231,16 @@ module EBNF::LL1
     #
     # @return [Token]
     def recover
-      until scanner.eos? do
-        begin
-          shift
-          return first
-        rescue Error, ArgumentError
-          # Ignore errors until something scans, or EOS.
-          scanner.pos = scanner.pos + 1
+       until scanner.eos? || tok = match_token
+        if scanner.skip_until(@whitespace).nil? # Skip past current "token"
+          # No whitespace at the end, must be and end of string
+          scanner.terminate
+        else
+          skip_whitespace
         end
       end
+      scanner.unscan if tok
+      first
     end
   protected
 
@@ -253,9 +252,10 @@ module EBNF::LL1
     def skip_whitespace
       # skip all white space, but keep track of the current line number
       while !scanner.eos?
-       if matched = scanner.scan(@whitespace)
+        if matched = scanner.scan(@whitespace)
           @lineno += matched.count("\n")
-        elsif (com = scanner.scan(@comment))
+        elsif (scanner.scan(@comment))
+          #
         else
           return
         end
@@ -472,7 +472,7 @@ module EBNF::LL1
       #
       # @return [String]
       def inspect
-        to_hash.inspect
+        "#{@value.inspect}#{'(' + @type.to_s + ')' if @type}"
       end
     end # class Token
 

data/lib/ebnf/ll1/parser.rb

@@ -10,7 +10,6 @@ module EBNF::LL1
     DEBUG_LEVEL = 10
 
     ##
-    # @!attribute [r] lineno
     # @return [Integer] line number of current token
     attr_reader :lineno
 
@@ -20,10 +19,10 @@ module EBNF::LL1
 
     # DSL for creating terminals and productions
     module ClassMethods
-      def start_handlers; @@start_handlers || {}; end
-      def production_handlers; @@production_handlers || {}; end
-      def terminal_handlers; @@terminal_handlers || {}; end
-      def patterns; @@patterns || []; end
+      def start_handlers; @start_handlers || {}; end
+      def production_handlers; @production_handlers || {}; end
+      def terminal_handlers; @terminal_handlers || {}; end
+      def patterns; @patterns || []; end
 
       ##
       # Defines the pattern for a terminal node and a block to be invoked
@@ -53,11 +52,11 @@ module EBNF::LL1
       #   Block passed to initialization for yielding to calling parser.
       #   Should conform to the yield specs for #initialize
       def terminal(term, regexp, options = {}, &block)
-        @@patterns ||= []
+        @patterns ||= []
         # Passed in order to define evaulation sequence
-        @@patterns << EBNF::LL1::Lexer::Terminal.new(term, regexp, options)
-        @@terminal_handlers ||= {}
-        @@terminal_handlers[term] = block if block_given?
+        @patterns << EBNF::LL1::Lexer::Terminal.new(term, regexp, options)
+        @terminal_handlers ||= {}
+        @terminal_handlers[term] = block if block_given?
       end
 
       ##
@@ -80,8 +79,8 @@ module EBNF::LL1
       #   Should conform to the yield specs for #initialize
       # Yield to generate a triple
       def start_production(term, &block)
-        @@start_handlers ||= {}
-        @@start_handlers[term] = block
+        @start_handlers ||= {}
+        @start_handlers[term] = block
       end
 
       ##
@@ -105,8 +104,8 @@ module EBNF::LL1
       #   Should conform to the yield specs for #initialize
       # Yield to generate a triple
       def production(term, &block)
-        @@production_handlers ||= {}
-        @@production_handlers[term] = block
+        @production_handlers ||= {}
+        @production_handlers[term] = block
       end
 
       # Evaluate a handler, delegating to the specified object.
@@ -115,15 +114,15 @@ module EBNF::LL1
       # @param [Object] object
       # @return [Object]
       def eval_with_binding(object)
-        @@delegate = object
+        @delegate = object
         object.instance_eval {yield}
       end
 
       private
 
       def method_missing(method, *args, &block)
-        if @@delegate ||= nil
-          @@delegate.send method, *args, &block
+        if @delegate ||= nil
+          @delegate.send method, *args, &block
         else
           super
         end
@@ -137,29 +136,40 @@ module EBNF::LL1
     #
     # @example
     #   require 'rdf/ll1/parser'
-    #   
+    #
     #   class MyParser
     #     include EBNF::LL1::Parser
-    #     
+    #
     #     branch      MyParser::BRANCH
-    #     
+    #
+    #     ##
+    #     # Defines a production called during before parsing a non-terminal
+    #     # with data from previous production along with data defined for the
+    #     # current production
+    #     #
+    #     start_production :object do |input, current, callback|
+    #       # Note production as triples for blankNodePropertyList
+    #       # to set :subject instead of :resource
+    #       current[:triples] = true
+    #     end
+    #
     #     ##
-    #     # Defines a production called during different phases of parsing
+    #     # Defines a production called during after parsing a non-terminal
     #     # with data from previous production along with data defined for the
     #     # current production
     #     #
-    #     # Yield to generate a triple
-    #     production :object do |parser, phase, input, current|
+    #     # callback to processor block
+    #     production :object do |input, current, callback|
     #       object = current[:resource]
-    #       yield :statement, RDF::Statement.new(input[:subject], input[:predicate], object)
+    #       callback.call :statement, RDF::Statement.new(input[:subject], input[:predicate], object)
     #     end
-    #     
+    #
     #     ##
     #     # Defines the pattern for a terminal node
-    #     terminal :BLANK_NODE_LABEL, %r(_:(#{PN_LOCAL})) do |parser, production, token, input|
+    #     terminal :BLANK_NODE_LABEL, %r(_:(#{PN_LOCAL})) do |production, token, input|
     #       input[:BLANK_NODE_LABEL] = RDF::Node.new(token)
     #     end
-    #     
+    #
     #     ##
     #     # Iterates the given block for each RDF statement in the input.
     #     #
@@ -168,7 +178,7 @@ module EBNF::LL1
     #     # @return [void]
     #     def each_statement(&block)
     #       @callback = block
-    #   
+    #
     #       parse(START.to_sym) do |context, *data|
     #         case context
     #         when :statement
@@ -176,11 +186,12 @@ module EBNF::LL1
     #         end
     #       end
     #     end
-    #     
+    #
     #   end
     #
     # @param  [String, #to_s]          input
-    # @param [Symbol, #to_s] prod The starting production for the parser. It may be a URI from the grammar, or a symbol representing the local_name portion of the grammar URI.
+    # @param [Symbol, #to_s] start
+    #   The starting production for the parser. It may be a URI from the grammar, or a symbol representing the local_name portion of the grammar URI.
     # @param  [Hash{Symbol => Object}] options
     # @option options [Hash{Symbol,String => Hash{Symbol,String => Array<Symbol,String>}}] :branch LL1 branch table.
     # @option options [HHash{Symbol,String => Array<Symbol,String>}] :first ({})
@@ -202,8 +213,11 @@ module EBNF::LL1
     # @yieldparam [Symbol] *data
     #   Data specific to the call
     # @return [EBNF::LL1::Parser]
+    # @raise [Exception] Raises exceptions for parsing errors
+    #   or errors raised during processing callbacks. Internal
+    #   errors are raised using {Error}.
     # @see http://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
-    def parse(input = nil, prod = nil, options = {}, &block)
+    def parse(input = nil, start = nil, options = {}, &block)
       @options = options.dup
       @branch  = options[:branch]
       @first  = options[:first] ||= {}
@@ -217,110 +231,171 @@ module EBNF::LL1
 
       # Unrecoverable errors
       raise Error, "Branch table not defined" unless @branch && @branch.length > 0
-      raise Error, "Starting production not defined" unless prod
+      raise Error, "Starting production not defined" unless start
 
       @prod_data = [{}]
-      prod = prod.split('#').last.to_sym unless prod.is_a?(Symbol)
-      todo_stack = [{:prod => prod, :terms => nil}]
+      start = start.split('#').last.to_sym unless start.is_a?(Symbol)
+      todo_stack = [{:prod => start, :terms => nil}]
 
       while !todo_stack.empty?
-        pushed = false
-        if todo_stack.last[:terms].nil?
-          todo_stack.last[:terms] = []
-          cur_prod = todo_stack.last[:prod]
-
-          # If cur_prod is the starting production, we can reset the stack
-          # to the beginning to avoid excessive growth in the production
-          # stack
-          if options[:reset_on_start] && cur_prod == prod
-            todo_stack = [{:prod => prod, :terms => []}]
-            @productions = []
-            @prod_data = [{}]
-          end
-
-          # Get this first valid token appropriate for the stacked productions,
-          # skipping invalid tokens until either a valid token is found (from @first),
-          # or a token appearing in @follow appears.
-          token = skip_until_valid(todo_stack)
-          
-          # At this point, token is either nil, in the first set of the production,
-          # or in the follow set of this production or any previous production
-          debug("parse(production)") do
-            "token #{token ? token.representation.inspect : 'nil'}, " + 
-            "prod #{cur_prod.inspect}, " + 
-            "depth #{depth}"
-          end
+        begin
+          @recovering = false
+          pushed = false
+          if todo_stack.last[:terms].nil?
+            todo_stack.last[:terms] = []
+            cur_prod = todo_stack.last[:prod]
+
+            # If cur_prod is the starting production, we can reset the stack
+            # to the beginning to avoid excessive growth in the production
+            # stack
+            if options[:reset_on_start] && cur_prod == start
+              todo_stack = [{:prod => start, :terms => []}]
+              @productions = []
+              @prod_data = [{}]
+            end
 
-          # Got an opened production
-          onStart(cur_prod)
-          break if token.nil?
+            # Fetch the current token
+            token = get_token(:recover)
 
-          if prod_branch = @branch[cur_prod]
-            @recovering = false
-            sequence = prod_branch[token.representation]
-            debug("parse(production)", :level => 2) do
-              "token #{token.representation.inspect} " +
-              "prod #{cur_prod.inspect}, " + 
-              "prod_branch #{prod_branch.keys.inspect}, " +
-              "sequence #{sequence.inspect}"
+            # At this point, token is either nil, in the first set of the production,
+            # or in the follow set of this production or any previous production
+            debug("parse(production)") do
+              "token #{token ? token.representation.inspect : 'nil'}, " +
+              "prod #{cur_prod.inspect}, " +
+              "depth #{depth}"
             end
 
-            if sequence.nil?
-              if prod_branch.has_key?(:_empty)
-                debug("parse(production)", :level => 2) {"empty sequence for _empty"}
+            # Got an opened production
+            onStart(cur_prod)
+
+            if token.nil?
+              if  !(first_include?(cur_prod, :_eps) && follow_include?(cur_prod, :_eof))
+                # End of file, and production does not contain eps, or it does, but follow does not contain eof 
+                raise Error.new("Unexpected end of input", :production => cur_prod)
               else
-                # If there is no sequence for this production, we're
-                # in error recovery, and _token_ has been advanced to
-                # the point where it can reasonably follow this production
+                debug("parse(production)") {"End of input prod #{cur_prod.inspect}"}
+              end
+            elsif prod_branch = @branch[cur_prod]
+              sequence = prod_branch.fetch(token.representation) do
+                raise Error.new("#{token.inspect} does not match production #{cur_prod.inspect}",
+                  :production => cur_prod)
+              end
+              debug("parse(production)") do
+                "token #{token.representation.inspect} " +
+                "prod #{cur_prod.inspect}, " +
+                "prod_branch #{prod_branch.keys.inspect}, " +
+                "sequence #{sequence.inspect}"
               end
+              todo_stack.last[:terms] += sequence
+            else
+              raise Error.new("No branches found for #{cur_prod.inspect}",
+                :production => cur_prod, :token => token)
             end
-            todo_stack.last[:terms] += sequence if sequence
-          else
-            # Is this a fatal error?
-            error("parse(fatal?)", "No branches found for #{cur_prod.inspect}",
-              :production => cur_prod, :token => token)
           end
-        end
-        
-        debug("parse(terms)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}"}
-        while !todo_stack.last[:terms].to_a.empty?
-          begin
+
+          debug("parse(terms)") {"todo #{todo_stack.last.inspect}, depth #{depth}"}
+          while !todo_stack.last[:terms].to_a.empty?
             # Get the next term in this sequence
             term = todo_stack.last[:terms].shift
             debug("parse(token)") {"accept #{term.inspect}"}
+
             if token = accept(term)
-              @recovering = false
               debug("parse(token)") {"token #{token.inspect}, term #{term.inspect}"}
-              onToken(term, token)
-            elsif terminals.include?(term)
+              onTerminal(term, token)
+            elsif terminals.include?(term) 
               # If term is a terminal, then it is an error if token does not
               # match it
-              skip_until_valid(todo_stack)
+              raise Error.new("#{get_token.inspect} does not match terminal #{term.inspect}",
+                :production => cur_prod)
             else
-              # If it's not a string (a symbol), it is a non-terminal and we push the new state
-              todo_stack << {:prod => term, :terms => nil}
-              debug("parse(push)", :level => 2) {"term #{term.inspect}, depth #{depth}"}
-              pushed = true
-              break
+              token = get_token
+
+              # If token is not in firsts of term, but eps is, skip to next
+              # term
+              if first_include?(term, :_eps) && !first_include?(term, token)
+                debug("parse(token)") {"skip optional term #{term.inspect} on #{token.inspect}"}
+                break
+              else
+                # Push term onto stack
+                todo_stack << {:prod => term, :terms => nil}
+                debug("parse(push)") {"term #{term.inspect}, depth #{depth}"}
+                pushed = true
+                break
+              end
             end
           end
-        end
-        
-        # After completing the last production in a sequence, pop down until we find a production
-        #
-        # If in recovery mode, continue popping until we find a term with a follow list
-        while !pushed &&
-              !todo_stack.empty? &&
-              ( (terms = todo_stack.last.fetch(:terms, [])).empty? ||
-                (@recovering && @follow.fetch(terms.last, []).none? {|t| (token || :_eps) == t}))
-          debug("parse(pop)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}"}
-          if terms.empty?
+        rescue Lexer::Error, Error => e
+          # Lexer encountered an illegal token or the parser encountered
+          # a terminal which is inappropriate for the current production.
+          # Perform error recovery to find a reasonable terminal based
+          # on the follow sets of the relevant productions. This includes
+          # remaining terms from the current production and the stacked
+          # productions
+          @lineno = e.lineno
+          if e.is_a?(Lexer::Error)
+            # Skip to the next valid terminal
+            @lexer.recover
+            error("parse(#{e.class})", "With input '#{e.input}': #{e.message}, skipped to #{(get_token(:recover) || :eof).inspect}",
+                  :production => @productions.last)
+          else
+            # Otherwise, the terminal is fine, just not for this production.
+            @lexer.shift
+            error("parse(#{e.class})", "#{e.message}, skipped to #{(get_token(:recover) || :eof).inspect}",
+                  :production => @productions.last, :token => e.token)
+          end
+
+          # Get the list of follows for this sequence, this production and the stacked productions.
+          debug("recovery", "stack follows:", :level => 4)
+          todo_stack.reverse.each do |todo|
+            debug("recovery", :level => 4) {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
+          end
+
+          # Find all follows to the top of the stack
+          follows = todo_stack.inject([]) do |follow, todo|
+            prod = todo[:prod]
+            follow += @follow[prod] || []
+          end.uniq
+          debug("recovery") {"follows: #{follows.inspect}"}
+
+          # Skip tokens until one is found in follows
+          while (token = get_token(:recover)) && follows.none? {|t| token === t}
+            skipped = @lexer.shift
+            progress("recovery") {"skip #{skipped.inspect}"}
+          end
+          debug("recovery") {"found #{token.inspect} in follows"}
+
+          # Pop stack elements until token is in follows
+          while !todo_stack.empty? &&
+                !follow_include?(todo_stack.last[:prod], token || :_eof)
+            debug("recovery(pop)") {"todo #{todo_stack.last.inspect}, depth #{depth}"}
             todo_stack.pop
             onFinish
+          end
+
+          # Token is now in the first of the top production
+          unless todo_stack.empty?
+            todo_stack.pop
+            onFinish
+          end
+
+          if todo_stack.empty?
+            # Recovered to end of last production
+            warn("recover", "recovered to end of productions")
           else
-            # Stop recovering when we a production which starts with the term
-            debug("parse(pop)", :level => 2) {"recovery complete"}
-            @recovering = false
+            warn("recover", "recovered to #{todo_stack.last[:prod].inspect} with #{token.inspect}")
+          end
+
+          @recovering = false
+        ensure
+          # After completing the last production in a sequence, pop down until we find a production
+          #
+          # If in recovery mode, continue popping until we find a term with a follow list
+          while !pushed &&
+                !todo_stack.empty? &&
+                todo_stack.last.fetch(:terms, []).empty?
+            debug("parse(pop)") {"todo #{todo_stack.last.inspect}, depth #{depth}"}
+            todo_stack.pop
+            onFinish
           end
         end
       end
@@ -329,12 +404,10 @@ module EBNF::LL1
 
       # Continue popping contexts off of the stack
       while !todo_stack.empty?
-        debug("parse(eof)", :level => 2) {"stack #{todo_stack.last.inspect}, depth #{depth}"}
+        debug("parse(eof)") {"stack #{todo_stack.last.inspect}, depth #{depth}"}
         # There can't be anything left to do, or if there is, it must be optional
         last_terms = todo_stack.last[:terms]
-        if last_terms.length > 0 && last_terms.none? {|t|
-          @first.has_key?(t) && @first[t].include?(:_eps)
-        }
+        if last_terms.length > 0 && last_terms.none? {|t|first_include?(t, :_eps)}
           error("parse(eof)",
             "End of input before end of production: stack #{todo_stack.last.inspect}, depth #{depth}"
           )
@@ -342,10 +415,10 @@ module EBNF::LL1
         todo_stack.pop
         onFinish
       end
-      
+
       # When all is said and done, raise the error log
       unless @error_log.empty?
-        raise Error, @error_log.join("\n\t") 
+        raise Error, @error_log.join("\n\t")
       end
     end
 
@@ -369,16 +442,108 @@ module EBNF::LL1
         prod_data[sym] << values
       end
     end
-    
+
     # Add values to production data, values aranged as an array
     def add_prod_data(sym, *values)
       return if values.compact.empty?
-      
+
       prod_data[sym] ||= []
       prod_data[sym] += values
       debug("add_prod_data(#{sym})") {"#{prod_data[sym].inspect} += #{values.inspect}"}
     end
-    
+
+  protected
+
+    ##
+    # Error information, used as level `0` debug messages.
+    #
+    # @param [String] node Relevant location associated with message
+    # @param [String] message Error string
+    # @param [Hash] options
+    # @option options [URI, #to_s] :production
+    # @option options [Token] :token
+    # @see {#debug}
+    def error(node, message, options = {})
+      message += ", found #{options[:token].inspect}" if options[:token]
+      message += " at line #{@lineno}" if @lineno
+      message += ", production = #{options[:production].inspect}" if options[:production]
+      @error_log << message unless @recovering
+      @recovering = true
+      debug(node, message, options.merge(:level => 0))
+    end
+
+    ##
+    # Warning information, used as level `1` debug messages.
+    #
+    # @param [String] node Relevant location associated with message
+    # @param [String] message Error string
+    # @param [Hash] options
+    # @option options [URI, #to_s] :production
+    # @option options [Token] :token
+    # @see {#debug}
+    def warn(node, message, options = {})
+      message += ", with token #{options[:token].inspect}" if options[:token]
+      message += " at line #{@lineno}" if @lineno
+      message += ", production = #{options[:production].inspect}" if options[:production]
+      @error_log << message unless @recovering
+      debug(node, message, options.merge(:level => 1))
+    end
+
+    ##
+    # Progress output when parsing. Passed as level `2` debug messages.
+    #
+    # @overload progress(node, message, options)
+    #   @param [String] node Relevant location associated with message
+    #   @param [String] message ("")
+    #   @param [Hash] options
+    #   @option options [Integer] :depth
+    #       Recursion depth for indenting output
+    # @see {#debug}
+    def progress(node, *args)
+      return unless @options[:progress] || @options[:debug]
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      message = args.join(",")
+      message += yield.to_s if block_given?
+      debug(node, message, options.merge(:level => 2))
+    end
+
+    ##
+    # Progress output when debugging.
+    #
+    # The call is ignored, unless `@options[:debug]` is set, in which
+    # case it yields tracing information as indicated. Additionally,
+    # if `@options[:debug]` is an Integer, the call is aborted if the
+    # `:level` option is less than than `:level`.
+    #
+    # @overload debug(node, message, options)
+    #   @param [Array<String>] args Relevant location associated with message
+    #   @param [Hash] options
+    #   @option options [Integer] :depth
+    #     Recursion depth for indenting output
+    #   @option options [Integer] :level
+    #     Level assigned to message, by convention, level `0` is for
+    #     errors, level `1` is for warnings, level `2` is for parser
+    #     progress information, and anything higher is for various levels
+    #     of debug information.
+    #
+    # @yield trace, level, lineno, depth, args
+    # @yieldparam [:trace] trace
+    # @yieldparam [Integer] level
+    # @yieldparam [Integer] lineno
+    # @yieldparam [Integer] depth Recursive depth of productions
+    # @yieldparam [Array<String>] args
+    # @yieldreturn [String] added to message
+    def debug(*args)
+      return unless @options[:debug] && @parse_callback
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      debug_level = options.fetch(:level, 3)
+      return if @options[:debug].is_a?(Integer) && debug_level > @options[:debug]
+
+      depth = options[:depth] || self.depth
+      args << yield if block_given?
+      @parse_callback.call(:trace, debug_level, @lineno, depth, *args)
+    end
+
   private
     # Start for production
     def onStart(prod)
@@ -389,15 +554,20 @@ module EBNF::LL1
         # to customize before pushing on the @prod_data stack
         progress("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
         data = {}
-        self.class.eval_with_binding(self) {
-          handler.call(@prod_data.last, data, @parse_callback)
-        }
+        begin
+          self.class.eval_with_binding(self) {
+            handler.call(@prod_data.last, data, @parse_callback)
+          }
+        rescue Exception => e
+          error("start", "#{e.class}: #{e.message}", :production => prod)
+          @recovering = false
+        end
         @prod_data << data
       else
         # Make sure we push as many was we pop, even if there is no
         # explicit start handler
         @prod_data << {} if self.class.production_handlers[prod]
-        progress("#{prod}(:start)") { get_token.inspect}
+        progress("#{prod}(:start)") { get_token.inspect + (@recovering ? ' recovering' : '')}
       end
       #puts "prod_data(s): " + @prod_data.inspect
     end
@@ -410,201 +580,105 @@ module EBNF::LL1
       if handler && !@recovering
         # Pop production data element from stack, potentially allowing handler to use it
         data = @prod_data.pop
-        self.class.eval_with_binding(self) {
-          handler.call(@prod_data.last, data, @parse_callback)
-        }
+        begin
+          self.class.eval_with_binding(self) {
+            handler.call(@prod_data.last, data, @parse_callback)
+          }
+        rescue Exception => e
+          error("finish", "#{e.class}: #{e.message}", :production => prod)
+          @recovering = false
+        end
         progress("#{prod}(:finish):#{@prod_data.length}") {@prod_data.last}
       else
-        progress("#{prod}(:finish)", "recovering: #{@recovering.inspect}")
+        progress("#{prod}(:finish)") { "recovering" if @recovering }
       end
       @productions.pop
     end
 
-    # A token
-    def onToken(prod, token)
+    # A terminal
+    def onTerminal(prod, token)
       unless @productions.empty?
         parentProd = @productions.last
         handler = self.class.terminal_handlers[prod]
         # Allows catch-all for simple string terminals
         handler ||= self.class.terminal_handlers[nil] if prod.is_a?(String)
         if handler
-          self.class.eval_with_binding(self) {
-            handler.call(parentProd, token, @prod_data.last)
-          }
-          progress("#{prod}(:token)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
+          begin
+            self.class.eval_with_binding(self) {
+              handler.call(parentProd, token, @prod_data.last, @parse_callback)
+            }
+          rescue Exception => e
+            error("terminal", "#{e.class}: #{e.message}", :production => prod)
+            @recovering = false
+          end
+          progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
         else
-          progress("#{prod}(:token)", "", :depth => (depth + 1)) {token.to_s}
+          progress("#{prod}(:terminal)", "", :depth => (depth + 1)) {token.to_s}
         end
       else
-        error("#{parentProd}(:token)", "Token has no parent production", :production => prod)
+        error("#{parentProd}(:terminal)", "Terminal has no parent production", :production => prod)
       end
     end
-    
-    # Skip through the input stream until something is found that
-    # is either valid based on the content of the production stack,
-    # or can follow a production in the stack.
+
+    ##
+    # Does first include the specified token
     #
-    # @return [Token]
-    def skip_until_valid(todo_stack)
-      cur_prod = todo_stack.last[:prod]
-      token = get_token
-      first = @first[cur_prod] || []
-      expected = @branch.fetch(cur_prod, {}).keys
-      expected << :_eps if first.include?(:_eps)  # Helps when testing
-
-      # If we've reached EOF, token is nil. This is fine, if _eof is in @follow
-      return if token.nil? && @follow.fetch(cur_prod, []).include?(:_eof)
-
-      # If this token can be used by the top production, return it
-      # Otherwise, if the banch table allows empty, also return the token
-      return token if !@recovering && (expected.any? {|t| (token || :_eps) === t})
-
-      # Otherwise, it's an error condition, and skip either until
-      # we find a valid token for this production, or until we find
-      # something that can follow this production
-      error("skip_until_valid", "expected one of #{expected.map(&:inspect).join(", ")}, found #{token.inspect}",
-        :production => cur_prod, :token => token)
-
-      debug("recovery", "stack follows:")
-      todo_stack.reverse.each do |todo|
-        debug("recovery") {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
+    # @param [Symbol] production
+    # @param [Symbol, Lexer::Token] token
+    #   A terminal, or symbol or string
+    # @return [Boolean]
+    def first_include?(production, token)
+      if token.is_a?(Lexer::Token)
+        @first.fetch(production, []).any? {|t| token === t}
+      else
+        @first.fetch(production, []).include?(token)
       end
+    end
 
-      # Find all follows to the top of the stack
-      follows = todo_stack.inject([]) do |follow, todo|
-        prod = todo[:prod]
-        follow += @follow[prod] || []
-      end.uniq
-      debug("recovery") {"follows: #{follows.inspect}"}
-
-      # Skip tokens until one is found in first or follows
-      while (token = get_token) && (first + follows).none? {|t| token === t}
-        skipped = @lexer.shift
-        progress("recovery") {"skip #{skipped.inspect}"}
-      end
-      debug("recovery") {"found #{token.inspect} in #{first.include?(token) ? 'first' : 'follows'}"}
-      
-      # If the token is a first, just return it. Otherwise, it is a follow
-      # and we need to skip to the end of the production
-      unless first.any? {|t| token == t} || todo_stack.last[:terms].empty?
-        debug("recovery") {"token in follows, skip past #{todo_stack.last[:terms].inspect}"}
-        todo_stack.last[:terms] = [] 
+    ##
+    # Does follow include the specified terminal
+    #
+    # @param [Symbol] production
+    # @param [Symbol, Lexer::Token] token
+    #   A terminal, or symbol or string
+    # @return [Boolean]
+    def follow_include?(production, token)
+      if token.is_a?(Lexer::Token)
+        @follow.fetch(production, []).any? {|t| token === t}
+      else
+        @follow.fetch(production, []).include?(token)
       end
-      token
     end
 
     ##
-    # Return the next token, entering error recovery if the token is invalid
+    # Return the next token, raising an error if the token is invalid
     #
+    # @param [:recover] recover
+    #   Recover from errors and go until next valid token or end of file
     # @return [Token]
-    def get_token
-      token = begin
-        @lexer.first
-      rescue EBNF::LL1::Lexer::Error => e
-        # Recover from lexer error
-        @lineno = e.lineno
-        error("get_token", "With input '#{e.input}': #{e.message}",
-              :production => @productions.last)
-
-        # Retrieve next valid token
-        t = @lexer.recover
-        debug("get_token", :level => 2) {"skipped to #{t.inspect}"}
-        t
-      end
+    # @raise [Lexer::Error]
+    def get_token(recover = nil)
+      token = @lexer.first
       #progress("token") {token.inspect}
       @lineno = token.lineno if token
       token
-    end
-
-    ##
-    # @param [String] node Relevant location associated with message
-    # @param [String] message Error string
-    # @param [Hash] options
-    # @option options [URI, #to_s] :production
-    # @option options [Token] :token
-    def error(node, message, options = {})
-      message += ", found #{options[:token].representation.inspect}" if options[:token]
-      message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production]
-      @error_log << message unless @recovering
-      @recovering = true
-      debug(node, message, options.merge(:level => 0))
-    end
-
-    ##
-    # Progress output when parsing
-    #   param [String] node Relevant location associated with message
-    #   param [String] message ("")
-    #   param [Hash] options
-    #   option options [Integer] :depth
-    #     Recursion depth for indenting output
-    #   yieldreturn [String] added to message
-    def progress(node, *args)
-      return unless @options[:progress] || @options[:debug]
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      message = args.join(",")
-      depth = options[:depth] || self.depth
-      message += yield.to_s if block_given?
-      debug(node, message, options.merge(:level => 1))
-    end
-
-    ##
-    # Progress output when debugging.
-    # Captures output to `@options[:debug]` if it is an array.
-    # Otherwise, if `@options[:debug]` is set, or
-    # `@options[:progress]` is set and `:level` <= 1, or
-    # `@options[:validate]` is set and `:level` == 0 output
-    # to standard error.
-    # 
-    # @overload debug(node, message)
-    #   @param [String] node Relevant location associated with message
-    #   @param [String] message ("")
-    #   @param [Hash] options
-    #   @option options [Integer] :depth
-    #     Recursion depth for indenting output
-    #   @option options [Integer] :level
-    #     Debug level, `0` for errors, `1` for progress, anything else
-    #     for debug output.
-    #
-    # @overload debug(message)
-    #   @param [String] node Relevant location associated with message
-    #   @param [Hash] options
-    #   @option options [Integer] :depth
-    #     Recursion depth for indenting output
-    #   @option options [Integer] :level
-    #     Debug level, `0` for errors, `1` for progress, anything else
-    #     for debug output.
-    # @yieldreturn [String] added to message
-    def debug(*args)
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      debug_level = options.fetch(:level, 2)
-      return unless @options[:debug]  && debug_level <= DEBUG_LEVEL ||
-                    @options[:progress] && debug_level <= 1 ||
-                    @options[:validate] && debug_level == 0
-      depth = options[:depth] || self.depth
-      d_str = depth > 20 ? ' ' * 20 + '+' : ' ' * depth
-      args << yield if block_given?
-      message = "#{args.join(': ')}"
-      str = "[#{@lineno}](#{debug_level})#{d_str}#{message}"
-      @options[:debug] << str if @options[:debug].is_a?(Array)
-      case
-      when @options[:yield]
-        @parse_callback.call(:trace, node, message, options)
-      when @options[:debug] == true
-        $stderr.puts str
-      when @options[:progress] && debug_level <= 1
-        $stderr.puts str
-      when @options[:validate] && debug_level == 0
-        $stderr.puts str
+    rescue Lexer::Error => e
+      if recover
+        # Recover from lexer error so that we can not bail out too early
+        @lexer.recover
+        error("get_token", "With input '#{e.input}': #{e.message}}")
+        retry
       end
+      raise
     end
 
     ##
     # Accept the first token in the input stream if it matches
-    # _type\_or\_value_. Return nil otherwise.
+    # `type\_or\_value`. Raise Error, otherwise.
     #
     # @param  [Symbol, String] type_or_value
     # @return [Token]
+    # @raise [Error, Lexer::Error]
     def accept(type_or_value)
       if (token = get_token) && token === type_or_value
         debug("accept") {"#{token.inspect} === #{type_or_value.inspect}"}

data/lib/ebnf/ll1/scanner.rb

@@ -13,7 +13,6 @@ module EBNF::LL1
     LOW_WATER  = 2048     # Hopefully large enough to deal with long multi-line comments
 
     ##
-    # @!attribute [r] input
     # @return [IO, StringIO]
     attr_reader :input
 

data/lib/ebnf/rule.rb

@@ -10,40 +10,47 @@ module EBNF
       diff hex range
     }.map(&:to_sym).freeze
 
-    # @!attribute [rw] sym for rule
+    # Symbol of rule
+    #
     # @return [Symbol]
     attr_accessor :sym
 
-    # @!attribute [rw] id of rule
+    # ID of rule
     # @return [String]
     attr_accessor :id
 
     # A comprehension is a sequence which contains all elements but the first of the original rule.
-    # @!attribute [rw] comprehension of this rule
+    #
     # @return [Rule]
     attr_accessor :comp
 
-    # @!attribute [rw] kind of rule
+    # Kind of rule
+    #
     # @return [:rule, :terminal, or :pass]
     attr_accessor :kind
 
-    # @!attribute [rw] expr rule expression
+    # Rule expression
+    #
     # @return [Array]
     attr_accessor :expr
 
-    # @!attribute [rw] orig original rule
+    # Original EBNF
+    #
     # @return [String]
     attr_accessor :orig
 
-    # @!attribute [r] first terminals that immediately procede this rule
+    # Terminals that immediately procede this rule
+    #
     # @return [Array<Rule>]
     attr_reader :first
 
-    # @!attribute [r] follow terminals that immediately follow this rule
+    # Terminals that immediately follow this rule
+    #
     # @return [Array<Rule>]
     attr_reader :follow
 
-    # @!attribute [rw] start indicates that this is a starting rule
+    # Indicates that this is a starting rule
+    #
     # @return [Boolean]
     attr_accessor :start
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Gregg Kellogg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp