rdf-turtle 1.0.0 → 1.0.2

data/lib/rdf/ll1/parser.rb

@@ -1,541 +0,0 @@
-require 'rdf'
-require 'rdf/ll1/lexer'
-
-module RDF::LL1
-  ##
-  # A Generic LL1 parser using a lexer and branch tables defined using the SWAP tool chain (modified).
-  module Parser
-    ##
-    # @private
-    # level above which debug messages are supressed
-    DEBUG_LEVEL = 10
-
-    ##
-    # @!attribute [r] lineno
-    # @return [Integer] line number of current token
-    attr_reader :lineno
-
-    def self.included(base)
-      base.extend(ClassMethods)
-    end
-
-    # DSL for creating terminals and productions
-    module ClassMethods
-      def production_handlers; @@production_handlers || {}; end
-      def terminal_handlers; @@terminal_handlers || {}; end
-      def patterns; @@patterns || []; end
-      def unescape_terms; @@unescape_terms || []; end
-
-      ##
-      # Defines a production called during different phases of parsing
-      # with data from previous production along with data defined for the
-      # current production
-      #
-      # @param [Symbol] term
-      #   Term which is a key in the branch table
-      # @yield [reader, phase, input, current]
-      # @yieldparam [RDF::Reader] reader
-      #   Reader instance
-      # @yieldparam [Symbol] phase
-      #   Phase of parsing, one of :start, or :finish
-      # @yieldparam [Hash] input
-      #   A Hash containing input from the parent production
-      # @yieldparam [Hash] current
-      #   A Hash defined for the current production, during :start
-      #   may be initialized with data to pass to further productions,
-      #   during :finish, it contains data placed by earlier productions
-      # @yieldparam [Prod] block
-      #   Block passed to initialization for yielding to calling reader.
-      #   Should conform to the yield specs for #initialize
-      # Yield to generate a triple
-      def production(term, &block)
-        @@production_handlers ||= {}
-        @@production_handlers[term] = block
-      end
-
-      ##
-      # Defines the pattern for a terminal node and a block to be invoked
-      # when ther terminal is encountered. If the block is missing, the
-      # value of the terminal will be placed on the input hash to be returned
-      # to a previous production.
-      #
-      # @param [Symbol, String] term
-      #   Defines a terminal production, which appears as within a sequence in the branch table
-      # @param [Regexp] regexp
-      #   Pattern used to scan for this terminal
-      # @param [Hash] options
-      # @option options [Boolean] :unescape
-      #   Cause strings and codepoints to be unescaped.
-      # @yield [reader, term, token, input]
-      # @yieldparam [RDF::Reader] reader
-      #   Reader instance
-      # @yieldparam [Symbol] term
-      #   A symbol indicating the production which referenced this terminal
-      # @yieldparam [String] token
-      #   The scanned token
-      # @yieldparam [Hash] input
-      #   A Hash containing input from the parent production
-      # @yieldparam [Prod] block
-      #   Block passed to initialization for yielding to calling reader.
-      #   Should conform to the yield specs for #initialize
-      def terminal(term, regexp, options = {}, &block)
-        @@patterns ||= []
-        @@patterns << [term, regexp]  # Passed in order to define evaulation sequence
-        @@terminal_handlers ||= {}
-        @@terminal_handlers[term] = block if block_given?
-        @@unescape_terms ||= []
-        @@unescape_terms << term if options[:unescape]
-      end
-    end
-
-    ##
-    # Initializes a new parser instance.
-    #
-    # Attempts to recover from errors.
-    #
-    # @example
-    #   require 'rdf/ll1/parser'
-    #   
-    #   class Reader << RDF::Reader
-    #     include RDF::LL1::Parser
-    #     
-    #     branch      RDF::Turtle::Reader::BRANCH
-    #     
-    #     ##
-    #     # Defines a production called during different phases of parsing
-    #     # with data from previous production along with data defined for the
-    #     # current production
-    #     #
-    #     # Yield to generate a triple
-    #     production :object do |reader, phase, input, current|
-    #       object = current[:resource]
-    #       yield :statement, RDF::Statement.new(input[:subject], input[:predicate], object)
-    #     end
-    #     
-    #     ##
-    #     # Defines the pattern for a terminal node
-    #     terminal :BLANK_NODE_LABEL, %r(_:(#{PN_LOCAL})) do |reader, production, token, input|
-    #       input[:BLANK_NODE_LABEL] = RDF::Node.new(token)
-    #     end
-    #     
-    #     ##
-    #     # Iterates the given block for each RDF statement in the input.
-    #     #
-    #     # @yield  [statement]
-    #     # @yieldparam [RDF::Statement] statement
-    #     # @return [void]
-    #     def each_statement(&block)
-    #       @callback = block
-    #   
-    #       parse(START.to_sym) do |context, *data|
-    #         case context
-    #         when :statement
-    #           yield *data
-    #         end
-    #       end
-    #     end
-    #     
-    #   end
-    #
-    # @param  [String, #to_s]          input
-    # @param [Symbol, #to_s] prod The starting production for the parser.
-    #   It may be a URI from the grammar, or a symbol representing the local_name portion of the grammar URI.
-    # @param  [Hash{Symbol => Object}] options
-    # @option options [Hash{Symbol,String => Hash{Symbol,String => Array<Symbol,String>}}] :branch
-    #   LL1 branch table.
-    # @option options [HHash{Symbol,String => Array<Symbol,String>}] :first ({})
-    #   Lists valid terminals that can precede each production (for error recovery).
-    # @option options [HHash{Symbol,String => Array<Symbol,String>}] :follow ({})
-    #   Lists valid terminals that can follow each production (for error recovery).
-    # @option options [Boolean]  :validate     (false)
-    #   whether to validate the parsed statements and values. If not validating,
-    #   the parser will attempt to recover from errors.
-    # @option options [Boolean] :progress
-    #   Show progress of parser productions
-    # @option options [Boolean] :debug
-    #   Detailed debug output
-    # @yield [context, *data]
-    #   Yields for to return data to reader
-    # @yieldparam [:statement, :trace] context
-    #   Context for block
-    # @yieldparam [Symbol] *data
-    #   Data specific to the call
-    # @return [RDF::LL1::Parser]
-    # @see http://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
-    def parse(input = nil, prod = nil, options = {}, &block)
-      @options = options.dup
-      @branch  = options[:branch]
-      @first  = options[:first] ||= {}
-      @follow  = options[:follow] ||= {}
-      @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, @options.merge(:unescape_terms => self.class.unescape_terms))
-      @productions = []
-      @parse_callback = block
-      @recovering = false
-      @error_log = []
-      terminals = self.class.patterns.map(&:first)  # Get defined terminals to help with branching
-
-      # Unrecoverable errors
-      raise Error, "Branch table not defined" unless @branch && @branch.length > 0
-      raise Error, "Starting production not defined" unless prod
-
-      @prod_data = [{}]
-      prod = RDF::URI(prod).fragment.to_sym unless prod.is_a?(Symbol)
-      todo_stack = [{:prod => prod, :terms => nil}]
-
-      while !todo_stack.empty?
-        pushed = false
-        if todo_stack.last[:terms].nil?
-          todo_stack.last[:terms] = []
-          cur_prod = todo_stack.last[:prod]
-
-          # Get this first valid token appropriate for the stacked productions,
-          # skipping invalid tokens until either a valid token is found (from @first),
-          # or a token appearing in @follow appears.
-          token = skip_until_valid(todo_stack)
-          
-          # At this point, token is either nil, in the first set of the production,
-          # or in the follow set of this production or any previous production
-          debug("parse(production)") do
-            "token #{token ? token.representation.inspect : 'nil'}, " + 
-            "prod #{cur_prod.inspect}, " + 
-            "depth #{depth}"
-          end
-          
-          # Got an opened production
-          onStart(cur_prod)
-          break if token.nil?
-          
-          if prod_branch = @branch[cur_prod]
-            @recovering = false
-            sequence = prod_branch[token.representation]
-            debug("parse(production)", :level => 2) do
-              "token #{token.representation.inspect} " +
-              "prod #{cur_prod.inspect}, " + 
-              "prod_branch #{prod_branch.keys.inspect}, " +
-              "sequence #{sequence.inspect}"
-            end
-
-            if sequence.nil?
-              if prod_branch.has_key?(:"ebnf:empty")
-                debug("parse(production)", :level => 2) {"empty sequence for ebnf:empty"}
-              else
-                # If there is no sequence for this production, we're
-                # in error recovery, and _token_ has been advanced to
-                # the point where it can reasonably follow this production
-              end
-            end
-            todo_stack.last[:terms] += sequence if sequence
-          else
-            # Is this a fatal error?
-            error("parse(fatal?)", "No branches found for #{cur_prod.inspect}",
-              :production => cur_prod, :token => token)
-          end
-        end
-        
-        debug("parse(terms)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}"}
-        while !todo_stack.last[:terms].to_a.empty?
-          begin
-            # Get the next term in this sequence
-            term = todo_stack.last[:terms].shift
-            debug("parse(token)") {"accept #{term.inspect}"}
-            if token = accept(term)
-              @recovering = false
-              debug("parse(token)") {"token #{token.inspect}, term #{term.inspect}"}
-              onToken(term, token)
-            elsif terminals.include?(term)
-              # If term is a terminal, then it is an error of token does not
-              # match it
-              skip_until_valid(todo_stack)
-            else
-              # If it's not a string (a symbol), it is a non-terminal and we push the new state
-              todo_stack << {:prod => term, :terms => nil}
-              debug("parse(push)", :level => 2) {"term #{term.inspect}, depth #{depth}"}
-              pushed = true
-              break
-            end
-          end
-        end
-        
-        # After completing the last production in a sequence, pop down until we find a production
-        #
-        # If in recovery mode, continue popping until we find a term with a follow list
-        while !pushed &&
-              !todo_stack.empty? &&
-              ( todo_stack.last[:terms].to_a.empty? ||
-                (@recovering && @follow[todo_stack.last[:term]].nil?))
-          debug("parse(pop)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}, recovering? #{@recovering.inspect}"}
-          prod = todo_stack.last[:prod]
-          @recovering = false if @follow[prod] # Stop recovering when we might have a match
-          todo_stack.pop
-          onFinish
-        end
-      end
-
-      error("parse(eof)", "Finished processing before end of file", :token => @lexer.first) if @lexer.first
-
-      # Continue popping contexts off of the stack
-      while !todo_stack.empty?
-        debug("parse(eof)", :level => 2) {"stack #{todo_stack.last.inspect}, depth #{depth}"}
-        if todo_stack.last[:terms].length > 0
-          error("parse(eof)",
-            "End of input before end of production: stack #{todo_stack.last.inspect}, depth #{depth}"
-          )
-        end
-        todo_stack.pop
-        onFinish
-      end
-      
-      # When all is said and done, raise the error log
-      unless @error_log.empty?
-        raise Error, @error_log.join("\n\t") 
-      end
-    end
-
-    def depth; (@productions || []).length; end
-
-  private
-    # Start for production
-    def onStart(prod)
-      handler = self.class.production_handlers[prod]
-      @productions << prod
-      if handler
-        # Create a new production data element, potentially allowing handler
-        # to customize before pushing on the @prod_data stack
-        progress("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
-        data = {}
-        handler.call(self, :start, @prod_data.last, data, @parse_callback)
-        @prod_data << data
-      else
-        progress("#{prod}(:start)") { get_token.inspect}
-      end
-      #puts @prod_data.inspect
-    end
-
-    # Finish of production
-    def onFinish
-      prod = @productions.last
-      handler = self.class.production_handlers[prod]
-      if handler
-        # Pop production data element from stack, potentially allowing handler to use it
-        data = @prod_data.pop
-        handler.call(self, :finish, @prod_data.last, data, @parse_callback)
-        progress("#{prod}(:finish):#{@prod_data.length}") {@prod_data.last}
-      else
-        progress("#{prod}(:finish)", '')
-      end
-      @productions.pop
-    end
-
-    # A token
-    def onToken(prod, token)
-      unless @productions.empty?
-        parentProd = @productions.last
-        handler = self.class.terminal_handlers[prod]
-        # Allows catch-all for simple string terminals
-        handler ||= self.class.terminal_handlers[nil] if prod.is_a?(String)
-        if handler
-          handler.call(self, parentProd, token, @prod_data.last)
-          progress("#{prod}(:token)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
-        else
-          progress("#{prod}(:token)", "", :depth => (depth + 1)) {token.to_s}
-        end
-      else
-        error("#{parentProd}(:token)", "Token has no parent production", :production => prod)
-      end
-    end
-    
-    # Skip through the input stream until something is found that
-    # is either valid based on the content of the production stack,
-    # or can follow a production in the stack.
-    #
-    # @return [Token]
-    def skip_until_valid(todo_stack)
-      cur_prod = todo_stack.last[:prod]
-      token = get_token
-      first = @first[cur_prod] || []
-      
-      # If this token can be used by the top production, return it
-      # Otherwise, if the banch table allows empty, also return the token
-      return token if !@recovering && (
-        (@branch[cur_prod] && @branch[cur_prod].has_key?(:"ebnf:empty")) ||
-        first.any? {|t| token === t})
-      
-      # Otherwise, it's an error condition, and skip either until
-      # we find a valid token for this production, or until we find
-      # something that can follow this production
-      expected = first.map {|v| v.inspect}.join(", ")
-      error("skip_until_valid", "expected one of #{expected}",
-        :production => cur_prod, :token => token)
-
-      debug("recovery", "stack follows:")
-      todo_stack.reverse.each do |todo|
-        debug("recovery") {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
-      end
-
-      # Find all follows to the top of the stack
-      follows = todo_stack.inject([]) do |follow, todo|
-        prod = todo[:prod]
-        follow += @follow[prod] || []
-      end.uniq
-      debug("recovery") {"follows: #{follows.inspect}"}
-
-      # Skip tokens until one is found in first or follows
-      while (token = get_token) && (first + follows).none? {|t| token === t}
-        skipped = @lexer.shift
-        progress("recovery") {"skip #{skipped.inspect}"}
-      end
-      debug("recovery") {"found #{token.inspect}"}
-      
-      # If the token is a first, just return it. Otherwise, it is a follow
-      # and we need to skip to the end of the production
-      unless first.any? {|t| token == t} || todo_stack.last[:terms].empty?
-        debug("recovery") {"token in follows, skip past #{todo_stack.last[:terms].inspect}"}
-        todo_stack.last[:terms] = [] 
-      end
-      token
-    end
-
-    ##
-    # @param [String] node Relevant location associated with message
-    # @param [String] message Error string
-    # @param [Hash] options
-    # @option options [URI, #to_s] :production
-    # @option options [Token] :token
-    def error(node, message, options = {})
-      message += ", found #{options[:token].representation.inspect}" if options[:token]
-      message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production] && @options[:debug]
-      @error_log << message unless @recovering
-      @recovering = true
-      debug(node, message, options.merge(:level => 0))
-    end
-
-    ##
-    # Return the next token, entering error recovery if the token is invalid
-    #
-    # @return [Token]
-    def get_token
-      token = begin
-        @lexer.first
-      rescue RDF::LL1::Lexer::Error => e
-        # Recover from lexer error
-        @lineno = e.lineno
-        error("get_token", "With input '#{e.input}': #{e.message}",
-              :production => @productions.last)
-
-        # Retrieve next valid token
-        t = @lexer.recover
-        debug("get_token", :level => 2) {"skipped to #{t.inspect}"}
-        t
-      end
-      #progress("token") {token.inspect}
-      @lineno = token.lineno if token
-      token
-    end
-
-    ##
-    # Progress output when parsing
-    #   param [String] node Relevant location associated with message
-    #   param [String] message ("")
-    #   param [Hash] options
-    #   option options [Integer] :depth
-    #     Recursion depth for indenting output
-    #   yieldreturn [String] added to message
-    def progress(node, *args)
-      return unless @options[:progress] || @options[:debug]
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      message = args.join(",")
-      depth = options[:depth] || self.depth
-      message += yield.to_s if block_given?
-      if @options[:debug]
-        return debug(node, message, {:level => 0}.merge(options))
-      else
-        str = "[#{@lineno}]#{' ' * depth}#{node}: #{message}"
-        $stderr.puts("[#{@lineno}]#{' ' * depth}#{node}: #{message}")
-      end
-    end
-
-    ##
-    # Progress output when debugging
-    # @param [String] node Relevant location associated with message
-    # @param [String] message ("")
-    # @param [Hash] options
-    # @option options [Integer] :depth
-    #   Recursion depth for indenting output
-    # @yieldreturn [String] added to message
-    def debug(node, message = "", options = {})
-      return unless @options[:debug]
-      debug_level = options.fetch(:level, 1)
-      return unless debug_level <= DEBUG_LEVEL
-      depth = options[:depth] || self.depth
-      message += yield if block_given?
-      str = "[#{@lineno}](#{debug_level})#{' ' * depth}#{node}: #{message}"
-      case @options[:debug]
-      when Array
-        @options[:debug] << str
-      when TrueClass
-        $stderr.puts str
-      when :yield
-        @parse_callback.call(:trace, node, message, options)
-      end
-    end
-
-    ##
-    # Accept the first token in the input stream if it matches
-    # _type\_or\_value_. Return nil otherwise.
-    #
-    # @param  [Symbol, String] type_or_value
-    # @return [Token]
-    def accept(type_or_value)
-      if (token = get_token) && token === type_or_value
-        debug("accept") {"#{token.inspect} === #{type_or_value.inspect}"}
-        @lexer.shift
-      end
-    end
-  public
-
-    ##
-    # Raised for errors during parsing.
-    #
-    # @example Raising a parser error
-    #   raise Error.new(
-    #     "invalid token '%' on line 10",
-    #     :token => '%', :lineno => 9, :production => :turtleDoc)
-    #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
-    class Error < StandardError
-      ##
-      # The current production.
-      #
-      # @return [Symbol]
-      attr_reader :production
-
-      ##
-      # The invalid token which triggered the error.
-      #
-      # @return [String]
-      attr_reader :token
-
-      ##
-      # The line number where the error occurred.
-      #
-      # @return [Integer]
-      attr_reader :lineno
-
-      ##
-      # Initializes a new lexer error instance.
-      #
-      # @param  [String, #to_s]          message
-      # @param  [Hash{Symbol => Object}] options
-      # @option options [Symbol]         :production  (nil)
-      # @option options [String]         :token  (nil)
-      # @option options [Integer]        :lineno (nil)
-      def initialize(message, options = {})
-        @production = options[:production]
-        @token      = options[:token]
-        @lineno     = options[:lineno]
-        super(message.to_s)
-      end
-    end # class Error
-  end # class Reader
-end # module RDF::Turtle