ebnf 1.1.2 → 2.1.1

data/lib/ebnf/ll1/lexer.rb

@@ -29,7 +29,7 @@ module EBNF::LL1
   #     warn error.inspect
   #   end
   #
-  # @see http://en.wikipedia.org/wiki/Lexical_analysis
+  # @see https://en.wikipedia.org/wiki/Lexical_analysis
   class Lexer
     include Enumerable
 
@@ -43,10 +43,10 @@ module EBNF::LL1
       "\\'"  => '\'',   # \u0027 (apostrophe-quote, single quote mark)
       '\\\\' => '\\'    # \u005C (backslash)
     }.freeze
-    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/.freeze    # \uXXXX
-    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/.freeze    # \UXXXXXXXX
-    ECHAR               = /\\./                               # More liberal unescaping
-    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/.freeze
+    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/u.freeze    # \uXXXX
+    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/u.freeze    # \UXXXXXXXX
+    ECHAR               = /\\./u.freeze                        # More liberal unescaping
+    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/n.freeze
 
     ##
     # @return [Regexp] defines whitespace, including comments, otherwise whitespace must be explicit in terminals
@@ -59,7 +59,7 @@ module EBNF::LL1
     #
     # @param  [String] string
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#codepointEscape
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#codepointEscape
     def self.unescape_codepoints(string)
       string = string.dup
       string.force_encoding(Encoding::ASCII_8BIT) if string.respond_to?(:force_encoding)
@@ -81,7 +81,7 @@ module EBNF::LL1
     #
     # @param  [String] input
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
     def self.unescape_string(input)
       input.gsub(ECHAR) { |escaped| ESCAPE_CHARS[escaped] || escaped[1..-1]}
     end
@@ -98,8 +98,8 @@ module EBNF::LL1
     # @yieldparam [Lexer] lexer
     # @return [Lexer]
     # @raise  [Lexer::Error] on invalid input
-    def self.tokenize(input, terminals, options = {}, &block)
-      lexer = self.new(input, terminals, options)
+    def self.tokenize(input, terminals, **options, &block)
+      lexer = self.new(input, terminals, **options)
       block_given? ? block.call(lexer) : lexer
     end
 
@@ -115,17 +115,23 @@ module EBNF::LL1
     #   Whitespace between tokens, including comments
     # @option options[Integer] :high_water passed to scanner
     # @option options[Integer] :low_water passed to scanner
-    def initialize(input = nil, terminals = nil, options = {})
+    def initialize(input = nil, terminals = nil, **options)
       @options        = options.dup
       @whitespace     = @options[:whitespace]
       @terminals      = terminals.map do |term|
-        term.is_a?(Array) ? Terminal.new(*term) : term
+        if term.is_a?(Array) && term.length ==3
+          # Last element is options
+          Terminal.new(term[0], term[1], **term[2])
+        elsif term.is_a?(Array)
+          Terminal.new(*term)
+        else
+          term
+        end
       end
 
       raise Error, "Terminal patterns not defined" unless @terminals && @terminals.length > 0
 
-      @lineno = 1
-      @scanner = Scanner.new(input, options)
+      @scanner = Scanner.new(input, **options)
     end
 
     ##
@@ -140,12 +146,6 @@ module EBNF::LL1
     # @return [String]
     attr_accessor :input
 
-    ##
-    # The current line number (zero-based).
-    #
-    # @return [Integer]
-    attr_reader   :lineno
-
     ##
     # Returns `true` if the input string is lexically valid.
     #
@@ -187,7 +187,7 @@ module EBNF::LL1
 
       @first ||= begin
         {} while !scanner.eos? && skip_whitespace
-        return @scanner = nil if scanner.eos?
+        return nil if scanner.eos?
 
         token = match_token(*types)
 
@@ -226,7 +226,7 @@ module EBNF::LL1
     # @return [Token]
     def recover(*types)
        until scanner.eos? || tok = match_token(*types)
-        if scanner.skip_until(@whitespace || /\s/m).nil? # Skip past current "token"
+        if scanner.skip_until(@whitespace || /\s+/m).nil? # Skip past current "token"
           # No whitespace at the end, must be and end of string
           scanner.terminate
         else
@@ -236,6 +236,14 @@ module EBNF::LL1
       scanner.unscan if tok
       first
     end
+
+    ##
+    # The current line number (one-based).
+    #
+    # @return [Integer]
+    def lineno
+      scanner.lineno
+    end
   protected
 
     # @return [StringScanner]
@@ -246,9 +254,7 @@ module EBNF::LL1
     def skip_whitespace
       # skip all white space, but keep track of the current line number
       while @whitespace && !scanner.eos?
-        if matched = scanner.scan(@whitespace)
-          @lineno += matched.count("\n")
-        else
+        unless scanner.scan(@whitespace)
           return
         end
       end
@@ -274,7 +280,6 @@ module EBNF::LL1
         if matched = scanner.scan(term.regexp)
           #STDERR.puts "  matched #{term.type.inspect}: #{matched.inspect}"
           tok = token(term.type, term.canonicalize(matched))
-          @lineno += matched.count("\n")
           return tok
         end
       end
@@ -300,7 +305,7 @@ module EBNF::LL1
       #   Cause strings and codepoints to be unescaped.
       # @option options [Regexp] :partial_regexp
       #   A regular expression matching the beginning of this terminal; useful for terminals that match things longer than the scanner low water mark.
-      def initialize(type, regexp, options = {})
+      def initialize(type, regexp, **options)
         @type, @regexp, @options = type, regexp, options
         @partial_regexp = options[:partial_regexp]
         @map = options.fetch(:map, {})
@@ -353,8 +358,8 @@ module EBNF::LL1
     #   Scanner instance with access to matched groups
     # @param  [Hash{Symbol => Object}] options
     # @return [Token]
-    def token(type, value, options = {})
-      Token.new(type, value, options.merge(lineno: lineno))
+    def token(type, value, **options)
+      Token.new(type, value, lineno: lineno, **options)
     end
 
     ##
@@ -365,7 +370,7 @@ module EBNF::LL1
     #   token.type   #=> :LANGTAG
     #   token.value  #=> "en"
     #
-    # @see http://en.wikipedia.org/wiki/Lexical_analysis#Token
+    # @see https://en.wikipedia.org/wiki/Lexical_analysis#Token
     class Token
       ##
       # The token's symbol type.
@@ -398,7 +403,7 @@ module EBNF::LL1
       # @param  [String]                 value
       # @param  [Hash{Symbol => Object}] options
       # @option options [Integer]        :lineno (nil)
-      def initialize(type, value, options = {})
+      def initialize(type, value, **options)
         @type = type.to_s.to_sym if type
         @value = value.to_s
         @options = options.dup
@@ -486,7 +491,7 @@ module EBNF::LL1
     #     "invalid token '%' on line 10",
     #     input: query, token: '%', lineno: 9)
     #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
+    # @see https://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError
       ##
       # The input string associated with the error.
@@ -514,7 +519,7 @@ module EBNF::LL1
       # @option options [String]         :input  (nil)
       # @option options [String]         :token  (nil)
       # @option options [Integer]        :lineno (nil)
-      def initialize(message, options = {})
+      def initialize(message, **options)
         @input  = options[:input]
         @token  = options[:token]
         @lineno = options[:lineno]

data/lib/ebnf/ll1/parser.rb

@@ -3,12 +3,52 @@ require 'ebnf/ll1/lexer'
 module EBNF::LL1
   ##
   # A Generic LL1 parser using a lexer and branch tables defined using the SWAP tool chain (modified).
+  #
+  #  # Creating terminal definitions and parser rules to parse generated grammars
+  #
+  #  The parser is initialized to callbacks invoked on entry and exit
+  #  to each `terminal` and `production`. A trivial parser loop can be described as follows:
+  #
+  #      require 'ebnf/ll1/parser'
+  #      require 'meta'
+  #
+  #      class Parser
+  #        include Meta
+  #        include EBNF::LL1::Parser
+  #
+  #        terminal(:SYMBOL, /([a-z]|[A-Z]|[0-9]|_)+/) do |prod, token, input|
+  #          # Add data based on scanned token to input
+  #          input[:symbol] = token.value
+  #        end
+  #
+  #        start_production(:rule) do |input, current, callback|
+  #          # Process on start of production
+  #          # Set state for entry into recursed rules through current
+  #
+  #          # Callback to parser loop with callback
+  #        end
+  #
+  #        production(:rule) do |input, current, callback|
+  #          # Process on end of production
+  #          # return results in input, retrieve results from recursed rules in current
+  #
+  #          # Callback to parser loop with callback
+  #        end
+  #
+  #        def initialize(input)
+  #          parse(input, start_symbol,
+  #            branch: BRANCH,
+  #            first: FIRST,
+  #            follow: FOLLOW,
+  #            cleanup: CLEANUP
+  #          ) do |context, *data|
+  #            # Process calls from callback from productions
+  #
+  #          rescue ArgumentError, RDF::LL1::Parser::Error => e
+  #            progress("Parsing completed with errors:\n\t#{e.message}")
+  #            raise RDF::ReaderError, e.message if validate?
+  #          end
   module Parser
-    ##
-    # @private
-    # level above which debug messages are supressed
-    DEBUG_LEVEL = 10
-
     ##
     # @return [Integer] line number of current token
     attr_reader :lineno
@@ -51,10 +91,10 @@ module EBNF::LL1
       # @yieldparam [Proc] block
       #   Block passed to initialization for yielding to calling parser.
       #   Should conform to the yield specs for #initialize
-      def terminal(term, regexp, options = {}, &block)
+      def terminal(term, regexp, **options, &block)
         @patterns ||= []
         # Passed in order to define evaulation sequence
-        @patterns << EBNF::LL1::Lexer::Terminal.new(term, regexp, options)
+        @patterns << EBNF::LL1::Lexer::Terminal.new(term, regexp, **options)
         @terminal_handlers ||= {}
         @terminal_handlers[term] = block if block_given?
       end
@@ -122,7 +162,14 @@ module EBNF::LL1
 
       def method_missing(method, *args, &block)
         if @delegate ||= nil
-          @delegate.send method, *args, &block
+          # special handling when last arg is **options
+          params = @delegate.method(method).parameters
+          if params.any? {|t, _| t == :keyrest} && args.last.is_a?(Hash)
+            opts = args.pop
+            @delegate.send(method, *args, **opts, &block)
+          else
+            @delegate.send(method, *args, &block)
+          end
         else
           super
         end
@@ -179,7 +226,7 @@ module EBNF::LL1
     #     def each_statement(&block)
     #       @callback = block
     #
-    #       parse(START.to_sym) do |context, *data|
+    #       parse(input, START.to_sym) do |context, *data|
     #         case context
     #         when :statement
     #           yield *data
@@ -198,16 +245,13 @@ module EBNF::LL1
     #   Lists valid terminals that can precede each production (for error recovery).
     # @option options [Hash{Symbol,String => Array<Symbol,String>}] :follow ({})
     #   Lists valid terminals that can follow each production (for error recovery).
-    # @option options [Boolean]  :validate     (false)
-    #   whether to validate the parsed statements and values. If not validating, the parser will attempt to recover from errors.
-    # @option options [Boolean] :progress
-    #   Show progress of parser productions
-    # @option options [Boolean] :debug
-    #   Detailed debug output
-    # @option options [Boolean] :reset_on_start
-    #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
     # @option options[Integer] :high_water passed to lexer
+    # @option options [Logger] :logger for errors/progress/debug.
     # @option options[Integer] :low_water passed to lexer
+    # @option options [Boolean] :reset_on_start
+    #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
+    # @option options [Boolean]  :validate     (false)
+    #   whether to validate the parsed statements and values. If not validating, the parser will attempt to recover from errors.
     # @yield [context, *data]
     #   Yields for to return data to parser
     # @yieldparam [:statement, :trace] context
@@ -218,18 +262,14 @@ module EBNF::LL1
     # @raise [Exception] Raises exceptions for parsing errors
     #   or errors raised during processing callbacks. Internal
     #   errors are raised using {Error}.
-    # @see http://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
-    def parse(input = nil, start = nil, options = {}, &block)
+    # @see https://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
+    def parse(input = nil, start = nil, **options, &block)
       @options = options.dup
-      @options[:debug] ||= case
-      when @options[:progress] then 2
-      when @options[:validate] then 1
-      end
       @branch  = options[:branch]
       @first   = options[:first] ||= {}
       @follow  = options[:follow] ||= {}
       @cleanup = options[:cleanup] ||= {}
-      @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, @options)
+      @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, **@options)
       @productions = []
       @parse_callback = block
       @recovering = false
@@ -349,9 +389,9 @@ module EBNF::LL1
           end
 
           # Get the list of follows for this sequence, this production and the stacked productions.
-          debug("recovery", "stack follows:", level: 4)
+          debug("recovery", "stack follows:")
           todo_stack.reverse.each do |todo|
-            debug("recovery", level: 4) {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
+            debug("recovery") {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
           end
 
           # Find all follows to the top of the stack
@@ -459,15 +499,16 @@ module EBNF::LL1
   protected
 
     ##
-    # Error information, used as level `0` debug messages.
+    # Error information, used as level `3` logger messages.
+    # Messages may be logged and are saved for reporting at end of parsing.
     #
     # @param [String] node Relevant location associated with message
     # @param [String] message Error string
-    # @param [Hash] options
+    # @param [Hash{Symbol => Object}] options
     # @option options [URI, #to_s] :production
     # @option options [Token] :token
-    # @see {#debug}
-    def error(node, message, options = {})
+    # @see #debug
+    def error(node, message, **options)
       lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       m = "ERROR "
       m += "[line: #{lineno}] " if lineno
@@ -476,83 +517,82 @@ module EBNF::LL1
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
       @recovering = true
-      debug(node, m, options.merge(level: 0))
+      debug(node, m, level: options.fetch(:level, 3), **options)
       if options[:raise] || @options[:validate]
         raise Error.new(m, lineno: lineno, token: options[:token], production: options[:production])
       end
     end
 
     ##
-    # Warning information, used as level `1` debug messages.
+    # Warning information, used as level `2` logger messages.
+    # Messages may be logged and are saved for reporting at end of parsing.
     #
     # @param [String] node Relevant location associated with message
     # @param [String] message Error string
     # @param [Hash] options
     # @option options [URI, #to_s] :production
     # @option options [Token] :token
-    # @see {#debug}
-    def warn(node, message, options = {})
+    # @see #debug
+    def warn(node, message, **options)
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       m = "WARNING "
-      m += "[line: #{@lineno}] " if @lineno
+      m += "[line: #{lineno}] " if lineno
       m += message
       m += " (found #{options[:token].inspect})" if options[:token]
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
-      debug(node, m, options.merge(level: 1))
+      debug(node, m, level: 2, lineno: lineno, **options)
     end
 
     ##
-    # Progress output when parsing. Passed as level `2` debug messages.
+    # Progress logged when parsing. Passed as level `1` logger messages.
+    #
+    # The call is ignored, unless `@options[:logger]` is set.
     #
-    # @overload progress(node, message, options)
+    # @overload progress(node, message, **options, &block)
     #   @param [String] node Relevant location associated with message
     #   @param [String] message ("")
     #   @param [Hash] options
     #   @option options [Integer] :depth
     #       Recursion depth for indenting output
-    # @see {#debug}
+    # @see #debug
     def progress(node, *args, &block)
-      return unless @options[:progress] || @options[:debug]
+      return unless @options[:logger]
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       args << {} unless args.last.is_a?(Hash)
-      args.last[:level] ||= 2
+      args.last[:level] ||= 1
+      args.last[:lineno] ||= lineno
       debug(node, *args, &block)
     end
 
     ##
-    # Progress output when debugging.
+    # Debug logging.
     #
-    # The call is ignored, unless `@options[:debug]` is set, in which
-    # case it yields tracing information as indicated. Additionally,
-    # if `@options[:debug]` is an Integer, the call is aborted if the
-    # `:level` option is less than than `:level`.
+    # The call is ignored, unless `@options[:logger]` is set.
     #
-    # @overload debug(node, message, options)
+    # @overload debug(node, message, **options)
     #   @param [Array<String>] args Relevant location associated with message
     #   @param [Hash] options
     #   @option options [Integer] :depth
     #     Recursion depth for indenting output
-    #   @option options [Integer] :level
-    #     Level assigned to message, by convention, level `0` is for
-    #     errors, level `1` is for warnings, level `2` is for parser
-    #     progress information, and anything higher is for various levels
-    #     of debug information.
-    #
-    # @yield trace, level, lineno, depth, args
-    # @yieldparam [:trace] trace
-    # @yieldparam [Integer] level
-    # @yieldparam [Integer] lineno
-    # @yieldparam [Integer] depth Recursive depth of productions
-    # @yieldparam [Array<String>] args
-    # @yieldreturn [String] added to message
-    def debug(*args)
-      return unless @options[:debug] && @parse_callback
+    #   @yieldreturn [String] additional string appended to `message`.
+    def debug(*args, &block)
+      return unless @options[:logger]
       options = args.last.is_a?(Hash) ? args.pop : {}
-      debug_level = options.fetch(:level, 3)
-      return if @options[:debug].is_a?(Integer) && debug_level > @options[:debug]
-
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
+      level = options.fetch(:level, 0)
       depth = options[:depth] || self.depth
-      args << yield if block_given?
-      @parse_callback.call(:trace, debug_level, @lineno, depth, *args)
+
+      if self.respond_to?(:log_debug)
+        level = [:debug, :info, :warn, :error, :fatal][level]
+        log_debug(*args, **options.merge(level: level, lineno: lineno, depth: depth), &block)
+      elsif @options[:logger].respond_to?(:add)
+        args << yield if block_given?
+        @options[:logger].add(level, "[#{lineno}]" + (" " * depth) + args.join(" "))
+      elsif @options[:logger].respond_to?(:<<)
+        args << yield if block_given?
+        @options[:logger] << "[#{lineno}]" + (" " * depth) + args.join(" ")
+      end
     end
 
   private
@@ -563,7 +603,7 @@ module EBNF::LL1
       if handler
         # Create a new production data element, potentially allowing handler
         # to customize before pushing on the @prod_data stack
-        progress("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
+        debug("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
         data = {}
         begin
           self.class.eval_with_binding(self) {
@@ -577,12 +617,12 @@ module EBNF::LL1
       elsif [:merge, :star].include?(@cleanup[prod])
         # Save current data to merge later
         @prod_data << {}
-        progress("#{prod}(:start}:#{@prod_data.length}:cleanup:#{@cleanup[prod]}") { get_token.inspect + (@recovering ? ' recovering' : '')}
+        debug("#{prod}(:start}:#{@prod_data.length}:cleanup:#{@cleanup[prod]}") { get_token.inspect + (@recovering ? ' recovering' : '')}
       else
         # Make sure we push as many was we pop, even if there is no
         # explicit start handler
         @prod_data << {} if self.class.production_handlers[prod]
-        progress("#{prod}(:start:#{@prod_data.length})") { get_token.inspect + (@recovering ? ' recovering' : '')}
+        debug("#{prod}(:start:#{@prod_data.length})") { get_token.inspect + (@recovering ? ' recovering' : '')}
       end
       #puts "prod_data(s): " + @prod_data.inspect
     end
@@ -616,7 +656,7 @@ module EBNF::LL1
           else Array(input[k]) + Array(v)
           end
         end
-        progress("#{prod}(:finish):#{@prod_data.length} cleanup:#{@cleanup[prod]}") {@prod_data.last}
+        debug("#{prod}(:finish):#{@prod_data.length} cleanup:#{@cleanup[prod]}") {@prod_data.last}
       else
         progress("#{prod}(:finish):#{@prod_data.length}") { "recovering" if @recovering }
       end
@@ -723,7 +763,7 @@ module EBNF::LL1
     #     "invalid token '%' on line 10",
     #     token: '%', lineno: 9, production: :turtleDoc)
     #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
+    # @see https://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError
       ##
       # The current production.
@@ -751,7 +791,7 @@ module EBNF::LL1
       # @option options [Symbol]         :production  (nil)
       # @option options [String]         :token  (nil)
       # @option options [Integer]        :lineno (nil)
-      def initialize(message, options = {})
+      def initialize(message, **options)
         @production = options[:production]
         @token      = options[:token]
         @lineno     = options[:lineno] || (@token.lineno if @token.respond_to?(:lineno))