ebnf 1.2.0 → 2.1.3

data/lib/ebnf/isoebnf/meta.rb

@@ -0,0 +1,75 @@
+# This file is automatically generated by ebnf version 2.0.0
+# Derived from etc/iso-ebnf.ebnf
+module ISOEBNFMeta
+  RULES = [
+    EBNF::Rule.new(:syntax, nil, [:star, :syntax_rule]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:syntax_rule, nil, [:seq, :meta_identifier, :defining_symbol, :definitions_list, :terminator_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definitions_list, nil, [:seq, :single_definition, :_definitions_list_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_1, nil, [:star, :_definitions_list_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_2, nil, [:seq, :definition_separator_symbol, :definitions_list]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:single_definition, nil, [:seq, :term, :_single_definition_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_1, nil, [:star, :_single_definition_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_2, nil, [:seq, ",", :term]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:term, nil, [:seq, :factor, :_term_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_1, nil, [:opt, :_term_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_2, nil, [:seq, "-", :exception]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:exception, nil, [:seq, :factor]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:factor, nil, [:seq, :_factor_1, :primary]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_1, nil, [:opt, :_factor_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_2, nil, [:seq, :integer, "*"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:primary, nil, [:alt, :optional_sequence, :repeated_sequence, :special_sequence, :grouped_sequence, :meta_identifier, :terminal_string, :empty]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:optional_sequence, nil, [:seq, :start_option_symbol, :definitions_list, :end_option_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repeated_sequence, nil, [:seq, :start_repeat_symbol, :definitions_list, :end_repeat_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:grouped_sequence, nil, [:seq, "(", :definitions_list, ")"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminals, nil, [:seq], kind: :terminals).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_string, nil, [:alt, :_terminal_string_1, :_terminal_string_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_1, nil, [:seq, "'", :_terminal_string_3, "'"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_3, nil, [:plus, :first_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_2, nil, [:seq, "\"", :_terminal_string_4, "\""]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_4, nil, [:plus, :second_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier, nil, [:seq, :letter, :_meta_identifier_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_meta_identifier_1, nil, [:star, :meta_identifier_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:integer, nil, [:plus, :decimal_digit], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence, nil, [:seq, "?", :_special_sequence_1, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_special_sequence_1, nil, [:star, :special_sequence_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment, nil, [:seq, :start_comment_symbol, :_comment_1, :end_comment_symbol], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_comment_1, nil, [:star, :comment_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment_symbol, nil, [:alt, :comment, :commentless_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:commentless_symbol, nil, [:alt, :terminal_character, :meta_identifier, :integer, :terminal_string, :special_sequence], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:letter, nil, [:range, "a-zA-Z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:decimal_digit, nil, [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier_character, nil, [:alt, :letter, :decimal_digit, "_"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_terminal_character, nil, [:diff, :terminal_character, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_terminal_character, nil, [:diff, :terminal_character, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_character, nil, [:diff, :terminal_character, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_character, nil, [:alt, :letter, :decimal_digit, :concatenate_symbol, :defining_symbol, :definition_separator_symbol, :end_comment_symbol, :end_group_symbol, :end_option_symbol, :end_repeat_symbol, :except_symbol, :first_quote_symbol, :repetition_symbol, :second_quote_symbol, :special_sequence_symbol, :start_comment_symbol, :start_group_symbol, :start_option_symbol, :start_repeat_symbol, :terminator_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:other_character, nil, [:alt, :_other_character_1, "\\"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_other_character_1, nil, [:range, ":+_%@&$<>^` ̃#x20#x23"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_separator, nil, [:range, "#x9#xa#xb#xc#xd#x20"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_pass, nil, [:alt, :__pass_1, :comment], kind: :pass).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:__pass_1, nil, [:plus, :gap_separator]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:empty, nil, [:seq, ""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:concatenate_symbol, nil, [:seq, ","], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repetition_symbol, nil, [:seq, "*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:except_symbol, nil, [:seq, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_quote_symbol, nil, [:seq, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_quote_symbol, nil, [:seq, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_comment_symbol, nil, [:seq, "(*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_comment_symbol, nil, [:seq, "*)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_group_symbol, nil, [:seq, "("], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_group_symbol, nil, [:seq, ")"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_symbol, nil, [:seq, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:defining_symbol, nil, [:alt, "=", ":"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definition_separator_symbol, nil, [:alt, "|", "/", "!"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminator_symbol, nil, [:alt, ";", "."], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_option_symbol, nil, [:seq, "["], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_option_symbol, nil, [:seq, "]"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_repeat_symbol, nil, [:alt, "{", "(:"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_repeat_symbol, nil, [:alt, "}", ":)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_free_symbol, nil, [:alt, :_gap_free_symbol_1, :terminal_string], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_1, nil, [:seq, :_gap_free_symbol_3, :terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_3, nil, [:not, :_gap_free_symbol_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_2, nil, [:range, "'\""], kind: :terminal).extend(EBNF::PEG::Rule),
+  ]
+end
+

data/lib/ebnf/ll1.rb

@@ -1,4 +1,86 @@
 module EBNF
+  ##
+  # This module extends {EBNF::Base} to create metadata including _branch_,  [First/Follow][], and other tables which is used by {EBNF::LL1::Parser} to recognize examples of the associated grammar.
+  #
+  ### Branch Table
+  #
+  #  The Branch table is a hash mapping production rules to a hash relating terminals appearing in input to sequence of productions to follow when the corresponding input terminal is found. This allows either the `seq` primitive, where all terminals map to the same sequence of productions, or the `alt` primitive, where each terminal may map to a different production.
+  #
+  #      BRANCH = {
+  #        :alt => {
+  #          "(" => [:seq, :_alt_1],
+  #          :HEX => [:seq, :_alt_1],
+  #          :O_RANGE => [:seq, :_alt_1],
+  #          :RANGE => [:seq, :_alt_1],
+  #          :STRING1 => [:seq, :_alt_1],
+  #          :STRING2 => [:seq, :_alt_1],
+  #          :SYMBOL => [:seq, :_alt_1],
+  #        },
+  #        ...
+  #        :declaration => {
+  #          "@pass" => [:pass],
+  #          "@terminals" => ["@terminals"],
+  #        },
+  #        ...
+  #      }
+  #
+  #  In this case the `alt` rule is `seq ('|' seq)*` can happen when any of the specified tokens appears on the input stream. The all cause the same token to be passed to the `seq` rule and follow with `_alt_1`, which handles the `('|' seq)*` portion of the rule, after the first sequence is matched.
+  #
+  #  The `declaration` rule is `@terminals' | pass` using the `alt` primitive determining the production to run based on the terminal appearing on the input stream. Eventually, a terminal production is found and the token is consumed.
+  #
+  ### First/Follow Table
+  #
+  #  The [First/Follow][] table is a hash mapping production rules to the terminals that may proceed or follow the rule. For example:
+  #
+  #      FIRST = {
+  #        :alt => [
+  #          :HEX,
+  #          :SYMBOL,
+  #          :RANGE,
+  #          :O_RANGE,
+  #          :STRING1,
+  #          :STRING2,
+  #          "("],
+  #        ...
+  #      }
+  #
+  ### Terminals Table
+  #
+  #  This table is a simple list of the terminal productions found in the grammar. For example:
+  #
+  #      TERMINALS = ["(", ")", "-",
+  #        "@pass", "@terminals",
+  #         :HEX, :LHS, :O_RANGE,:POSTFIX,
+  #        :RANGE, :STRING1, :STRING2, :SYMBOL,"|"
+  #      ].freeze
+  #
+  ### Cleanup Table
+  #
+  #  This table identifies productions which used EBNF rules, which are transformed to BNF for actual parsing. This allows the parser, in some cases, to reproduce *star*, *plus*, and *opt* rule matches. For example:
+  #
+  #      CLEANUP = {
+  #        :_alt_1 => :star,
+  #        :_alt_3 => :merge,
+  #        :_diff_1 => :opt,
+  #        :ebnf => :star,
+  #        :_ebnf_2 => :merge,
+  #        :_postfix_1 => :opt,
+  #        :seq => :plus,
+  #        :_seq_1 => :star,
+  #        :_seq_2 => :merge,
+  #      }.freeze
+  #
+  #  In this case the `ebnf` rule was `(declaration | rule)*`. As BNF does not support a star operator, this is decomposed into a set of rules using `alt` and `seq` primitives:
+  #
+  #      ebnf    ::= _empty _ebnf_2
+  #      _ebnf_1 ::= declaration | rule
+  #      _ebnf_2 ::= _ebnf_1 ebnf
+  #      _ebnf_3 ::= ebnf
+  #
+  #  The `_empty` production matches an empty string, so allows for now value. `_ebnf_2` matches `declaration | rule` (using the `alt` primitive) followed by `ebnf`, creating a sequence of zero or more `declaration` or `alt` members.
+  #
+  # [First/Follow]: https://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
+
   module LL1
     autoload :Lexer,    "ebnf/ll1/lexer"
     autoload :Parser,   "ebnf/ll1/parser"
@@ -51,8 +133,40 @@ module EBNF
     ##
     # Create first/follow for each rule using techniques defined for LL(1) parsers.
     #
+    # This takes rules which have transformed into BNF and adds first/follow and otehr information to the rules to allow the generation of metadata tables used for driving a parser.
+    #
+    # Given an initial rule in EBNF:
+    #
+    #     (rule enbf "1" (star declaration rule))
+    #
+    # The BNF transformation becomes:
+    #
+    #     (rule ebnf "1" (alt _empty _ebnf_2))
+    #     (rule _ebnf_1 "1.1" (alt declaration rule))
+    #     (rule _ebnf_2 "1.2" (seq _ebnf_1 ebnf))
+    #     (rule _ebnf_3 "1.3" (seq ebnf))
+    #
+    # After running this method, the rules are annotated with first/follow and cleanup rules:
+    #
+    #     (rule ebnf "1"
+    #      (start #t)
+    #      (first "@pass" "@terminals" LHS _eps)
+    #      (follow _eof)
+    #      (cleanup star)
+    #      (alt _empty _ebnf_2))
+    #     (rule _ebnf_1 "1.1"
+    #      (first "@pass" "@terminals" LHS)
+    #      (follow "@pass" "@terminals" LHS _eof)
+    #      (alt declaration rule))
+    #     (rule _ebnf_2 "1.2"
+    #      (first "@pass" "@terminals" LHS)
+    #      (follow _eof)
+    #      (cleanup merge)
+    #      (seq _ebnf_1 ebnf))
+    #     (rule _ebnf_3 "1.3" (first "@pass" "@terminals" LHS _eps) (follow _eof) (seq ebnf))
+    #
     # @return [EBNF] self
-    # @see http://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
+    # @see https://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
     # @param [Array<Symbol>] starts
     #   Set of symbols which are start rules
     def first_follow(*starts)
@@ -96,8 +210,9 @@ module EBNF
           firsts, follows = 0, 0
           # add Fi(wi) to Fi(Ai) for every rule Ai → wi
           #
-          # For sequences, this is the first rule in the sequence.
-          # For alts, this is every rule in the sequence
+          # * For sequences, this is the first rule in the sequence.
+          # * For alts, this is every rule in the sequence
+          # * Other rules don't matter, as they don't appear in strict BNF
           each(:rule) do |ai|
             # Fi(a w' ) = { a } for every terminal a
             ai.terminals(ast).each do |t|
@@ -276,6 +391,19 @@ module EBNF
       end
     end
 
+    ##
+    # Output Ruby parser files for LL(1) parsing
+    #
+    # @param [IO, StringIO] output
+    def to_ruby_ll1(output, **options)
+      self.outputTable(output, 'BRANCH', self.branch, 1)
+      self.outputTable(output, 'TERMINALS', self.terminals, 1)
+      self.outputTable(output, 'FIRST', self.first, 1)
+      self.outputTable(output, 'FOLLOW', self.follow, 1)
+      self.outputTable(output, 'CLEANUP', self.cleanup, 1)
+      self.outputTable(output, 'PASS', [self.pass], 1) if self.pass
+    end
+
     private
     def do_production(lhs)
       rule = find_rule(lhs)

data/lib/ebnf/ll1/lexer.rb

@@ -29,7 +29,7 @@ module EBNF::LL1
   #     warn error.inspect
   #   end
   #
-  # @see http://en.wikipedia.org/wiki/Lexical_analysis
+  # @see https://en.wikipedia.org/wiki/Lexical_analysis
   class Lexer
     include Enumerable
 
@@ -43,10 +43,10 @@ module EBNF::LL1
       "\\'"  => '\'',   # \u0027 (apostrophe-quote, single quote mark)
       '\\\\' => '\\'    # \u005C (backslash)
     }.freeze
-    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/.freeze    # \uXXXX
-    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/.freeze    # \UXXXXXXXX
-    ECHAR               = /\\./                               # More liberal unescaping
-    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/.freeze
+    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/u.freeze    # \uXXXX
+    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/u.freeze    # \UXXXXXXXX
+    ECHAR               = /\\./u.freeze                        # More liberal unescaping
+    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/n.freeze
 
     ##
     # @return [Regexp] defines whitespace, including comments, otherwise whitespace must be explicit in terminals
@@ -59,7 +59,7 @@ module EBNF::LL1
     #
     # @param  [String] string
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#codepointEscape
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#codepointEscape
     def self.unescape_codepoints(string)
       string = string.dup
       string.force_encoding(Encoding::ASCII_8BIT) if string.respond_to?(:force_encoding)
@@ -81,7 +81,7 @@ module EBNF::LL1
     #
     # @param  [String] input
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
     def self.unescape_string(input)
       input.gsub(ECHAR) { |escaped| ESCAPE_CHARS[escaped] || escaped[1..-1]}
     end
@@ -131,7 +131,6 @@ module EBNF::LL1
 
       raise Error, "Terminal patterns not defined" unless @terminals && @terminals.length > 0
 
-      @lineno = 1
       @scanner = Scanner.new(input, **options)
     end
 
@@ -147,12 +146,6 @@ module EBNF::LL1
     # @return [String]
     attr_accessor :input
 
-    ##
-    # The current line number (zero-based).
-    #
-    # @return [Integer]
-    attr_reader   :lineno
-
     ##
     # Returns `true` if the input string is lexically valid.
     #
@@ -194,7 +187,7 @@ module EBNF::LL1
 
       @first ||= begin
         {} while !scanner.eos? && skip_whitespace
-        return @scanner = nil if scanner.eos?
+        return nil if scanner.eos?
 
         token = match_token(*types)
 
@@ -233,7 +226,7 @@ module EBNF::LL1
     # @return [Token]
     def recover(*types)
        until scanner.eos? || tok = match_token(*types)
-        if scanner.skip_until(@whitespace || /\s/m).nil? # Skip past current "token"
+        if scanner.skip_until(@whitespace || /\s+/m).nil? # Skip past current "token"
           # No whitespace at the end, must be and end of string
           scanner.terminate
         else
@@ -243,6 +236,14 @@ module EBNF::LL1
       scanner.unscan if tok
       first
     end
+
+    ##
+    # The current line number (one-based).
+    #
+    # @return [Integer]
+    def lineno
+      scanner.lineno
+    end
   protected
 
     # @return [StringScanner]
@@ -253,9 +254,7 @@ module EBNF::LL1
     def skip_whitespace
       # skip all white space, but keep track of the current line number
       while @whitespace && !scanner.eos?
-        if matched = scanner.scan(@whitespace)
-          @lineno += matched.count("\n")
-        else
+        unless scanner.scan(@whitespace)
           return
         end
       end
@@ -281,7 +280,6 @@ module EBNF::LL1
         if matched = scanner.scan(term.regexp)
           #STDERR.puts "  matched #{term.type.inspect}: #{matched.inspect}"
           tok = token(term.type, term.canonicalize(matched))
-          @lineno += matched.count("\n")
           return tok
         end
       end
@@ -372,7 +370,7 @@ module EBNF::LL1
     #   token.type   #=> :LANGTAG
     #   token.value  #=> "en"
     #
-    # @see http://en.wikipedia.org/wiki/Lexical_analysis#Token
+    # @see https://en.wikipedia.org/wiki/Lexical_analysis#Token
     class Token
       ##
       # The token's symbol type.
@@ -493,7 +491,7 @@ module EBNF::LL1
     #     "invalid token '%' on line 10",
     #     input: query, token: '%', lineno: 9)
     #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
+    # @see https://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError
       ##
       # The input string associated with the error.

data/lib/ebnf/ll1/parser.rb

@@ -3,12 +3,52 @@ require 'ebnf/ll1/lexer'
 module EBNF::LL1
   ##
   # A Generic LL1 parser using a lexer and branch tables defined using the SWAP tool chain (modified).
+  #
+  #  # Creating terminal definitions and parser rules to parse generated grammars
+  #
+  #  The parser is initialized to callbacks invoked on entry and exit
+  #  to each `terminal` and `production`. A trivial parser loop can be described as follows:
+  #
+  #      require 'ebnf/ll1/parser'
+  #      require 'meta'
+  #
+  #      class Parser
+  #        include Meta
+  #        include EBNF::LL1::Parser
+  #
+  #        terminal(:SYMBOL, /([a-z]|[A-Z]|[0-9]|_)+/) do |prod, token, input|
+  #          # Add data based on scanned token to input
+  #          input[:symbol] = token.value
+  #        end
+  #
+  #        start_production(:rule) do |input, current, callback|
+  #          # Process on start of production
+  #          # Set state for entry into recursed rules through current
+  #
+  #          # Callback to parser loop with callback
+  #        end
+  #
+  #        production(:rule) do |input, current, callback|
+  #          # Process on end of production
+  #          # return results in input, retrieve results from recursed rules in current
+  #
+  #          # Callback to parser loop with callback
+  #        end
+  #
+  #        def initialize(input)
+  #          parse(input, start_symbol,
+  #            branch: BRANCH,
+  #            first: FIRST,
+  #            follow: FOLLOW,
+  #            cleanup: CLEANUP
+  #          ) do |context, *data|
+  #            # Process calls from callback from productions
+  #
+  #          rescue ArgumentError, RDF::LL1::Parser::Error => e
+  #            progress("Parsing completed with errors:\n\t#{e.message}")
+  #            raise RDF::ReaderError, e.message if validate?
+  #          end
   module Parser
-    ##
-    # @private
-    # level above which debug messages are supressed
-    DEBUG_LEVEL = 10
-
     ##
     # @return [Integer] line number of current token
     attr_reader :lineno
@@ -186,7 +226,7 @@ module EBNF::LL1
     #     def each_statement(&block)
     #       @callback = block
     #
-    #       parse(START.to_sym) do |context, *data|
+    #       parse(input, START.to_sym) do |context, *data|
     #         case context
     #         when :statement
     #           yield *data
@@ -205,16 +245,13 @@ module EBNF::LL1
     #   Lists valid terminals that can precede each production (for error recovery).
     # @option options [Hash{Symbol,String => Array<Symbol,String>}] :follow ({})
     #   Lists valid terminals that can follow each production (for error recovery).
-    # @option options [Boolean]  :validate     (false)
-    #   whether to validate the parsed statements and values. If not validating, the parser will attempt to recover from errors.
-    # @option options [Boolean] :progress
-    #   Show progress of parser productions
-    # @option options [Boolean] :debug
-    #   Detailed debug output
-    # @option options [Boolean] :reset_on_start
-    #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
     # @option options[Integer] :high_water passed to lexer
+    # @option options [Logger] :logger for errors/progress/debug.
     # @option options[Integer] :low_water passed to lexer
+    # @option options [Boolean] :reset_on_start
+    #   Reset the parser state if the start token set with `prod` is found in a production. This reduces the production stack depth growth, which is appropriate for some grammars.
+    # @option options [Boolean]  :validate     (false)
+    #   whether to validate the parsed statements and values. If not validating, the parser will attempt to recover from errors.
     # @yield [context, *data]
     #   Yields for to return data to parser
     # @yieldparam [:statement, :trace] context
@@ -225,13 +262,9 @@ module EBNF::LL1
     # @raise [Exception] Raises exceptions for parsing errors
     #   or errors raised during processing callbacks. Internal
     #   errors are raised using {Error}.
-    # @see http://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
+    # @see https://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
     def parse(input = nil, start = nil, **options, &block)
       @options = options.dup
-      @options[:debug] ||= case
-      when @options[:progress] then 2
-      when @options[:validate] then 1
-      end
       @branch  = options[:branch]
       @first   = options[:first] ||= {}
       @follow  = options[:follow] ||= {}
@@ -356,9 +389,9 @@ module EBNF::LL1
           end
 
           # Get the list of follows for this sequence, this production and the stacked productions.
-          debug("recovery", "stack follows:", level: 4)
+          debug("recovery", "stack follows:")
           todo_stack.reverse.each do |todo|
-            debug("recovery", level: 4) {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
+            debug("recovery") {"  #{todo[:prod]}: #{@follow[todo[:prod]].inspect}"}
           end
 
           # Find all follows to the top of the stack
@@ -466,14 +499,15 @@ module EBNF::LL1
   protected
 
     ##
-    # Error information, used as level `0` debug messages.
+    # Error information, used as level `3` logger messages.
+    # Messages may be logged and are saved for reporting at end of parsing.
     #
     # @param [String] node Relevant location associated with message
     # @param [String] message Error string
-    # @param [Hash] options
+    # @param [Hash{Symbol => Object}] options
     # @option options [URI, #to_s] :production
     # @option options [Token] :token
-    # @see {#debug}
+    # @see #debug
     def error(node, message, **options)
       lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       m = "ERROR "
@@ -483,83 +517,82 @@ module EBNF::LL1
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
       @recovering = true
-      debug(node, m, level: 0, **options)
+      debug(node, m, level: options.fetch(:level, 3), **options)
       if options[:raise] || @options[:validate]
         raise Error.new(m, lineno: lineno, token: options[:token], production: options[:production])
       end
     end
 
     ##
-    # Warning information, used as level `1` debug messages.
+    # Warning information, used as level `2` logger messages.
+    # Messages may be logged and are saved for reporting at end of parsing.
     #
     # @param [String] node Relevant location associated with message
     # @param [String] message Error string
     # @param [Hash] options
     # @option options [URI, #to_s] :production
     # @option options [Token] :token
-    # @see {#debug}
+    # @see #debug
     def warn(node, message, **options)
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       m = "WARNING "
-      m += "[line: #{@lineno}] " if @lineno
+      m += "[line: #{lineno}] " if lineno
       m += message
       m += " (found #{options[:token].inspect})" if options[:token]
       m += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << m unless @recovering
-      debug(node, m, level: 1, **options)
+      debug(node, m, level: 2, lineno: lineno, **options)
     end
 
     ##
-    # Progress output when parsing. Passed as level `2` debug messages.
+    # Progress logged when parsing. Passed as level `1` logger messages.
     #
-    # @overload progress(node, message, **options)
+    # The call is ignored, unless `@options[:logger]` is set.
+    #
+    # @overload progress(node, message, **options, &block)
     #   @param [String] node Relevant location associated with message
     #   @param [String] message ("")
     #   @param [Hash] options
     #   @option options [Integer] :depth
     #       Recursion depth for indenting output
-    # @see {#debug}
+    # @see #debug
     def progress(node, *args, &block)
-      return unless @options[:progress] || @options[:debug]
+      return unless @options[:logger]
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       args << {} unless args.last.is_a?(Hash)
-      args.last[:level] ||= 2
+      args.last[:level] ||= 1
+      args.last[:lineno] ||= lineno
       debug(node, *args, &block)
     end
 
     ##
-    # Progress output when debugging.
+    # Debug logging.
     #
-    # The call is ignored, unless `@options[:debug]` is set, in which
-    # case it yields tracing information as indicated. Additionally,
-    # if `@options[:debug]` is an Integer, the call is aborted if the
-    # `:level` option is less than than `:level`.
+    # The call is ignored, unless `@options[:logger]` is set.
     #
     # @overload debug(node, message, **options)
     #   @param [Array<String>] args Relevant location associated with message
     #   @param [Hash] options
     #   @option options [Integer] :depth
     #     Recursion depth for indenting output
-    #   @option options [Integer] :level
-    #     Level assigned to message, by convention, level `0` is for
-    #     errors, level `1` is for warnings, level `2` is for parser
-    #     progress information, and anything higher is for various levels
-    #     of debug information.
-    #
-    # @yield trace, level, lineno, depth, args
-    # @yieldparam [:trace] trace
-    # @yieldparam [Integer] level
-    # @yieldparam [Integer] lineno
-    # @yieldparam [Integer] depth Recursive depth of productions
-    # @yieldparam [Array<String>] args
-    # @yieldreturn [String] added to message
-    def debug(*args)
-      return unless @options[:debug] && @parse_callback
+    #   @yieldreturn [String] additional string appended to `message`.
+    def debug(*args, &block)
+      return unless @options[:logger]
       options = args.last.is_a?(Hash) ? args.pop : {}
-      debug_level = options.fetch(:level, 3)
-      return if @options[:debug].is_a?(Integer) && debug_level > @options[:debug]
-
+      lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
+      level = options.fetch(:level, 0)
       depth = options[:depth] || self.depth
-      args << yield if block_given?
-      @parse_callback.call(:trace, debug_level, @lineno, depth, *args)
+
+      if self.respond_to?(:log_debug)
+        level = [:debug, :info, :warn, :error, :fatal][level]
+        log_debug(*args, **options.merge(level: level, lineno: lineno, depth: depth), &block)
+      elsif @options[:logger].respond_to?(:add)
+        args << yield if block_given?
+        @options[:logger].add(level, "[#{lineno}]" + (" " * depth) + args.join(" "))
+      elsif @options[:logger].respond_to?(:<<)
+        args << yield if block_given?
+        @options[:logger] << "[#{lineno}]" + (" " * depth) + args.join(" ")
+      end
     end
 
   private
@@ -570,7 +603,7 @@ module EBNF::LL1
       if handler
         # Create a new production data element, potentially allowing handler
         # to customize before pushing on the @prod_data stack
-        progress("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
+        debug("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
         data = {}
         begin
           self.class.eval_with_binding(self) {
@@ -584,12 +617,12 @@ module EBNF::LL1
       elsif [:merge, :star].include?(@cleanup[prod])
         # Save current data to merge later
         @prod_data << {}
-        progress("#{prod}(:start}:#{@prod_data.length}:cleanup:#{@cleanup[prod]}") { get_token.inspect + (@recovering ? ' recovering' : '')}
+        debug("#{prod}(:start}:#{@prod_data.length}:cleanup:#{@cleanup[prod]}") { get_token.inspect + (@recovering ? ' recovering' : '')}
       else
         # Make sure we push as many was we pop, even if there is no
         # explicit start handler
         @prod_data << {} if self.class.production_handlers[prod]
-        progress("#{prod}(:start:#{@prod_data.length})") { get_token.inspect + (@recovering ? ' recovering' : '')}
+        debug("#{prod}(:start:#{@prod_data.length})") { get_token.inspect + (@recovering ? ' recovering' : '')}
       end
       #puts "prod_data(s): " + @prod_data.inspect
     end
@@ -623,7 +656,7 @@ module EBNF::LL1
           else Array(input[k]) + Array(v)
           end
         end
-        progress("#{prod}(:finish):#{@prod_data.length} cleanup:#{@cleanup[prod]}") {@prod_data.last}
+        debug("#{prod}(:finish):#{@prod_data.length} cleanup:#{@cleanup[prod]}") {@prod_data.last}
       else
         progress("#{prod}(:finish):#{@prod_data.length}") { "recovering" if @recovering }
       end
@@ -730,7 +763,7 @@ module EBNF::LL1
     #     "invalid token '%' on line 10",
     #     token: '%', lineno: 9, production: :turtleDoc)
     #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
+    # @see https://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError
       ##
       # The current production.