ebnf 1.1.3 → 2.1.2

data/lib/ebnf/isoebnf/meta.rb

@@ -0,0 +1,75 @@
+# This file is automatically generated by ebnf version 2.0.0
+# Derived from etc/iso-ebnf.ebnf
+module ISOEBNFMeta
+  RULES = [
+    EBNF::Rule.new(:syntax, nil, [:star, :syntax_rule]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:syntax_rule, nil, [:seq, :meta_identifier, :defining_symbol, :definitions_list, :terminator_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definitions_list, nil, [:seq, :single_definition, :_definitions_list_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_1, nil, [:star, :_definitions_list_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_2, nil, [:seq, :definition_separator_symbol, :definitions_list]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:single_definition, nil, [:seq, :term, :_single_definition_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_1, nil, [:star, :_single_definition_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_2, nil, [:seq, ",", :term]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:term, nil, [:seq, :factor, :_term_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_1, nil, [:opt, :_term_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_2, nil, [:seq, "-", :exception]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:exception, nil, [:seq, :factor]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:factor, nil, [:seq, :_factor_1, :primary]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_1, nil, [:opt, :_factor_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_2, nil, [:seq, :integer, "*"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:primary, nil, [:alt, :optional_sequence, :repeated_sequence, :special_sequence, :grouped_sequence, :meta_identifier, :terminal_string, :empty]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:optional_sequence, nil, [:seq, :start_option_symbol, :definitions_list, :end_option_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repeated_sequence, nil, [:seq, :start_repeat_symbol, :definitions_list, :end_repeat_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:grouped_sequence, nil, [:seq, "(", :definitions_list, ")"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminals, nil, [:seq], kind: :terminals).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_string, nil, [:alt, :_terminal_string_1, :_terminal_string_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_1, nil, [:seq, "'", :_terminal_string_3, "'"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_3, nil, [:plus, :first_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_2, nil, [:seq, "\"", :_terminal_string_4, "\""]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_4, nil, [:plus, :second_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier, nil, [:seq, :letter, :_meta_identifier_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_meta_identifier_1, nil, [:star, :meta_identifier_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:integer, nil, [:plus, :decimal_digit], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence, nil, [:seq, "?", :_special_sequence_1, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_special_sequence_1, nil, [:star, :special_sequence_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment, nil, [:seq, :start_comment_symbol, :_comment_1, :end_comment_symbol], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_comment_1, nil, [:star, :comment_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment_symbol, nil, [:alt, :comment, :commentless_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:commentless_symbol, nil, [:alt, :terminal_character, :meta_identifier, :integer, :terminal_string, :special_sequence], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:letter, nil, [:range, "a-zA-Z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:decimal_digit, nil, [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier_character, nil, [:alt, :letter, :decimal_digit, "_"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_terminal_character, nil, [:diff, :terminal_character, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_terminal_character, nil, [:diff, :terminal_character, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_character, nil, [:diff, :terminal_character, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_character, nil, [:alt, :letter, :decimal_digit, :concatenate_symbol, :defining_symbol, :definition_separator_symbol, :end_comment_symbol, :end_group_symbol, :end_option_symbol, :end_repeat_symbol, :except_symbol, :first_quote_symbol, :repetition_symbol, :second_quote_symbol, :special_sequence_symbol, :start_comment_symbol, :start_group_symbol, :start_option_symbol, :start_repeat_symbol, :terminator_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:other_character, nil, [:alt, :_other_character_1, "\\"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_other_character_1, nil, [:range, ":+_%@&$<>^` ̃#x20#x23"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_separator, nil, [:range, "#x9#xa#xb#xc#xd#x20"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_pass, nil, [:alt, :__pass_1, :comment], kind: :pass).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:__pass_1, nil, [:plus, :gap_separator]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:empty, nil, [:seq, ""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:concatenate_symbol, nil, [:seq, ","], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repetition_symbol, nil, [:seq, "*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:except_symbol, nil, [:seq, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_quote_symbol, nil, [:seq, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_quote_symbol, nil, [:seq, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_comment_symbol, nil, [:seq, "(*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_comment_symbol, nil, [:seq, "*)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_group_symbol, nil, [:seq, "("], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_group_symbol, nil, [:seq, ")"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_symbol, nil, [:seq, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:defining_symbol, nil, [:alt, "=", ":"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definition_separator_symbol, nil, [:alt, "|", "/", "!"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminator_symbol, nil, [:alt, ";", "."], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_option_symbol, nil, [:seq, "["], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_option_symbol, nil, [:seq, "]"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_repeat_symbol, nil, [:alt, "{", "(:"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_repeat_symbol, nil, [:alt, "}", ":)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_free_symbol, nil, [:alt, :_gap_free_symbol_1, :terminal_string], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_1, nil, [:seq, :_gap_free_symbol_3, :terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_3, nil, [:not, :_gap_free_symbol_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_2, nil, [:range, "'\""], kind: :terminal).extend(EBNF::PEG::Rule),
+  ]
+end
+

data/lib/ebnf/ll1.rb

@@ -1,4 +1,86 @@
 module EBNF
+  ##
+  # This module extends {EBNF::Base} to create metadata including _branch_,  [First/Follow][], and other tables which is used by {EBNF::LL1::Parser} to recognize examples of the associated grammar.
+  #
+  ### Branch Table
+  #
+  #  The Branch table is a hash mapping production rules to a hash relating terminals appearing in input to sequence of productions to follow when the corresponding input terminal is found. This allows either the `seq` primitive, where all terminals map to the same sequence of productions, or the `alt` primitive, where each terminal may map to a different production.
+  #
+  #      BRANCH = {
+  #        :alt => {
+  #          "(" => [:seq, :_alt_1],
+  #          :HEX => [:seq, :_alt_1],
+  #          :O_RANGE => [:seq, :_alt_1],
+  #          :RANGE => [:seq, :_alt_1],
+  #          :STRING1 => [:seq, :_alt_1],
+  #          :STRING2 => [:seq, :_alt_1],
+  #          :SYMBOL => [:seq, :_alt_1],
+  #        },
+  #        ...
+  #        :declaration => {
+  #          "@pass" => [:pass],
+  #          "@terminals" => ["@terminals"],
+  #        },
+  #        ...
+  #      }
+  #
+  #  In this case the `alt` rule is `seq ('|' seq)*` can happen when any of the specified tokens appears on the input stream. The all cause the same token to be passed to the `seq` rule and follow with `_alt_1`, which handles the `('|' seq)*` portion of the rule, after the first sequence is matched.
+  #
+  #  The `declaration` rule is `@terminals' | pass` using the `alt` primitive determining the production to run based on the terminal appearing on the input stream. Eventually, a terminal production is found and the token is consumed.
+  #
+  ### First/Follow Table
+  #
+  #  The [First/Follow][] table is a hash mapping production rules to the terminals that may proceed or follow the rule. For example:
+  #
+  #      FIRST = {
+  #        :alt => [
+  #          :HEX,
+  #          :SYMBOL,
+  #          :RANGE,
+  #          :O_RANGE,
+  #          :STRING1,
+  #          :STRING2,
+  #          "("],
+  #        ...
+  #      }
+  #
+  ### Terminals Table
+  #
+  #  This table is a simple list of the terminal productions found in the grammar. For example:
+  #
+  #      TERMINALS = ["(", ")", "-",
+  #        "@pass", "@terminals",
+  #         :HEX, :LHS, :O_RANGE,:POSTFIX,
+  #        :RANGE, :STRING1, :STRING2, :SYMBOL,"|"
+  #      ].freeze
+  #
+  ### Cleanup Table
+  #
+  #  This table identifies productions which used EBNF rules, which are transformed to BNF for actual parsing. This allows the parser, in some cases, to reproduce *star*, *plus*, and *opt* rule matches. For example:
+  #
+  #      CLEANUP = {
+  #        :_alt_1 => :star,
+  #        :_alt_3 => :merge,
+  #        :_diff_1 => :opt,
+  #        :ebnf => :star,
+  #        :_ebnf_2 => :merge,
+  #        :_postfix_1 => :opt,
+  #        :seq => :plus,
+  #        :_seq_1 => :star,
+  #        :_seq_2 => :merge,
+  #      }.freeze
+  #
+  #  In this case the `ebnf` rule was `(declaration | rule)*`. As BNF does not support a star operator, this is decomposed into a set of rules using `alt` and `seq` primitives:
+  #
+  #      ebnf    ::= _empty _ebnf_2
+  #      _ebnf_1 ::= declaration | rule
+  #      _ebnf_2 ::= _ebnf_1 ebnf
+  #      _ebnf_3 ::= ebnf
+  #
+  #  The `_empty` production matches an empty string, so allows for now value. `_ebnf_2` matches `declaration | rule` (using the `alt` primitive) followed by `ebnf`, creating a sequence of zero or more `declaration` or `alt` members.
+  #
+  # [First/Follow]: https://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
+
   module LL1
     autoload :Lexer,    "ebnf/ll1/lexer"
     autoload :Parser,   "ebnf/ll1/parser"
@@ -51,8 +133,40 @@ module EBNF
     ##
     # Create first/follow for each rule using techniques defined for LL(1) parsers.
     #
+    # This takes rules which have transformed into BNF and adds first/follow and otehr information to the rules to allow the generation of metadata tables used for driving a parser.
+    #
+    # Given an initial rule in EBNF:
+    #
+    #     (rule enbf "1" (star declaration rule))
+    #
+    # The BNF transformation becomes:
+    #
+    #     (rule ebnf "1" (alt _empty _ebnf_2))
+    #     (rule _ebnf_1 "1.1" (alt declaration rule))
+    #     (rule _ebnf_2 "1.2" (seq _ebnf_1 ebnf))
+    #     (rule _ebnf_3 "1.3" (seq ebnf))
+    #
+    # After running this method, the rules are annotated with first/follow and cleanup rules:
+    #
+    #     (rule ebnf "1"
+    #      (start #t)
+    #      (first "@pass" "@terminals" LHS _eps)
+    #      (follow _eof)
+    #      (cleanup star)
+    #      (alt _empty _ebnf_2))
+    #     (rule _ebnf_1 "1.1"
+    #      (first "@pass" "@terminals" LHS)
+    #      (follow "@pass" "@terminals" LHS _eof)
+    #      (alt declaration rule))
+    #     (rule _ebnf_2 "1.2"
+    #      (first "@pass" "@terminals" LHS)
+    #      (follow _eof)
+    #      (cleanup merge)
+    #      (seq _ebnf_1 ebnf))
+    #     (rule _ebnf_3 "1.3" (first "@pass" "@terminals" LHS _eps) (follow _eof) (seq ebnf))
+    #
     # @return [EBNF] self
-    # @see http://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
+    # @see https://en.wikipedia.org/wiki/LL_parser#Constructing_an_LL.281.29_parsing_table
     # @param [Array<Symbol>] starts
     #   Set of symbols which are start rules
     def first_follow(*starts)
@@ -96,8 +210,9 @@ module EBNF
           firsts, follows = 0, 0
           # add Fi(wi) to Fi(Ai) for every rule Ai → wi
           #
-          # For sequences, this is the first rule in the sequence.
-          # For alts, this is every rule in the sequence
+          # * For sequences, this is the first rule in the sequence.
+          # * For alts, this is every rule in the sequence
+          # * Other rules don't matter, as they don't appear in strict BNF
           each(:rule) do |ai|
             # Fi(a w' ) = { a } for every terminal a
             ai.terminals(ast).each do |t|
@@ -168,6 +283,10 @@ module EBNF
           progress("first_follow") {"(#{ittr}) firsts #{firsts}, follows #{follows}"}
           ittr += 1
         end while (firsts + follows) > 0
+
+        debug("Fi.2-post: non-terminals without first") do
+          ast.reject(&:terminal?).reject(&:first).map(&:sym)
+        end if ast.reject(&:terminal?).any? {|r| r.first.nil?}
       end
     end
 
@@ -272,6 +391,19 @@ module EBNF
       end
     end
 
+    ##
+    # Output Ruby parser files for LL(1) parsing
+    #
+    # @param [IO, StringIO] output
+    def to_ruby_ll1(output, **options)
+      self.outputTable(output, 'BRANCH', self.branch, 1)
+      self.outputTable(output, 'TERMINALS', self.terminals, 1)
+      self.outputTable(output, 'FIRST', self.first, 1)
+      self.outputTable(output, 'FOLLOW', self.follow, 1)
+      self.outputTable(output, 'CLEANUP', self.cleanup, 1)
+      self.outputTable(output, 'PASS', [self.pass], 1) if self.pass
+    end
+
     private
     def do_production(lhs)
       rule = find_rule(lhs)
@@ -287,16 +419,16 @@ module EBNF
 
       if rule.expr.first == :matches
         debug("prod") {"Rule is regexp: #{rule}"}
-
-        error("No record of what token #{lhs} can start with") unless rule.first
         return
       end
 
+      error("No record of what token #{lhs.inspect} can start with") unless rule.first
+
       if rule.alt?
         # A First/Follow conflict appears when _eps is in the first
         # of one rule and there is a token in the first and
         # follow of the same rule
-        if rule.first.include?(:_eps) && !(overlap = ((rule.first & (rule.follow || [])) - [:eps])).empty?
+        if Array(rule.first).include?(:_eps) && !(overlap = ((Array(rule.first) & (rule.follow || [])) - [:eps])).empty?
           error("First/Follow Conflict: #{overlap.first.inspect} is both first and follow of #{rule.sym}")
         end
 

data/lib/ebnf/ll1/lexer.rb

@@ -29,7 +29,7 @@ module EBNF::LL1
   #     warn error.inspect
   #   end
   #
-  # @see http://en.wikipedia.org/wiki/Lexical_analysis
+  # @see https://en.wikipedia.org/wiki/Lexical_analysis
   class Lexer
     include Enumerable
 
@@ -43,10 +43,10 @@ module EBNF::LL1
       "\\'"  => '\'',   # \u0027 (apostrophe-quote, single quote mark)
       '\\\\' => '\\'    # \u005C (backslash)
     }.freeze
-    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/.freeze    # \uXXXX
-    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/.freeze    # \UXXXXXXXX
-    ECHAR               = /\\./                               # More liberal unescaping
-    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/.freeze
+    ESCAPE_CHAR4        = /\\u(?:[0-9A-Fa-f]{4,4})/u.freeze    # \uXXXX
+    ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/u.freeze    # \UXXXXXXXX
+    ECHAR               = /\\./u.freeze                        # More liberal unescaping
+    UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/n.freeze
 
     ##
     # @return [Regexp] defines whitespace, including comments, otherwise whitespace must be explicit in terminals
@@ -59,7 +59,7 @@ module EBNF::LL1
     #
     # @param  [String] string
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#codepointEscape
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#codepointEscape
     def self.unescape_codepoints(string)
       string = string.dup
       string.force_encoding(Encoding::ASCII_8BIT) if string.respond_to?(:force_encoding)
@@ -81,7 +81,7 @@ module EBNF::LL1
     #
     # @param  [String] input
     # @return [String]
-    # @see    http://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
+    # @see    https://www.w3.org/TR/rdf-sparql-query/#grammarEscapes
     def self.unescape_string(input)
       input.gsub(ECHAR) { |escaped| ESCAPE_CHARS[escaped] || escaped[1..-1]}
     end
@@ -98,8 +98,8 @@ module EBNF::LL1
     # @yieldparam [Lexer] lexer
     # @return [Lexer]
     # @raise  [Lexer::Error] on invalid input
-    def self.tokenize(input, terminals, options = {}, &block)
-      lexer = self.new(input, terminals, options)
+    def self.tokenize(input, terminals, **options, &block)
+      lexer = self.new(input, terminals, **options)
       block_given? ? block.call(lexer) : lexer
     end
 
@@ -115,17 +115,23 @@ module EBNF::LL1
     #   Whitespace between tokens, including comments
     # @option options[Integer] :high_water passed to scanner
     # @option options[Integer] :low_water passed to scanner
-    def initialize(input = nil, terminals = nil, options = {})
+    def initialize(input = nil, terminals = nil, **options)
       @options        = options.dup
       @whitespace     = @options[:whitespace]
       @terminals      = terminals.map do |term|
-        term.is_a?(Array) ? Terminal.new(*term) : term
+        if term.is_a?(Array) && term.length ==3
+          # Last element is options
+          Terminal.new(term[0], term[1], **term[2])
+        elsif term.is_a?(Array)
+          Terminal.new(*term)
+        else
+          term
+        end
       end
 
       raise Error, "Terminal patterns not defined" unless @terminals && @terminals.length > 0
 
-      @lineno = 1
-      @scanner = Scanner.new(input, options)
+      @scanner = Scanner.new(input, **options)
     end
 
     ##
@@ -140,12 +146,6 @@ module EBNF::LL1
     # @return [String]
     attr_accessor :input
 
-    ##
-    # The current line number (zero-based).
-    #
-    # @return [Integer]
-    attr_reader   :lineno
-
     ##
     # Returns `true` if the input string is lexically valid.
     #
@@ -187,7 +187,7 @@ module EBNF::LL1
 
       @first ||= begin
         {} while !scanner.eos? && skip_whitespace
-        return @scanner = nil if scanner.eos?
+        return nil if scanner.eos?
 
         token = match_token(*types)
 
@@ -226,7 +226,7 @@ module EBNF::LL1
     # @return [Token]
     def recover(*types)
        until scanner.eos? || tok = match_token(*types)
-        if scanner.skip_until(@whitespace || /\s/m).nil? # Skip past current "token"
+        if scanner.skip_until(@whitespace || /\s+/m).nil? # Skip past current "token"
           # No whitespace at the end, must be and end of string
           scanner.terminate
         else
@@ -236,6 +236,14 @@ module EBNF::LL1
       scanner.unscan if tok
       first
     end
+
+    ##
+    # The current line number (one-based).
+    #
+    # @return [Integer]
+    def lineno
+      scanner.lineno
+    end
   protected
 
     # @return [StringScanner]
@@ -246,9 +254,7 @@ module EBNF::LL1
     def skip_whitespace
       # skip all white space, but keep track of the current line number
       while @whitespace && !scanner.eos?
-        if matched = scanner.scan(@whitespace)
-          @lineno += matched.count("\n")
-        else
+        unless scanner.scan(@whitespace)
           return
         end
       end
@@ -274,7 +280,6 @@ module EBNF::LL1
         if matched = scanner.scan(term.regexp)
           #STDERR.puts "  matched #{term.type.inspect}: #{matched.inspect}"
           tok = token(term.type, term.canonicalize(matched))
-          @lineno += matched.count("\n")
           return tok
         end
       end
@@ -300,7 +305,7 @@ module EBNF::LL1
       #   Cause strings and codepoints to be unescaped.
       # @option options [Regexp] :partial_regexp
       #   A regular expression matching the beginning of this terminal; useful for terminals that match things longer than the scanner low water mark.
-      def initialize(type, regexp, options = {})
+      def initialize(type, regexp, **options)
         @type, @regexp, @options = type, regexp, options
         @partial_regexp = options[:partial_regexp]
         @map = options.fetch(:map, {})
@@ -353,8 +358,8 @@ module EBNF::LL1
     #   Scanner instance with access to matched groups
     # @param  [Hash{Symbol => Object}] options
     # @return [Token]
-    def token(type, value, options = {})
-      Token.new(type, value, options.merge(lineno: lineno))
+    def token(type, value, **options)
+      Token.new(type, value, lineno: lineno, **options)
     end
 
     ##
@@ -365,7 +370,7 @@ module EBNF::LL1
     #   token.type   #=> :LANGTAG
     #   token.value  #=> "en"
     #
-    # @see http://en.wikipedia.org/wiki/Lexical_analysis#Token
+    # @see https://en.wikipedia.org/wiki/Lexical_analysis#Token
     class Token
       ##
       # The token's symbol type.
@@ -398,7 +403,7 @@ module EBNF::LL1
       # @param  [String]                 value
       # @param  [Hash{Symbol => Object}] options
       # @option options [Integer]        :lineno (nil)
-      def initialize(type, value, options = {})
+      def initialize(type, value, **options)
         @type = type.to_s.to_sym if type
         @value = value.to_s
         @options = options.dup
@@ -486,7 +491,7 @@ module EBNF::LL1
     #     "invalid token '%' on line 10",
     #     input: query, token: '%', lineno: 9)
     #
-    # @see http://ruby-doc.org/core/classes/StandardError.html
+    # @see https://ruby-doc.org/core/classes/StandardError.html
     class Error < StandardError
       ##
       # The input string associated with the error.
@@ -514,7 +519,7 @@ module EBNF::LL1
       # @option options [String]         :input  (nil)
       # @option options [String]         :token  (nil)
       # @option options [Integer]        :lineno (nil)
-      def initialize(message, options = {})
+      def initialize(message, **options)
         @input  = options[:input]
         @token  = options[:token]
         @lineno = options[:lineno]