ebnf 2.0.0 → 2.1.0

data/lib/ebnf/isoebnf.rb

@@ -0,0 +1,229 @@
+require_relative 'isoebnf/meta'
+require 'logger'
+
+# ISO EBNF parser
+# Parses ISO EBNF into an array of {EBNF::Rule}.
+module EBNF
+  class ISOEBNF
+    include EBNF::PEG::Parser
+
+    # The base for terminal-character, which omits "'", '"', and '?'.
+    # Could be more optimized, and one might quible
+    # with the overly-strictly defined character set,
+    # but it is correct.
+    TERMINAL_CHARACTER_BASE = %r{
+      [a-zA-Z0-9] | # letter | decimal digit
+      ,           | # concatenate symbol
+      =           | # defining symbol
+      [\|\/!]     | # definition separator symbol
+      \*\)        | # end comment symbol
+      \)          | # end group symbol
+      \]          | # end option symbol
+      \}          | # end repeat symbol
+      \-          | # except symbol
+      #\'          | # first quote symbol
+      \*          | # repetition symbol
+      #\"          | # second quote symbol
+      #\?          | # special sequence symbol
+      \(\*        | # start comment symbol
+      \(          | # start group symbol
+      \[          | # start option symbol
+      \{          | # start repeat symbol
+      [;\.]       | # terminator symbol
+      [:+_%@&$<>^\x20\x23\\`~]  # other character
+    }x
+
+    TERMINAL_CHARACTER         = %r{#{TERMINAL_CHARACTER_BASE}|['"\?]}
+    FIRST_TERMINAL_CHARACTER   = %r{#{TERMINAL_CHARACTER_BASE}|["\?]}
+    SECOND_TERMINAL_CHARACTER  = %r{#{TERMINAL_CHARACTER_BASE}|['\?]}
+    SPECIAL_SEQUENCE_CHARACTER = %r{#{TERMINAL_CHARACTER_BASE}|['"]}
+
+    # Abstract syntax tree from parse
+    #
+    # @return [Array<EBNF::Rule>]
+    attr_reader :ast
+
+    # `[14] integer           ::= decimal_digit+`
+    terminal(:integer, /\d+/) do |value, prod|
+      value.to_i
+    end
+
+    # `[15] meta_identifier             ::= letter meta_identifier_character*`
+    terminal(:meta_identifier, /[a-zA-Z][a-zA-Z0-9_]*/) do |value|
+      value.to_sym
+    end
+
+    # `[17] terminal_string             ::= ("'" first_terminal_character+ "'")`
+    # `                                   | ('"' second_terminal_character+ '"')`
+    terminal(:terminal_string, /(?:'#{FIRST_TERMINAL_CHARACTER}+')|(?:"#{SECOND_TERMINAL_CHARACTER}+")/x) do |value|
+      value[1..-2]
+    end
+
+    # `[20] special_sequence            ::= '?' special_sequence_character* '?'`
+    terminal(:special_sequence, /\?#{SPECIAL_SEQUENCE_CHARACTER}+\?/)
+
+    # `[22] terminal_character          ::= [a-zA-Z0-9]`
+    # `                                   | [,=;*}#x2d?([{;]`
+    # `                                   | '*)'`
+    # `                                   | '(*'`
+    # `                                   | ']'`
+    # `                                   | other_character`
+    terminal(:terminal_character, TERMINAL_CHARACTER)
+
+    # `[25] empty                      ::= ''`
+    terminal(:empty, //)
+
+    # `[26] definition_separator_symbol ::= '|' | '/' | '!'`
+    terminal(:definition_separator_symbol, /[\|\/!]/)
+
+    # `[27] terminator_symbol           ::= ';' | '.'`
+    terminal(:terminator_symbol, /[;\.]/)
+
+    # `[28] start_option_symbol         ::= '['
+    terminal(:start_option_symbol, /\[|(?:\(\/)/)
+
+    # `[29] end_option_symbol           ::= ']'`
+    terminal(:end_option_symbol, /\]/)
+
+    # `[30] start_repeat_symbol         ::= '{' | '(:'`
+    terminal(:start_repeat_symbol, /{|\(:/)
+
+    # `[31] end_repeat_symbol           ::= '}' | ':)'`
+    terminal(:end_repeat_symbol, /}|:\)/)
+
+    # ## Non-terminal productions
+
+    # `[2]  syntax_rule       ::= meta_identifier '=' definitions_list terminator_symbol`
+    production(:syntax_rule, clear_packrat: true) do |value, data, callback|
+      # value contains an expression.
+      # Invoke callback
+      sym = value[0][:meta_identifier]
+      definitions_list = value[2][:definitions_list]
+      callback.call(:rule, EBNF::Rule.new(sym.to_sym, nil, definitions_list))
+      nil
+    end
+
+    # Setting `as_hash: true` in the start production makes the value of the form of a hash, rather than an array of hashes.
+    #
+    # `[3]  definitions_list  ::= single_definition (definition_separator_symbol definitions_list)*`
+    start_production(:definitions_list, as_hash: true)
+    production(:definitions_list) do |value|
+      if value[:_definitions_list_1].length > 0
+        [:alt, value[:single_definition]] + value[:_definitions_list_1]
+      else
+        value[:single_definition]
+      end
+    end
+    production(:_definitions_list_1) do |value|
+      Array(value.first)
+    end
+    start_production(:_definitions_list_2, as_hash: true)
+    production(:_definitions_list_2) do |value|
+      if Array(value[:definitions_list]).first == :alt
+        value[:definitions_list][1..-1]
+      else
+        [value[:definitions_list]]
+      end
+    end
+
+    # `[4]  single_definition ::= term (',' term)*`
+    start_production(:single_definition, as_hash: true)
+    production(:single_definition) do |value|
+      if value[:_single_definition_1].length > 0
+        [:seq, value[:term]] + value[:_single_definition_1]
+      else
+        value[:term]
+      end
+    end
+    production(:_single_definition_1) do |value|
+      value.map {|a1| a1.last[:term]}.compact # Get rid of '|'
+    end
+
+    # `[5]  term              ::= factor ('-' exception)?`
+    start_production(:term, as_hash: true)
+    production(:term) do |value|
+      if value[:_term_1]
+        [:diff, value[:factor], value[:_term_1]]
+      else
+        value[:factor]
+      end
+    end
+    production(:_term_1) do |value|
+      value.last[:exception] if value
+    end
+
+    # `[6]  exception         ::= factor`
+    start_production(:exception, as_hash: true)
+    production(:exception) do |value|
+      value[:factor]
+    end
+
+    # `[7]  factor            ::= (integer '*')? primary`
+    start_production(:factor, as_hash: true)
+    production(:factor) do |value|
+      if value[:_factor_1]
+        [:rept, value[:_factor_1], value[:_factor_1], value[:primary]]
+      else
+        value[:primary]
+      end
+    end
+    production(:_factor_2) do |value|
+      value.first[:integer]
+    end
+
+    # `[9]  optional_sequence ::= start_option_symbol definitions_list end_option_symbol`
+    production(:optional_sequence) do |value|
+      [:opt, value[1][:definitions_list]]
+    end
+
+    # `[10] repeated_sequence ::= start_repeat_symbol definitions_list end_repeat_symbol`
+    production(:repeated_sequence) do |value|
+      [:star, value[1][:definitions_list]]
+    end
+
+    # `[11] grouped_sequence  ::= '(' definitions_list ')'`
+    production(:grouped_sequence) do |value|
+      [:seq, value[1][:definitions_list]]
+    end
+
+    # ## Parser invocation.
+    # On start, yield ourselves if a block is given, otherwise, return this parser instance
+    #
+    # @param  [#read, #to_s]          input
+    # @param  [Hash{Symbol => Object}] options
+    # @option options [Boolean] :level
+    #   Trace level. 0(debug), 1(info), 2(warn), 3(error).
+    # @return [EBNFParser]
+    def initialize(input, **options, &block)
+      # If the `level` option is set, instantiate a logger for collecting trace information.
+      if options.has_key?(:level)
+        options[:logger] = Logger.new(STDERR)
+        options[:logger].level = options[:level]
+        options[:logger].formatter = lambda {|severity, datetime, progname, msg| "#{severity} #{msg}\n"}
+      end
+
+      # Read input, if necessary, which will be used in a Scanner.
+      @input = input.respond_to?(:read) ? input.read : input.to_s
+
+      parsing_terminals = false
+      @ast = []
+      parse(@input,
+            :syntax,
+            ISOEBNFMeta::RULES,
+            whitespace: %r{([\x09-\x0d\x20]|(?:\(\*(?:(?:\*[^\)])|[^*])*\*\)))+},
+            **options
+      ) do |context, *data|
+        rule = case context
+        when :rule
+          # A rule which has already been turned into a `Rule` object.
+          rule = data.first
+          rule.kind = :terminal if parsing_terminals
+          rule
+        end
+        @ast << rule if rule
+      end
+    rescue EBNF::PEG::Parser::Error => e
+      raise SyntaxError, e.message
+    end
+  end
+end

data/lib/ebnf/isoebnf/meta.rb

@@ -0,0 +1,75 @@
+# This file is automatically generated by ebnf version 2.0.0
+# Derived from etc/iso-ebnf.ebnf
+module ISOEBNFMeta
+  RULES = [
+    EBNF::Rule.new(:syntax, nil, [:star, :syntax_rule]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:syntax_rule, nil, [:seq, :meta_identifier, :defining_symbol, :definitions_list, :terminator_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definitions_list, nil, [:seq, :single_definition, :_definitions_list_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_1, nil, [:star, :_definitions_list_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_definitions_list_2, nil, [:seq, :definition_separator_symbol, :definitions_list]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:single_definition, nil, [:seq, :term, :_single_definition_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_1, nil, [:star, :_single_definition_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_single_definition_2, nil, [:seq, ",", :term]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:term, nil, [:seq, :factor, :_term_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_1, nil, [:opt, :_term_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_term_2, nil, [:seq, "-", :exception]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:exception, nil, [:seq, :factor]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:factor, nil, [:seq, :_factor_1, :primary]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_1, nil, [:opt, :_factor_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_factor_2, nil, [:seq, :integer, "*"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:primary, nil, [:alt, :optional_sequence, :repeated_sequence, :special_sequence, :grouped_sequence, :meta_identifier, :terminal_string, :empty]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:optional_sequence, nil, [:seq, :start_option_symbol, :definitions_list, :end_option_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repeated_sequence, nil, [:seq, :start_repeat_symbol, :definitions_list, :end_repeat_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:grouped_sequence, nil, [:seq, "(", :definitions_list, ")"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminals, nil, [:seq], kind: :terminals).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_string, nil, [:alt, :_terminal_string_1, :_terminal_string_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_1, nil, [:seq, "'", :_terminal_string_3, "'"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_3, nil, [:plus, :first_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_2, nil, [:seq, "\"", :_terminal_string_4, "\""]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminal_string_4, nil, [:plus, :second_terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier, nil, [:seq, :letter, :_meta_identifier_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_meta_identifier_1, nil, [:star, :meta_identifier_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:integer, nil, [:plus, :decimal_digit], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence, nil, [:seq, "?", :_special_sequence_1, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_special_sequence_1, nil, [:star, :special_sequence_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment, nil, [:seq, :start_comment_symbol, :_comment_1, :end_comment_symbol], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_comment_1, nil, [:star, :comment_symbol]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:comment_symbol, nil, [:alt, :comment, :commentless_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:commentless_symbol, nil, [:alt, :terminal_character, :meta_identifier, :integer, :terminal_string, :special_sequence], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:letter, nil, [:range, "a-zA-Z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:decimal_digit, nil, [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:meta_identifier_character, nil, [:alt, :letter, :decimal_digit, "_"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_terminal_character, nil, [:diff, :terminal_character, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_terminal_character, nil, [:diff, :terminal_character, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_character, nil, [:diff, :terminal_character, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminal_character, nil, [:alt, :letter, :decimal_digit, :concatenate_symbol, :defining_symbol, :definition_separator_symbol, :end_comment_symbol, :end_group_symbol, :end_option_symbol, :end_repeat_symbol, :except_symbol, :first_quote_symbol, :repetition_symbol, :second_quote_symbol, :special_sequence_symbol, :start_comment_symbol, :start_group_symbol, :start_option_symbol, :start_repeat_symbol, :terminator_symbol, :other_character], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:other_character, nil, [:alt, :_other_character_1, "\\"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_other_character_1, nil, [:range, ":+_%@&$<>^` ̃#x20#x23"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_separator, nil, [:range, "#x9#xa#xb#xc#xd#x20"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_pass, nil, [:alt, :__pass_1, :comment], kind: :pass).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:__pass_1, nil, [:plus, :gap_separator]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:empty, nil, [:seq, ""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:concatenate_symbol, nil, [:seq, ","], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:repetition_symbol, nil, [:seq, "*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:except_symbol, nil, [:seq, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:first_quote_symbol, nil, [:seq, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:second_quote_symbol, nil, [:seq, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_comment_symbol, nil, [:seq, "(*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_comment_symbol, nil, [:seq, "*)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_group_symbol, nil, [:seq, "("], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_group_symbol, nil, [:seq, ")"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:special_sequence_symbol, nil, [:seq, "?"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:defining_symbol, nil, [:alt, "=", ":"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:definition_separator_symbol, nil, [:alt, "|", "/", "!"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:terminator_symbol, nil, [:alt, ";", "."], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_option_symbol, nil, [:seq, "["], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_option_symbol, nil, [:seq, "]"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:start_repeat_symbol, nil, [:alt, "{", "(:"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:end_repeat_symbol, nil, [:alt, "}", ":)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:gap_free_symbol, nil, [:alt, :_gap_free_symbol_1, :terminal_string], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_1, nil, [:seq, :_gap_free_symbol_3, :terminal_character]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_3, nil, [:not, :_gap_free_symbol_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_gap_free_symbol_2, nil, [:range, "'\""], kind: :terminal).extend(EBNF::PEG::Rule),
+  ]
+end
+

data/lib/ebnf/ll1.rb

@@ -9,9 +9,7 @@ module EBNF
   #      BRANCH = {
   #        :alt => {
   #          "(" => [:seq, :_alt_1],
-  #          :ENUM => [:seq, :_alt_1],
   #          :HEX => [:seq, :_alt_1],
-  #          :O_ENUM => [:seq, :_alt_1],
   #          :O_RANGE => [:seq, :_alt_1],
   #          :RANGE => [:seq, :_alt_1],
   #          :STRING1 => [:seq, :_alt_1],
@@ -38,8 +36,6 @@ module EBNF
   #        :alt => [
   #          :HEX,
   #          :SYMBOL,
-  #          :ENUM,
-  #          :O_ENUM,
   #          :RANGE,
   #          :O_RANGE,
   #          :STRING1,
@@ -54,7 +50,7 @@ module EBNF
   #
   #      TERMINALS = ["(", ")", "-",
   #        "@pass", "@terminals",
-  #        :ENUM, :HEX, :LHS, :O_ENUM, :O_RANGE,:POSTFIX,
+  #         :HEX, :LHS, :O_RANGE,:POSTFIX,
   #        :RANGE, :STRING1, :STRING2, :SYMBOL,"|"
   #      ].freeze
   #
@@ -214,8 +210,9 @@ module EBNF
           firsts, follows = 0, 0
           # add Fi(wi) to Fi(Ai) for every rule Ai → wi
           #
-          # For sequences, this is the first rule in the sequence.
-          # For alts, this is every rule in the sequence
+          # * For sequences, this is the first rule in the sequence.
+          # * For alts, this is every rule in the sequence
+          # * Other rules don't matter, as they don't appear in strict BNF
           each(:rule) do |ai|
             # Fi(a w' ) = { a } for every terminal a
             ai.terminals(ast).each do |t|

data/lib/ebnf/ll1/parser.rb

@@ -576,15 +576,23 @@ module EBNF::LL1
     #   @option options [Integer] :depth
     #     Recursion depth for indenting output
     #   @yieldreturn [String] additional string appended to `message`.
-    def debug(*args)
+    def debug(*args, &block)
       return unless @options[:logger]
       options = args.last.is_a?(Hash) ? args.pop : {}
       lineno = @lineno || (options[:token].lineno if options[:token].respond_to?(:lineno))
       level = options.fetch(:level, 0)
-
       depth = options[:depth] || self.depth
-      args << yield if block_given?
-      @options[:logger].add(level, "[#{@lineno}]" + (" " * depth) + args.join(" "))
+
+      if self.respond_to?(:log_debug)
+        level = [:debug, :info, :warn, :error, :fatal][level]
+        log_debug(*args, **options.merge(level: level, lineno: lineno, depth: depth), &block)
+      elsif @options[:logger].respond_to?(:add)
+        args << yield if block_given?
+        @options[:logger].add(level, "[#{lineno}]" + (" " * depth) + args.join(" "))
+      elsif @options[:logger].respond_to?(:<<)
+        args << yield if block_given?
+        @options[:logger] << "[#{lineno}]" + (" " * depth) + args.join(" ")
+      end
     end
 
   private

data/lib/ebnf/native.rb

@@ -0,0 +1,320 @@
+module EBNF
+  module Native
+    ##
+    # Native parser for EBNF; less accurate, but appropriate when changing EBNF grammar, itself.
+    #
+    # Iterate over rule strings.
+    # a line that starts with '\[' or '@' starts a new rule
+    #
+    # @param [StringScanner] scanner
+    # @yield rule_string
+    # @yieldparam [String] rule_string
+    def eachRule(scanner)
+      cur_lineno = 1
+      r = ''
+      until scanner.eos?
+        case
+        when s = scanner.scan(%r(\s+)m)
+          # Eat whitespace
+          cur_lineno += s.count("\n")
+          #debug("eachRule(ws)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r(/\*([^\*]|\*[^\/])*\*/)m)
+          # Eat comments /* .. */
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r(\(\*([^\*]|\*[^\)])*\*\))m)
+          # Eat comments (* .. *)
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(%r((#(?!x)|//).*$))
+          # Eat comments // & #
+          cur_lineno += s.count("\n")
+          debug("eachRule(comment)") { "[#{cur_lineno}] #{s.inspect}" }
+        when s = scanner.scan(/\A["']/)
+          # Found a quote, scan until end of matching quote
+          s += scanner.scan_until(/#{scanner.matched}|$/)
+          r += s
+        when s = scanner.scan(%r(^@terminals))
+          #debug("eachRule(@terminals)") { "[#{cur_lineno}] #{s.inspect}" }
+          yield(r) unless r.empty?
+          @lineno = cur_lineno
+          yield(s)
+          r = ''
+        when s = scanner.scan(/@pass/)
+          # Found rule start, if we've already collected a rule, yield it
+          #debug("eachRule(@pass)") { "[#{cur_lineno}] #{s.inspect}" }
+          yield r unless r.empty?
+          @lineno = cur_lineno
+          r = s
+        when s = scanner.scan(EBNF::Terminals::LHS)
+          # Found rule start, if we've already collected a rule, yield it
+          yield r unless r.empty?
+          #debug("eachRule(rule)") { "[#{cur_lineno}] #{s.inspect}" }
+          @lineno = cur_lineno
+          r = s
+        else
+          # Collect until end of line, or start of comment or quote
+          s = scanner.scan_until(%r{(?:[/\(]\*)|#(?!x)|//|["']|$})
+          if scanner.matched.length > 0
+            # Back up scan head before ending match
+            scanner.pos = scanner.pos - scanner.matched.length
+
+            # Remove matched from end of string
+            s = s[0..-(scanner.matched.length+1)]
+          end
+          cur_lineno += s.count("\n")
+          #debug("eachRule(rest)") { "[#{cur_lineno}] #{s.inspect}" }
+          r += s
+        end
+      end
+      yield r unless r.empty?
+    end
+  
+    ##
+    # Parse a rule into an optional rule number, a symbol and an expression
+    #
+    # @param [String] rule
+    # @return [Rule]
+    def ruleParts(rule)
+      num_sym, expr = rule.split('::=', 2).map(&:strip)
+      num, sym = num_sym.split(']', 2).map(&:strip)
+      num, sym = "", num if sym.nil?
+      num = num[1..-1]
+      r = Rule.new(sym && sym.to_sym, num, expression(expr).first, ebnf: self)
+      debug("ruleParts") { r.inspect }
+      r
+    end
+
+    ##
+    # Parse a string into an expression tree and a remaining string
+    #
+    # @example
+    #     >>> expression("a b c")
+    #     ((seq a b c) '')
+    #     
+    #     >>> expression("a? b+ c*")
+    #     ((seq (opt a) (plus b) (star c)) '')
+    #     
+    #     >>> expression(" | x xlist")
+    #     ((alt (seq) (seq x xlist)) '')
+    #     
+    #     >>> expression("a | (b - c)")
+    #     ((alt a (diff b c)) '')
+    #     
+    #     >>> expression("a b | c d")
+    #     ((alt (seq a b) (seq c d)) '')
+    #     
+    #     >>> expression("a | b | c")
+    #     ((alt a b c) '')
+    #     
+    #     >>> expression("a) b c")
+    #     (a ' b c')
+    #     
+    #     >>> expression("BaseDecl? PrefixDecl*")
+    #     ((seq (opt BaseDecl) (star PrefixDecl)) '')
+    #     
+    #     >>> expression("NCCHAR1 | diff | [0-9] | #x00B7 | [#x0300-#x036F] | \[#x203F-#x2040\]")
+    #     ((alt NCCHAR1 diff
+    #           (range '0-9')
+    #           (hex '#x00B7')
+    #           (range '#x0300-#x036F')
+    #           (range, '#x203F-#x2040')) '')
+    #     
+    # @param [String] s
+    # @return [Array]
+    def expression(s)
+      debug("expression") {"(#{s.inspect})"}
+      e, s = depth {alt(s)}
+      debug {"=> alt returned #{[e, s].inspect}"}
+      unless s.to_s.empty?
+        t, ss = depth {terminal(s)}
+        debug {"=> terminal returned #{[t, ss].inspect}"}
+        return [e, ss] if t.is_a?(Array) && t.first == :")"
+      end
+      [e, s]
+    end
+  
+    ##
+    # Parse alt
+    #     >>> alt("a | b | c")
+    #     ((alt a b c) '')
+    # @param [String] s
+    # @return [Array]
+    def alt(s)
+      debug("alt") {"(#{s.inspect})"}
+      args = []
+      while !s.to_s.empty?
+        e, s = depth {seq(s)}
+        debug {"=> seq returned #{[e, s].inspect}"}
+        if e.to_s.empty?
+          break unless args.empty?
+          e = [:seq, []] # empty sequence
+        end
+        args << e
+        unless s.to_s.empty?
+          t, ss = depth {terminal(s)}
+          break unless t[0] == :alt
+          s = ss
+        end
+      end
+      args.length > 1 ? [args.unshift(:alt), s] : [e, s]
+    end
+  
+    ##
+    # parse seq
+    #
+    #     >>> seq("a b c")
+    #     ((seq a b c) '')
+    #     
+    #     >>> seq("a b? c")
+    #     ((seq a (opt b) c) '')
+    def seq(s)
+      debug("seq") {"(#{s.inspect})"}
+      args = []
+      while !s.to_s.empty?
+        e, ss = depth {diff(s)}
+        debug {"=> diff returned #{[e, ss].inspect}"}
+        unless e.to_s.empty?
+          args << e
+          s = ss
+        else
+          break;
+        end
+      end
+      if args.length > 1
+        [args.unshift(:seq), s]
+      elsif args.length == 1
+        args + [s]
+      else
+        ["", s]
+      end
+    end
+  
+    ##
+    # parse diff
+    # 
+    #     >>> diff("a - b")
+    #     ((diff a b) '')
+    def diff(s)
+      debug("diff") {"(#{s.inspect})"}
+      e1, s = depth {postfix(s)}
+      debug {"=> postfix returned #{[e1, s].inspect}"}
+      unless e1.to_s.empty?
+        unless s.to_s.empty?
+          t, ss = depth {terminal(s)}
+          debug {"diff #{[t, ss].inspect}"}
+          if t.is_a?(Array) && t.first == :diff
+            s = ss
+            e2, s = primary(s)
+            unless e2.to_s.empty?
+              return [[:diff, e1, e2], s]
+            else
+              error("diff", "Syntax Error")
+              raise SyntaxError, "diff missing second operand"
+            end
+          end
+        end
+      end
+      [e1, s]
+    end
+  
+    ##
+    # parse postfix
+    # 
+    #     >>> postfix("a b c")
+    #     (a ' b c')
+    #     
+    #     >>> postfix("a? b c")
+    #     ((opt, a) ' b c')
+    def postfix(s)
+      debug("postfix") {"(#{s.inspect})"}
+      e, s = depth {primary(s)}
+      debug {"=> primary returned #{[e, s].inspect}"}
+      return ["", s] if e.to_s.empty?
+      if !s.to_s.empty?
+        t, ss = depth {terminal(s)}
+        debug {"=> #{[t, ss].inspect}"}
+        if t.is_a?(Array) && [:opt, :star, :plus].include?(t.first)
+          return [[t.first, e], ss]
+        end
+      end
+      [e, s]
+    end
+
+    ##
+    # parse primary
+    # 
+    #     >>> primary("a b c")
+    #     (a ' b c')
+    def primary(s)
+      debug("primary") {"(#{s.inspect})"}
+      t, s = depth {terminal(s)}
+      debug {"=> terminal returned #{[t, s].inspect}"}
+      if t.is_a?(Symbol) || t.is_a?(String)
+        [t, s]
+      elsif %w(range hex).map(&:to_sym).include?(t.first)
+        [t, s]
+      elsif t.first == :"("
+        e, s = depth {expression(s)}
+        debug {"=> expression returned #{[e, s].inspect}"}
+        [e, s]
+      else
+        ["", s]
+      end
+    end
+  
+    ##
+    # parse one terminal; return the terminal and the remaining string
+    # 
+    # A terminal is represented as a tuple whose 1st item gives the type;
+    # some types have additional info in the tuple.
+    # 
+    # @example
+    #     >>> terminal("'abc' def")
+    #     ('abc' ' def')
+    #     
+    #     >>> terminal("[0-9]")
+    #     ((range '0-9') '')
+    #     >>> terminal("#x00B7")
+    #     ((hex '#x00B7') '')
+    #     >>> terminal ("\[#x0300-#x036F\]")
+    #     ((range '#x0300-#x036F') '')
+    #     >>> terminal("\[^<>'{}|^`\]-\[#x00-#x20\]")
+    #     ((range "^<>'{}|^`") '-\[#x00-#x20\]')
+    def terminal(s)
+      s = s.strip
+      #STDERR.puts s.inspect
+      case m = s[0,1]
+      when '"', "'" # STRING1 or STRING2
+        l, s = s[1..-1].split(m.rstrip, 2)
+        [LL1::Lexer.unescape_string(l), s]
+      when '[' # RANGE, O_RANGE
+        l, s = s[1..-1].split(/(?<=[^\\])\]/, 2)
+        [[:range, LL1::Lexer.unescape_string(l)], s]
+      when '#' # HEX
+        s.match(/(#x\h+)(.*)$/)
+        l, s = $1, $2
+        [[:hex, l], s]
+      when /[\w\.]/ # SYMBOL
+        s.match(/([\w\.]+)(.*)$/)
+        l, s = $1, $2
+        [l.to_sym, s]
+      when '-'
+        [[:diff], s[1..-1]]
+      when '?'
+        [[:opt], s[1..-1]]
+      when '|'
+        [[:alt], s[1..-1]]
+      when '+'
+        [[:plus], s[1..-1]]
+      when '*'
+        [[:star], s[1..-1]]
+      when /[\(\)]/ # '(' or ')'
+        [[m.to_sym], s[1..-1]]
+      else
+        error("terminal", "unrecognized terminal: #{s.inspect}")
+        raise SyntaxError, "unrecognized terminal: #{s.inspect}"
+      end
+    end
+  end
+end