ebnf 1.1.3 → 2.1.2

data/lib/ebnf/base.rb

@@ -2,7 +2,7 @@ require 'strscan'
 
 # Extended Bakus-Nour Form (EBNF), being the W3C variation is
 # originaly defined in the
-# [W3C XML 1.0 Spec](http://www.w3.org/TR/REC-xml/#sec-notation).
+# [W3C XML 1.0 Spec](https://www.w3.org/TR/REC-xml/#sec-notation).
 #
 # This version attempts to be less strict than the strict definition
 # to allow for coloquial variations (such as in the Turtle syntax).
@@ -12,8 +12,8 @@ require 'strscan'
 #
 # Comments include the content between '/*' and '*/'
 #
-# @see http://www.w3.org/2000/10/swap/grammar/ebnf2turtle.py
-# @see http://www.w3.org/2000/10/swap/grammar/ebnf2bnf.n3
+# @see https://www.w3.org/2000/10/swap/grammar/ebnf2turtle.py
+# @see https://www.w3.org/2000/10/swap/grammar/ebnf2bnf.n3
 #
 # Based on bnf2turtle by Dan Connolly.
 #
@@ -36,7 +36,7 @@ require 'strscan'
 # derived mechanically from the specification.
 # 
 # 
-# [N3 design note]: http://www.w3.org/DesignIssues/Notation3
+# [N3 design note]: https://www.w3.org/DesignIssues/Notation3
 # 
 # Related Work
 # ------------
@@ -59,31 +59,15 @@ require 'strscan'
 # expression of the grammar in terms of the higher level EBNF
 # constructs.
 # 
-# [goal]: http://www.w3.org/2002/02/mid/1086902566.21030.1479.camel@dirk;list=public-cwm-bugs
-# [n3p announcement]: http://lists.w3.org/Archives/Public/public-cwm-talk/2004OctDec/0029.html
-# [Yacker]: http://www.w3.org/1999/02/26-modules/User/Yacker
-# [SPARQL specification]: http://www.w3.org/TR/rdf-sparql-query/
-# [Cwm Release 1.1.0rc1]: http://lists.w3.org/Archives/Public/public-cwm-announce/2005JulSep/0000.html
-# [bnf-rules.n3]: http://www.w3.org/2000/10/swap/grammar/bnf-rules.n3
+# [goal]: https://www.w3.org/2002/02/mid/1086902566.21030.1479.camel@dirk;list=public-cwm-bugs
+# [n3p announcement]: https://lists.w3.org/Archives/Public/public-cwm-talk/2004OctDec/0029.html
+# [Yacker]: https://rubygems/02/26-modules/User/Yacker
+# [SPARQL specification]: https://www.w3.org/TR/rdf-sparql-query/
+# [Cwm Release 1.1.0rc1]: https://lists.w3.org/Archives/Public/public-cwm-announce/2005JulSep/0000.html
+# [bnf-rules.n3]: https://www.w3.org/2000/10/swap/grammar/bnf-rules.n3
 # 
-# Open Issues and Future Work
-# ---------------------------
-# 
-# The yacker output also has the terminals compiled to elaborate regular
-# expressions. The best strategy for dealing with lexical tokens is not
-# yet clear. Many tokens in SPARQL are case insensitive; this is not yet
-# captured formally.
-# 
-# The schema for the EBNF vocabulary used here (``g:seq``, ``g:alt``, ...)
-# is not yet published; it should be aligned with [swap/grammar/bnf][]
-# and the [bnf2html.n3][] rules (and/or the style of linked XHTML grammar
-# in the SPARQL and XML specificiations).
-# 
-# It would be interesting to corroborate the claim in the SPARQL spec
-# that the grammar is LL(1) with a mechanical proof based on N3 rules.
-# 
-# [swap/grammar/bnf]: http://www.w3.org/2000/10/swap/grammar/bnf
-# [bnf2html.n3]: http://www.w3.org/2000/10/swap/grammar/bnf2html.n3  
+# [swap/grammar/bnf]: https://www.w3.org/2000/10/swap/grammar/bnf
+# [bnf2html.n3]: https://www.w3.org/2000/10/swap/grammar/bnf2html.n3  
 # 
 # Background
 # ----------
@@ -93,14 +77,15 @@ require 'strscan'
 # of N3 that maps directly to (and from) the standard XML syntax for
 # RDF.
 #
-# [N3 Primer]: http://www.w3.org/2000/10/swap/Primer.html
+# [N3 Primer]: https://www.w3.org/2000/10/swap/Primer.html
 # 
 # @author Gregg Kellogg
 module EBNF
   class Base
     include BNF
     include LL1
-    include Parser
+    include Native
+    include PEG
 
     # Abstract syntax tree from parse
     #
@@ -116,24 +101,33 @@ module EBNF
     # in S-Expressions (similar to SPARQL SSE)
     #
     # @param [#read, #to_s] input
+    # @param [Symbol] format (:ebnf)
+    #   Format of input, one of `:abnf`, `:ebnf`, `:isoebnf`, `:isoebnf`, `:native`, or `:sxp`.
+    #   Use `:native` for the native EBNF parser, rather than the PEG parser.
     # @param [Hash{Symbol => Object}] options
-    # @param [Symbol] :format (:ebnf)
-    #   Format of input, one of :ebnf, or :sxp
     # @option options [Boolean, Array] :debug
     #   Output debug information to an array or $stdout.
+    # @option options [Boolean, Array] :validate
+    #   Validate resulting grammar.
     def initialize(input, format: :ebnf, **options)
       @options = options.dup
       @lineno, @depth, @errors = 1, 0, []
-      terminal = false
       @ast = []
 
       input = input.respond_to?(:read) ? input.read : input.to_s
 
       case format
-      when :sxp
-        require 'sxp' unless defined?(SXP)
-        @ast = SXP::Reader::Basic.read(input).map {|e| Rule.from_sxp(e)}
+      when :abnf
+        abnf = ABNF.new(input, **options)
+        @ast = abnf.ast
       when :ebnf
+        ebnf = Parser.new(input, **options)
+        @ast = ebnf.ast
+      when :isoebnf
+        iso = ISOEBNF.new(input, **options)
+        @ast = iso.ast
+      when :native
+        terminals = false
         scanner = StringScanner.new(input)
 
         eachRule(scanner) do |r|
@@ -141,7 +135,9 @@ module EBNF
           case r
           when /^@terminals/
             # Switch mode to parsing terminals
-            terminal = true
+            terminals = true
+            rule = Rule.new(nil, nil, nil, kind: :terminals, ebnf: self)
+            @ast << rule
           when /^@pass\s*(.*)$/m
             expr = expression($1).first
             rule = Rule.new(nil, nil, expr, kind: :pass, ebnf: self)
@@ -150,14 +146,49 @@ module EBNF
           else
             rule = depth {ruleParts(r)}
 
-            rule.kind = :terminal if terminal # Override after we've parsed @terminals
+            rule.kind = :terminal if terminals # Override after we've parsed @terminals
             rule.orig = r
             @ast << rule
           end
         end
+      when :sxp
+        require 'sxp' unless defined?(SXP)
+        @ast = SXP::Reader::Basic.read(input).map {|e| Rule.from_sxp(e)}
       else
         raise "unknown input format #{format.inspect}"
       end
+
+      validate! if @options[:validate]
+    end
+
+    ##
+    # Validate the grammar.
+    #
+    # Makes sure that rules reference either strings or other defined rules.
+    #
+    # @raise [RangeError]
+    def validate!
+      ast.each do |rule|
+        begin
+          rule.validate!(@ast)
+        rescue SyntaxError => e
+          error("In rule #{rule.sym}: #{e.message}")
+        end
+      end
+      raise SyntaxError, errors.join("\n") unless errors.empty?
+    end
+
+    ##
+    # Is the grammar valid?
+    #
+    # Uses `#validate!` and catches `RangeError`
+    #
+    # @return [Boolean]
+    def valid?
+      validate!
+      true
+    rescue SyntaxError
+      false
     end
 
     # Iterate over each rule or terminal, except empty
@@ -173,64 +204,62 @@ module EBNF
     # @return [String]
     def to_sxp
       require 'sxp' unless defined?(SXP)
-      SXP::Generator.string(ast.sort_by{|r| r.id.to_f}.map(&:for_sxp))
+      SXP::Generator.string(ast.map(&:for_sxp))
     end
 
     ##
     # Output formatted EBNF
+    #
+    # @param [:abnf, :ebnf, :isoebnf] format (:ebnf)
     # @return [String]
-    def to_s
-      Writer.string(*ast)
+    def to_s(format: :ebnf)
+      Writer.string(*ast, format: format)
     end
 
     ##
     # Output formatted EBNF as HTML
+    #
+    # @param [:abnf, :ebnf, :isoebnf] format (:ebnf)
     # @return [String]
-    def to_html
-      Writer.html(*ast)
+    def to_html(format: :ebnf)
+      Writer.html(*ast, format: format)
     end
 
     ##
     # Output Ruby parser files
     #
     # @param [IO, StringIO] output
-    # @param [String] :grammarFile
-    # @param [String] :mod_name ('Branch')
-    def to_ruby(output = $stdout, grammarFile: nil, mod_name: 'Branch')
+    # @param [String] grammarFile
+    # @param [String] mod_name ('Meta')
+    def to_ruby(output = $stdout, grammarFile: nil, mod_name: 'Meta', **options)
       unless output == $stdout
-        output.puts "# This file is automatically generated by #{__FILE__}"
-        output.puts "# BRANCH derived from #{grammarFile}" if grammarFile
+        output.puts "# This file is automatically generated by ebnf version #{EBNF::VERSION}"
+        output.puts "# Derived from #{grammarFile}" if grammarFile
         unless self.errors.empty?
-          output.puts "# Note, tables completed with errors, may need to be resolved manually:"
+          output.puts "# Note, grammar has errors, may need to be resolved manually:"
           #output.puts "#   #{pp.conflicts.map{|c| c.join("\n#      ")}.join("\n#   ")}"
         end
         output.puts "module #{mod_name}"
-        output.puts "  START = #{self.start.inspect}"
-        output.puts
+        output.puts "  START = #{self.start.inspect}\n" if self.start
+      end
+
+      # Either output LL(1) BRANCH tables or rules for PEG parsing
+      if ast.first.first
+        to_ruby_ll1(output)
+      else
+        to_ruby_peg(output)
       end
-      self.outputTable(output, 'BRANCH', self.branch, 1)
-      self.outputTable(output, 'TERMINALS', self.terminals, 1)
-      self.outputTable(output, 'FIRST', self.first, 1)
-      self.outputTable(output, 'FOLLOW', self.follow, 1)
-      self.outputTable(output, 'CLEANUP', self.cleanup, 1)
-      self.outputTable(output, 'PASS', [self.pass], 1) if self.pass
       unless output == $stdout
         output.puts "end"
       end
     end
 
-    def dup
-      new_obj = super
-      new_obj.instance_variable_set(:@ast, @ast.dup)
-      new_obj
-    end
-
     ##
-    # Find a rule given a symbol
-    # @param [Symbol] sym
-    # @return [Rule]
-    def find_rule(sym)
-      (@find ||= {})[sym] ||= ast.detect {|r| r.sym == sym}
+    # Renumber, rule identifiers
+    def renumber!
+      ast.each_with_index do |rule, index|
+        rule.id = (index + 1).to_s
+      end
     end
 
     ##
@@ -241,6 +270,7 @@ module EBNF
     def to_ttl(prefix = nil, ns = "http://example.org/")
       unless ast.empty?
         [
+          "@prefix dc: <http://purl.org/dc/terms/>.",
           "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.",
           "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.",
           ("@prefix #{prefix}: <#{ns}>." if prefix),
@@ -253,7 +283,21 @@ module EBNF
         ].compact
       end.join("\n") +
 
-      ast.sort.map(&:to_ttl).join("\n")
+      ast.map(&:to_ttl).join("\n")
+    end
+
+    def dup
+      new_obj = super
+      new_obj.instance_variable_set(:@ast, @ast.dup)
+      new_obj
+    end
+
+    ##
+    # Find a rule given a symbol
+    # @param [Symbol] sym
+    # @return [Rule]
+    def find_rule(sym)
+      (@find ||= {})[sym] ||= ast.detect {|r| r.sym == sym}
     end
 
     def depth

data/lib/ebnf/bnf.rb

@@ -17,32 +17,7 @@ module EBNF
         new_ast += new_rules
       end
 
-      # Consolodate equivalent terminal rules
-      to_rewrite = {}
-      new_ast.select {|r| r.terminal?}.each do |src_rule|
-        new_ast.select {|r| r.terminal?}.each do |dst_rule|
-          if src_rule.equivalent?(dst_rule) && src_rule != dst_rule
-            debug("make_bnf") {"equivalent rules: #{src_rule.inspect} and #{dst_rule.inspect}"}
-            (to_rewrite[src_rule] ||= []) << dst_rule
-          end
-        end
-      end
-
-      # Replace references to equivalent rules with canonical rule
-      to_rewrite.each do |src_rule, dst_rules|
-        dst_rules.each do |dst_rule|
-          new_ast.each do |mod_rule|
-            debug("make_bnf") {"rewrite #{mod_rule.inspect} from #{dst_rule.sym} to #{src_rule.sym}"}
-            mod_rule.rewrite(dst_rule, src_rule)
-          end
-        end
-      end
-
-      # AST now has just rewritten rules
-      compacted_ast = new_ast - to_rewrite.values.flatten.compact
-
-      # Sort AST by number
-      @ast = compacted_ast
+      @ast = new_ast
       progress("make_bnf") {"End: #{@ast.length} rules"}
       self
     end

data/lib/ebnf/ebnf/meta.rb

@@ -0,0 +1,90 @@
+# This file is automatically generated by ebnf version 2.0.0
+# Derived from etc/ebnf.ebnf
+module EBNFMeta
+  RULES = [
+    EBNF::Rule.new(:ebnf, "1", [:star, :_ebnf_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_ebnf_1, "1.1", [:alt, :declaration, :rule]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:declaration, "2", [:alt, "@terminals", :pass]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:rule, "3", [:seq, :LHS, :expression]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:expression, "4", [:seq, :alt]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:alt, "5", [:seq, :seq, :_alt_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_alt_1, "5.1", [:star, :_alt_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_alt_2, "5.2", [:seq, "|", :seq]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:seq, "6", [:plus, :diff]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:diff, "7", [:seq, :postfix, :_diff_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_diff_1, "7.1", [:opt, :_diff_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_diff_2, "7.2", [:seq, "-", :postfix]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:postfix, "8", [:seq, :primary, :_postfix_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_postfix_1, "8.1", [:opt, :POSTFIX]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:primary, "9", [:alt, :HEX, :SYMBOL, :O_RANGE, :RANGE, :STRING1, :STRING2, :_primary_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_primary_1, "9.1", [:seq, "(", :expression, ")"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:pass, "10", [:seq, "@pass", :expression]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminals, nil, [:seq], kind: :terminals).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:LHS, "11", [:seq, :_LHS_1, :SYMBOL, :_LHS_2, "::="], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_1, "11.1", [:opt, :_LHS_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_3, "11.3", [:seq, "[", :SYMBOL, "]", :_LHS_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_4, "11.4", [:plus, " "], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_2, "11.2", [:star, " "], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:SYMBOL, "12", [:plus, :_SYMBOL_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_1, "12.1", [:alt, :_SYMBOL_2, :_SYMBOL_3, :_SYMBOL_4, "_", "."], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_2, "12.2", [:range, "a-z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_3, "12.3", [:range, "A-Z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_4, "12.4", [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:HEX, "13", [:seq, "#x", :_HEX_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_1, "13.1", [:plus, :_HEX_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_2, "13.2", [:alt, :_HEX_3, :_HEX_4, :_HEX_5], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_3, "13.3", [:range, "a-f"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_4, "13.4", [:range, "A-F"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_5, "13.5", [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:RANGE, "14", [:seq, "[", :_RANGE_1, :_RANGE_2, :_RANGE_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_1, "14.1", [:plus, :_RANGE_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_4, "14.4", [:alt, :_RANGE_5, :_RANGE_6, :R_CHAR, :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_5, "14.5", [:seq, :R_CHAR, "-", :R_CHAR], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_6, "14.6", [:seq, :HEX, "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_2, "14.2", [:opt, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_3, "14.3", [:diff, "]", :LHS], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:O_RANGE, "15", [:seq, "[^", :_O_RANGE_1, :_O_RANGE_2, "]"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_1, "15.1", [:plus, :_O_RANGE_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_3, "15.3", [:alt, :_O_RANGE_4, :_O_RANGE_5, :R_CHAR, :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_4, "15.4", [:seq, :R_CHAR, "-", :R_CHAR], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_5, "15.5", [:seq, :HEX, "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_2, "15.2", [:opt, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:STRING1, "16", [:seq, "\"", :_STRING1_1, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING1_1, "16.1", [:star, :_STRING1_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING1_2, "16.2", [:diff, :CHAR, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:STRING2, "17", [:seq, "'", :_STRING2_1, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING2_1, "17.1", [:star, :_STRING2_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING2_2, "17.2", [:diff, :CHAR, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:CHAR, "18", [:alt, :_CHAR_1, :_CHAR_2, :_CHAR_3, :_CHAR_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_1, "18.1", [:range, "#x9#xA#xD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_2, "18.2", [:range, "#x20-#xD7FF"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_3, "18.3", [:range, "#xE000-#xFFFD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_4, "18.4", [:range, "#x10000-#x10FFFF"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:R_CHAR, "19", [:diff, :CHAR, :_R_CHAR_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_R_CHAR_1, "19.1", [:alt, "]", "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:POSTFIX, "20", [:range, "?*+"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:PASS, "21", [:alt, :_PASS_1, :_PASS_2, :_PASS_3, :_PASS_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_1, "21.1", [:range, "#x9#xA#xD#x20"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_2, "21.2", [:seq, :_PASS_5, :_PASS_6], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_5, "21.5", [:alt, :_PASS_7, "//"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_7, "21.7", [:diff, "#", "#x"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_6, "21.6", [:star, :_PASS_8], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_8, "21.8", [:range, "^#xA#xD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_3, "21.3", [:seq, "/*", :_PASS_9, "*/"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_9, "21.9", [:star, :_PASS_10], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_10, "21.10", [:alt, :_PASS_11, :_PASS_12], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_11, "21.11", [:opt, :_PASS_13], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_13, "21.13", [:seq, "*", :_PASS_14], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_14, "21.14", [:range, "^/"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_12, "21.12", [:range, "^*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_4, "21.4", [:seq, "(*", :_PASS_15, "*)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_15, "21.15", [:star, :_PASS_16], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_16, "21.16", [:alt, :_PASS_17, :_PASS_18], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_17, "21.17", [:opt, :_PASS_19], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_19, "21.19", [:seq, "*", :_PASS_20], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_20, "21.20", [:range, "^)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_18, "21.18", [:range, "^*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_pass, nil, [:seq, :PASS], kind: :pass).extend(EBNF::PEG::Rule),
+  ]
+end
+

data/lib/ebnf/isoebnf.rb

@@ -0,0 +1,229 @@
+require_relative 'isoebnf/meta'
+require 'logger'
+
+# ISO EBNF parser
+# Parses ISO EBNF into an array of {EBNF::Rule}.
+module EBNF
+  class ISOEBNF
+    include EBNF::PEG::Parser
+
+    # The base for terminal-character, which omits "'", '"', and '?'.
+    # Could be more optimized, and one might quible
+    # with the overly-strictly defined character set,
+    # but it is correct.
+    TERMINAL_CHARACTER_BASE = %r{
+      [a-zA-Z0-9] | # letter | decimal digit
+      ,           | # concatenate symbol
+      =           | # defining symbol
+      [\|\/!]     | # definition separator symbol
+      \*\)        | # end comment symbol
+      \)          | # end group symbol
+      \]          | # end option symbol
+      \}          | # end repeat symbol
+      \-          | # except symbol
+      #\'          | # first quote symbol
+      \*          | # repetition symbol
+      #\"          | # second quote symbol
+      #\?          | # special sequence symbol
+      \(\*        | # start comment symbol
+      \(          | # start group symbol
+      \[          | # start option symbol
+      \{          | # start repeat symbol
+      [;\.]       | # terminator symbol
+      [:+_%@&$<>^\x20\x23\\`~]  # other character
+    }x
+
+    TERMINAL_CHARACTER         = %r{#{TERMINAL_CHARACTER_BASE}|['"\?]}
+    FIRST_TERMINAL_CHARACTER   = %r{#{TERMINAL_CHARACTER_BASE}|["\?]}
+    SECOND_TERMINAL_CHARACTER  = %r{#{TERMINAL_CHARACTER_BASE}|['\?]}
+    SPECIAL_SEQUENCE_CHARACTER = %r{#{TERMINAL_CHARACTER_BASE}|['"]}
+
+    # Abstract syntax tree from parse
+    #
+    # @return [Array<EBNF::Rule>]
+    attr_reader :ast
+
+    # `[14] integer           ::= decimal_digit+`
+    terminal(:integer, /\d+/) do |value, prod|
+      value.to_i
+    end
+
+    # `[15] meta_identifier             ::= letter meta_identifier_character*`
+    terminal(:meta_identifier, /[a-zA-Z][a-zA-Z0-9_]*/) do |value|
+      value.to_sym
+    end
+
+    # `[17] terminal_string             ::= ("'" first_terminal_character+ "'")`
+    # `                                   | ('"' second_terminal_character+ '"')`
+    terminal(:terminal_string, /(?:'#{FIRST_TERMINAL_CHARACTER}+')|(?:"#{SECOND_TERMINAL_CHARACTER}+")/x) do |value|
+      value[1..-2]
+    end
+
+    # `[20] special_sequence            ::= '?' special_sequence_character* '?'`
+    terminal(:special_sequence, /\?#{SPECIAL_SEQUENCE_CHARACTER}+\?/)
+
+    # `[22] terminal_character          ::= [a-zA-Z0-9]`
+    # `                                   | [,=;*}#x2d?([{;]`
+    # `                                   | '*)'`
+    # `                                   | '(*'`
+    # `                                   | ']'`
+    # `                                   | other_character`
+    terminal(:terminal_character, TERMINAL_CHARACTER)
+
+    # `[25] empty                      ::= ''`
+    terminal(:empty, //)
+
+    # `[26] definition_separator_symbol ::= '|' | '/' | '!'`
+    terminal(:definition_separator_symbol, /[\|\/!]/)
+
+    # `[27] terminator_symbol           ::= ';' | '.'`
+    terminal(:terminator_symbol, /[;\.]/)
+
+    # `[28] start_option_symbol         ::= '['
+    terminal(:start_option_symbol, /\[|(?:\(\/)/)
+
+    # `[29] end_option_symbol           ::= ']'`
+    terminal(:end_option_symbol, /\]/)
+
+    # `[30] start_repeat_symbol         ::= '{' | '(:'`
+    terminal(:start_repeat_symbol, /{|\(:/)
+
+    # `[31] end_repeat_symbol           ::= '}' | ':)'`
+    terminal(:end_repeat_symbol, /}|:\)/)
+
+    # ## Non-terminal productions
+
+    # `[2]  syntax_rule       ::= meta_identifier '=' definitions_list terminator_symbol`
+    production(:syntax_rule, clear_packrat: true) do |value, data, callback|
+      # value contains an expression.
+      # Invoke callback
+      sym = value[0][:meta_identifier]
+      definitions_list = value[2][:definitions_list]
+      callback.call(:rule, EBNF::Rule.new(sym.to_sym, nil, definitions_list))
+      nil
+    end
+
+    # Setting `as_hash: true` in the start production makes the value of the form of a hash, rather than an array of hashes.
+    #
+    # `[3]  definitions_list  ::= single_definition (definition_separator_symbol definitions_list)*`
+    start_production(:definitions_list, as_hash: true)
+    production(:definitions_list) do |value|
+      if value[:_definitions_list_1].length > 0
+        [:alt, value[:single_definition]] + value[:_definitions_list_1]
+      else
+        value[:single_definition]
+      end
+    end
+    production(:_definitions_list_1) do |value|
+      Array(value.first)
+    end
+    start_production(:_definitions_list_2, as_hash: true)
+    production(:_definitions_list_2) do |value|
+      if Array(value[:definitions_list]).first == :alt
+        value[:definitions_list][1..-1]
+      else
+        [value[:definitions_list]]
+      end
+    end
+
+    # `[4]  single_definition ::= term (',' term)*`
+    start_production(:single_definition, as_hash: true)
+    production(:single_definition) do |value|
+      if value[:_single_definition_1].length > 0
+        [:seq, value[:term]] + value[:_single_definition_1]
+      else
+        value[:term]
+      end
+    end
+    production(:_single_definition_1) do |value|
+      value.map {|a1| a1.last[:term]}.compact # Get rid of '|'
+    end
+
+    # `[5]  term              ::= factor ('-' exception)?`
+    start_production(:term, as_hash: true)
+    production(:term) do |value|
+      if value[:_term_1]
+        [:diff, value[:factor], value[:_term_1]]
+      else
+        value[:factor]
+      end
+    end
+    production(:_term_1) do |value|
+      value.last[:exception] if value
+    end
+
+    # `[6]  exception         ::= factor`
+    start_production(:exception, as_hash: true)
+    production(:exception) do |value|
+      value[:factor]
+    end
+
+    # `[7]  factor            ::= (integer '*')? primary`
+    start_production(:factor, as_hash: true)
+    production(:factor) do |value|
+      if value[:_factor_1]
+        [:rept, value[:_factor_1], value[:_factor_1], value[:primary]]
+      else
+        value[:primary]
+      end
+    end
+    production(:_factor_2) do |value|
+      value.first[:integer]
+    end
+
+    # `[9]  optional_sequence ::= start_option_symbol definitions_list end_option_symbol`
+    production(:optional_sequence) do |value|
+      [:opt, value[1][:definitions_list]]
+    end
+
+    # `[10] repeated_sequence ::= start_repeat_symbol definitions_list end_repeat_symbol`
+    production(:repeated_sequence) do |value|
+      [:star, value[1][:definitions_list]]
+    end
+
+    # `[11] grouped_sequence  ::= '(' definitions_list ')'`
+    production(:grouped_sequence) do |value|
+      [:seq, value[1][:definitions_list]]
+    end
+
+    # ## Parser invocation.
+    # On start, yield ourselves if a block is given, otherwise, return this parser instance
+    #
+    # @param  [#read, #to_s]          input
+    # @param  [Hash{Symbol => Object}] options
+    # @option options [Boolean] :level
+    #   Trace level. 0(debug), 1(info), 2(warn), 3(error).
+    # @return [EBNFParser]
+    def initialize(input, **options, &block)
+      # If the `level` option is set, instantiate a logger for collecting trace information.
+      if options.has_key?(:level)
+        options[:logger] = Logger.new(STDERR)
+        options[:logger].level = options[:level]
+        options[:logger].formatter = lambda {|severity, datetime, progname, msg| "#{severity} #{msg}\n"}
+      end
+
+      # Read input, if necessary, which will be used in a Scanner.
+      @input = input.respond_to?(:read) ? input.read : input.to_s
+
+      parsing_terminals = false
+      @ast = []
+      parse(@input,
+            :syntax,
+            ISOEBNFMeta::RULES,
+            whitespace: %r{([\x09-\x0d\x20]|(?:\(\*(?:(?:\*[^\)])|[^*])*\*\)))+},
+            **options
+      ) do |context, *data|
+        rule = case context
+        when :rule
+          # A rule which has already been turned into a `Rule` object.
+          rule = data.first
+          rule.kind = :terminal if parsing_terminals
+          rule
+        end
+        @ast << rule if rule
+      end
+    rescue EBNF::PEG::Parser::Error => e
+      raise SyntaxError, e.message
+    end
+  end
+end