ebnf 1.1.1 → 2.1.0

data/lib/ebnf/base.rb

@@ -2,7 +2,7 @@ require 'strscan'
 
 # Extended Bakus-Nour Form (EBNF), being the W3C variation is
 # originaly defined in the
-# [W3C XML 1.0 Spec](http://www.w3.org/TR/REC-xml/#sec-notation).
+# [W3C XML 1.0 Spec](https://www.w3.org/TR/REC-xml/#sec-notation).
 #
 # This version attempts to be less strict than the strict definition
 # to allow for coloquial variations (such as in the Turtle syntax).
@@ -12,8 +12,8 @@ require 'strscan'
 #
 # Comments include the content between '/*' and '*/'
 #
-# @see http://www.w3.org/2000/10/swap/grammar/ebnf2turtle.py
-# @see http://www.w3.org/2000/10/swap/grammar/ebnf2bnf.n3
+# @see https://www.w3.org/2000/10/swap/grammar/ebnf2turtle.py
+# @see https://www.w3.org/2000/10/swap/grammar/ebnf2bnf.n3
 #
 # Based on bnf2turtle by Dan Connolly.
 #
@@ -36,7 +36,7 @@ require 'strscan'
 # derived mechanically from the specification.
 # 
 # 
-# [N3 design note]: http://www.w3.org/DesignIssues/Notation3
+# [N3 design note]: https://www.w3.org/DesignIssues/Notation3
 # 
 # Related Work
 # ------------
@@ -59,31 +59,15 @@ require 'strscan'
 # expression of the grammar in terms of the higher level EBNF
 # constructs.
 # 
-# [goal]: http://www.w3.org/2002/02/mid/1086902566.21030.1479.camel@dirk;list=public-cwm-bugs
-# [n3p announcement]: http://lists.w3.org/Archives/Public/public-cwm-talk/2004OctDec/0029.html
-# [Yacker]: http://www.w3.org/1999/02/26-modules/User/Yacker
-# [SPARQL specification]: http://www.w3.org/TR/rdf-sparql-query/
-# [Cwm Release 1.1.0rc1]: http://lists.w3.org/Archives/Public/public-cwm-announce/2005JulSep/0000.html
-# [bnf-rules.n3]: http://www.w3.org/2000/10/swap/grammar/bnf-rules.n3
+# [goal]: https://www.w3.org/2002/02/mid/1086902566.21030.1479.camel@dirk;list=public-cwm-bugs
+# [n3p announcement]: https://lists.w3.org/Archives/Public/public-cwm-talk/2004OctDec/0029.html
+# [Yacker]: https://rubygems/02/26-modules/User/Yacker
+# [SPARQL specification]: https://www.w3.org/TR/rdf-sparql-query/
+# [Cwm Release 1.1.0rc1]: https://lists.w3.org/Archives/Public/public-cwm-announce/2005JulSep/0000.html
+# [bnf-rules.n3]: https://www.w3.org/2000/10/swap/grammar/bnf-rules.n3
 # 
-# Open Issues and Future Work
-# ---------------------------
-# 
-# The yacker output also has the terminals compiled to elaborate regular
-# expressions. The best strategy for dealing with lexical tokens is not
-# yet clear. Many tokens in SPARQL are case insensitive; this is not yet
-# captured formally.
-# 
-# The schema for the EBNF vocabulary used here (``g:seq``, ``g:alt``, ...)
-# is not yet published; it should be aligned with [swap/grammar/bnf][]
-# and the [bnf2html.n3][] rules (and/or the style of linked XHTML grammar
-# in the SPARQL and XML specificiations).
-# 
-# It would be interesting to corroborate the claim in the SPARQL spec
-# that the grammar is LL(1) with a mechanical proof based on N3 rules.
-# 
-# [swap/grammar/bnf]: http://www.w3.org/2000/10/swap/grammar/bnf
-# [bnf2html.n3]: http://www.w3.org/2000/10/swap/grammar/bnf2html.n3  
+# [swap/grammar/bnf]: https://www.w3.org/2000/10/swap/grammar/bnf
+# [bnf2html.n3]: https://www.w3.org/2000/10/swap/grammar/bnf2html.n3  
 # 
 # Background
 # ----------
@@ -93,14 +77,15 @@ require 'strscan'
 # of N3 that maps directly to (and from) the standard XML syntax for
 # RDF.
 #
-# [N3 Primer]: http://www.w3.org/2000/10/swap/Primer.html
+# [N3 Primer]: https://www.w3.org/2000/10/swap/Primer.html
 # 
 # @author Gregg Kellogg
 module EBNF
   class Base
     include BNF
     include LL1
-    include Parser
+    include Native
+    include PEG
 
     # Abstract syntax tree from parse
     #
@@ -116,24 +101,33 @@ module EBNF
     # in S-Expressions (similar to SPARQL SSE)
     #
     # @param [#read, #to_s] input
+    # @param [Symbol] format (:ebnf)
+    #   Format of input, one of `:abnf`, `:ebnf`, `:isoebnf`, `:isoebnf`, `:native`, or `:sxp`.
+    #   Use `:native` for the native EBNF parser, rather than the PEG parser.
     # @param [Hash{Symbol => Object}] options
     # @option options [Boolean, Array] :debug
-    #   Output debug information to an array or STDOUT.
-    # @option options [Symbol] :format (:ebnf)
-    #   Format of input, one of :ebnf, or :sxp
-    def initialize(input, options = {})
-      @options = {format: :ebnf}.merge(options)
+    #   Output debug information to an array or $stdout.
+    # @option options [Boolean, Array] :validate
+    #   Validate resulting grammar.
+    def initialize(input, format: :ebnf, **options)
+      @options = options.dup
       @lineno, @depth, @errors = 1, 0, []
-      terminal = false
       @ast = []
 
       input = input.respond_to?(:read) ? input.read : input.to_s
 
-      case @options[:format]
-      when :sxp
-        require 'sxp' unless defined?(SXP)
-        @ast = SXP::Reader::Basic.read(input).map {|e| Rule.from_sxp(e)}
+      case format
+      when :abnf
+        abnf = ABNF.new(input, **options)
+        @ast = abnf.ast
       when :ebnf
+        ebnf = Parser.new(input, **options)
+        @ast = ebnf.ast
+      when :isoebnf
+        iso = ISOEBNF.new(input, **options)
+        @ast = iso.ast
+      when :native
+        terminals = false
         scanner = StringScanner.new(input)
 
         eachRule(scanner) do |r|
@@ -141,23 +135,60 @@ module EBNF
           case r
           when /^@terminals/
             # Switch mode to parsing terminals
-            terminal = true
+            terminals = true
+            rule = Rule.new(nil, nil, nil, kind: :terminals, ebnf: self)
+            @ast << rule
           when /^@pass\s*(.*)$/m
             expr = expression($1).first
-            rule = Rule.new(nil, nil, expr, kind: :pass)
+            rule = Rule.new(nil, nil, expr, kind: :pass, ebnf: self)
             rule.orig = expr
             @ast << rule
           else
             rule = depth {ruleParts(r)}
 
-            rule.kind = :terminal if terminal # Override after we've parsed @terminals
+            rule.kind = :terminal if terminals # Override after we've parsed @terminals
             rule.orig = r
             @ast << rule
           end
         end
+      when :sxp
+        require 'sxp' unless defined?(SXP)
+        @ast = SXP::Reader::Basic.read(input).map {|e| Rule.from_sxp(e)}
       else
-        raise "unknown input format #{options[:format].inspect}"
+        raise "unknown input format #{format.inspect}"
+      end
+
+      validate! if @options[:validate]
+    end
+
+    ##
+    # Validate the grammar.
+    #
+    # Makes sure that rules reference either strings or other defined rules.
+    #
+    # @raise [RangeError]
+    def validate!
+      ast.each do |rule|
+        begin
+          rule.validate!(@ast)
+        rescue SyntaxError => e
+          error("In rule #{rule.sym}: #{e.message}")
+        end
       end
+      raise SyntaxError, errors.join("\n") unless errors.empty?
+    end
+
+    ##
+    # Is the grammar valid?
+    #
+    # Uses `#validate!` and catches `RangeError`
+    #
+    # @return [Boolean]
+    def valid?
+      validate!
+      true
+    rescue SyntaxError
+      false
     end
 
     # Iterate over each rule or terminal, except empty
@@ -173,64 +204,62 @@ module EBNF
     # @return [String]
     def to_sxp
       require 'sxp' unless defined?(SXP)
-      SXP::Generator.string(ast.sort_by{|r| r.id.to_f}.map(&:for_sxp))
+      SXP::Generator.string(ast.map(&:for_sxp))
     end
 
     ##
     # Output formatted EBNF
+    #
+    # @param [:abnf, :ebnf, :isoebnf] format (:ebnf)
     # @return [String]
-    def to_s
-      Writer.string(*ast)
+    def to_s(format: :ebnf)
+      Writer.string(*ast, format: format)
     end
 
     ##
     # Output formatted EBNF as HTML
+    #
+    # @param [:abnf, :ebnf, :isoebnf] format (:ebnf)
     # @return [String]
-    def to_html
-      Writer.html(*ast)
+    def to_html(format: :ebnf)
+      Writer.html(*ast, format: format)
     end
 
     ##
     # Output Ruby parser files
     #
     # @param [IO, StringIO] output
-    # @param [Hash{Symbol => Object}] options
-    # @option options [String] :grammarFile
-    def to_ruby(output = STDOUT, options = {})
-      unless output == STDOUT
-        output.puts "# This file is automatically generated by #{__FILE__}"
-        output.puts "# BRANCH derived from #{options[:grammarFile]}" if options[:grammarFile]
+    # @param [String] grammarFile
+    # @param [String] mod_name ('Meta')
+    def to_ruby(output = $stdout, grammarFile: nil, mod_name: 'Meta', **options)
+      unless output == $stdout
+        output.puts "# This file is automatically generated by ebnf version #{EBNF::VERSION}"
+        output.puts "# Derived from #{grammarFile}" if grammarFile
         unless self.errors.empty?
-          output.puts "# Note, tables completed with errors, may need to be resolved manually:"
+          output.puts "# Note, grammar has errors, may need to be resolved manually:"
           #output.puts "#   #{pp.conflicts.map{|c| c.join("\n#      ")}.join("\n#   ")}"
         end
-        output.puts "module #{options.fetch(:mod_name, 'Branch')}"
-        output.puts "  START = #{self.start.inspect}"
-        output.puts
+        output.puts "module #{mod_name}"
+        output.puts "  START = #{self.start.inspect}\n" if self.start
+      end
+
+      # Either output LL(1) BRANCH tables or rules for PEG parsing
+      if ast.first.first
+        to_ruby_ll1(output)
+      else
+        to_ruby_peg(output)
       end
-      self.outputTable(output, 'BRANCH', self.branch, 1)
-      self.outputTable(output, 'TERMINALS', self.terminals, 1)
-      self.outputTable(output, 'FIRST', self.first, 1)
-      self.outputTable(output, 'FOLLOW', self.follow, 1)
-      self.outputTable(output, 'CLEANUP', self.cleanup, 1)
-      self.outputTable(output, 'PASS', [self.pass], 1) if self.pass
-      unless output == STDOUT
+      unless output == $stdout
         output.puts "end"
       end
     end
 
-    def dup
-      new_obj = super
-      new_obj.instance_variable_set(:@ast, @ast.dup)
-      new_obj
-    end
-
     ##
-    # Find a rule given a symbol
-    # @param [Symbol] sym
-    # @return [Rule]
-    def find_rule(sym)
-      (@find ||= {})[sym] ||= ast.detect {|r| r.sym == sym}
+    # Renumber, rule identifiers
+    def renumber!
+      ast.each_with_index do |rule, index|
+        rule.id = (index + 1).to_s
+      end
     end
 
     ##
@@ -238,22 +267,37 @@ module EBNF
     # @param [String] prefix for language
     # @param [String] ns URI for language
     # @return [String]
-    def to_ttl(prefix, ns)
+    def to_ttl(prefix = nil, ns = "http://example.org/")
       unless ast.empty?
         [
+          "@prefix dc: <http://purl.org/dc/terms/>.",
           "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.",
           "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.",
-          "@prefix #{prefix}: <#{ns}>.",
+          ("@prefix #{prefix}: <#{ns}>." if prefix),
           "@prefix : <#{ns}>.",
           "@prefix re: <http://www.w3.org/2000/10/swap/grammar/regex#>.",
           "@prefix g: <http://www.w3.org/2000/10/swap/grammar/ebnf#>.",
           "",
           ":language rdfs:isDefinedBy <>; g:start :#{ast.first.id}.",
           "",
-        ]
+        ].compact
       end.join("\n") +
 
-      ast.sort.map(&:to_ttl).join("\n")
+      ast.map(&:to_ttl).join("\n")
+    end
+
+    def dup
+      new_obj = super
+      new_obj.instance_variable_set(:@ast, @ast.dup)
+      new_obj
+    end
+
+    ##
+    # Find a rule given a symbol
+    # @param [Symbol] sym
+    # @return [Rule]
+    def find_rule(sym)
+      (@find ||= {})[sym] ||= ast.detect {|r| r.sym == sym}
     end
 
     def depth
@@ -264,9 +308,8 @@ module EBNF
     end
 
     # Progress output, less than debugging
-    def progress(*args)
+    def progress(*args, **options)
       return unless @options[:progress] || @options[:debug]
-      options = args.last.is_a?(Hash) ? args.pop : {}
       depth = options[:depth] || @depth
       args << yield if block_given?
       message = "#{args.join(': ')}"
@@ -276,8 +319,7 @@ module EBNF
     end
 
     # Error output
-    def error(*args)
-      options = args.last.is_a?(Hash) ? args.pop : {}
+    def error(*args, **options)
       depth = options[:depth] || @depth
       args << yield if block_given?
       message = "#{args.join(': ')}"
@@ -298,9 +340,8 @@ module EBNF
     #   @param [String] message ("")
     #
     # @yieldreturn [String] added to message
-    def debug(*args)
+    def debug(*args, **options)
       return unless @options[:debug]
-      options = args.last.is_a?(Hash) ? args.pop : {}
       depth = options[:depth] || @depth
       args << yield if block_given?
       message = "#{args.join(': ')}"

data/lib/ebnf/bnf.rb

@@ -17,32 +17,7 @@ module EBNF
         new_ast += new_rules
       end
 
-      # Consolodate equivalent terminal rules
-      to_rewrite = {}
-      new_ast.select {|r| r.terminal?}.each do |src_rule|
-        new_ast.select {|r| r.terminal?}.each do |dst_rule|
-          if src_rule.equivalent?(dst_rule) && src_rule != dst_rule
-            debug("make_bnf") {"equivalent rules: #{src_rule.inspect} and #{dst_rule.inspect}"}
-            (to_rewrite[src_rule] ||= []) << dst_rule
-          end
-        end
-      end
-
-      # Replace references to equivalent rules with canonical rule
-      to_rewrite.each do |src_rule, dst_rules|
-        dst_rules.each do |dst_rule|
-          new_ast.each do |mod_rule|
-            debug("make_bnf") {"rewrite #{mod_rule.inspect} from #{dst_rule.sym} to #{src_rule.sym}"}
-            mod_rule.rewrite(dst_rule, src_rule)
-          end
-        end
-      end
-
-      # AST now has just rewritten rules
-      compacted_ast = new_ast - to_rewrite.values.flatten.compact
-
-      # Sort AST by number
-      @ast = compacted_ast
+      @ast = new_ast
       progress("make_bnf") {"End: #{@ast.length} rules"}
       self
     end

data/lib/ebnf/ebnf/meta.rb

@@ -0,0 +1,90 @@
+# This file is automatically generated by ebnf version 2.0.0
+# Derived from etc/ebnf.ebnf
+module EBNFMeta
+  RULES = [
+    EBNF::Rule.new(:ebnf, "1", [:star, :_ebnf_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_ebnf_1, "1.1", [:alt, :declaration, :rule]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:declaration, "2", [:alt, "@terminals", :pass]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:rule, "3", [:seq, :LHS, :expression]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:expression, "4", [:seq, :alt]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:alt, "5", [:seq, :seq, :_alt_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_alt_1, "5.1", [:star, :_alt_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_alt_2, "5.2", [:seq, "|", :seq]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:seq, "6", [:plus, :diff]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:diff, "7", [:seq, :postfix, :_diff_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_diff_1, "7.1", [:opt, :_diff_2]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_diff_2, "7.2", [:seq, "-", :postfix]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:postfix, "8", [:seq, :primary, :_postfix_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_postfix_1, "8.1", [:opt, :POSTFIX]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:primary, "9", [:alt, :HEX, :SYMBOL, :O_RANGE, :RANGE, :STRING1, :STRING2, :_primary_1]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_primary_1, "9.1", [:seq, "(", :expression, ")"]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:pass, "10", [:seq, "@pass", :expression]).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_terminals, nil, [:seq], kind: :terminals).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:LHS, "11", [:seq, :_LHS_1, :SYMBOL, :_LHS_2, "::="], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_1, "11.1", [:opt, :_LHS_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_3, "11.3", [:seq, "[", :SYMBOL, "]", :_LHS_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_4, "11.4", [:plus, " "], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_LHS_2, "11.2", [:star, " "], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:SYMBOL, "12", [:plus, :_SYMBOL_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_1, "12.1", [:alt, :_SYMBOL_2, :_SYMBOL_3, :_SYMBOL_4, "_", "."], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_2, "12.2", [:range, "a-z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_3, "12.3", [:range, "A-Z"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_SYMBOL_4, "12.4", [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:HEX, "13", [:seq, "#x", :_HEX_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_1, "13.1", [:plus, :_HEX_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_2, "13.2", [:alt, :_HEX_3, :_HEX_4, :_HEX_5], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_3, "13.3", [:range, "a-f"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_4, "13.4", [:range, "A-F"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_HEX_5, "13.5", [:range, "0-9"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:RANGE, "14", [:seq, "[", :_RANGE_1, :_RANGE_2, :_RANGE_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_1, "14.1", [:plus, :_RANGE_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_4, "14.4", [:alt, :_RANGE_5, :_RANGE_6, :R_CHAR, :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_5, "14.5", [:seq, :R_CHAR, "-", :R_CHAR], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_6, "14.6", [:seq, :HEX, "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_2, "14.2", [:opt, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_RANGE_3, "14.3", [:diff, "]", :LHS], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:O_RANGE, "15", [:seq, "[^", :_O_RANGE_1, :_O_RANGE_2, "]"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_1, "15.1", [:plus, :_O_RANGE_3], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_3, "15.3", [:alt, :_O_RANGE_4, :_O_RANGE_5, :R_CHAR, :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_4, "15.4", [:seq, :R_CHAR, "-", :R_CHAR], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_5, "15.5", [:seq, :HEX, "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_O_RANGE_2, "15.2", [:opt, "-"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:STRING1, "16", [:seq, "\"", :_STRING1_1, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING1_1, "16.1", [:star, :_STRING1_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING1_2, "16.2", [:diff, :CHAR, "\""], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:STRING2, "17", [:seq, "'", :_STRING2_1, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING2_1, "17.1", [:star, :_STRING2_2], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_STRING2_2, "17.2", [:diff, :CHAR, "'"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:CHAR, "18", [:alt, :_CHAR_1, :_CHAR_2, :_CHAR_3, :_CHAR_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_1, "18.1", [:range, "#x9#xA#xD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_2, "18.2", [:range, "#x20-#xD7FF"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_3, "18.3", [:range, "#xE000-#xFFFD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_CHAR_4, "18.4", [:range, "#x10000-#x10FFFF"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:R_CHAR, "19", [:diff, :CHAR, :_R_CHAR_1], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_R_CHAR_1, "19.1", [:alt, "]", "-", :HEX], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:POSTFIX, "20", [:range, "?*+"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:PASS, "21", [:alt, :_PASS_1, :_PASS_2, :_PASS_3, :_PASS_4], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_1, "21.1", [:range, "#x9#xA#xD#x20"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_2, "21.2", [:seq, :_PASS_5, :_PASS_6], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_5, "21.5", [:alt, :_PASS_7, "//"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_7, "21.7", [:diff, "#", "#x"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_6, "21.6", [:star, :_PASS_8], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_8, "21.8", [:range, "^#xA#xD"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_3, "21.3", [:seq, "/*", :_PASS_9, "*/"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_9, "21.9", [:star, :_PASS_10], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_10, "21.10", [:alt, :_PASS_11, :_PASS_12], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_11, "21.11", [:opt, :_PASS_13], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_13, "21.13", [:seq, "*", :_PASS_14], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_14, "21.14", [:range, "^/"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_12, "21.12", [:range, "^*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_4, "21.4", [:seq, "(*", :_PASS_15, "*)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_15, "21.15", [:star, :_PASS_16], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_16, "21.16", [:alt, :_PASS_17, :_PASS_18], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_17, "21.17", [:opt, :_PASS_19], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_19, "21.19", [:seq, "*", :_PASS_20], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_20, "21.20", [:range, "^)"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_PASS_18, "21.18", [:range, "^*"], kind: :terminal).extend(EBNF::PEG::Rule),
+    EBNF::Rule.new(:_pass, nil, [:seq, :PASS], kind: :pass).extend(EBNF::PEG::Rule),
+  ]
+end
+