ebnf 0.3.1 → 0.3.2

data/etc/turtle.sxp

@@ -1,14 +1,13 @@
-
-( 
+(
  (rule turtleDoc "1" (star statement))
  (rule statement "2" (alt directive (seq triples ".")))
  (rule directive "3" (alt prefixID base sparqlPrefix sparqlBase))
  (rule prefixID "4" (seq "@prefix" PNAME_NS IRIREF "."))
  (rule base "5" (seq "@base" IRIREF "."))
  (rule triples "6"
-  (alt 
+  (alt
    (seq subject predicateObjectList)
-   (seq blankNodePropertyList (opt predicateObjectList))))
+   (seq blankNodePropertyList (opt predicateObjectList))) )
  (rule predicateObjectList "7"
   (seq verb objectList (star (seq ";" (opt (seq verb objectList))))))
  (rule objectList "8" (seq object (star (seq "," object))))
@@ -21,29 +20,29 @@
  (rule collection "15" (seq "(" (star object) ")"))
  (rule NumericLiteral "16" (alt INTEGER DECIMAL DOUBLE))
  (rule String "17"
-  (alt
- STRING_LITERAL_QUOTE STRING_LITERAL_SINGLE_QUOTE STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_LONG_QUOTE  ))
- (terminal IRIREF "18" (seq "<" (star (alt (range "^#x00-#x20<>\"{}|^`\\") UCHAR)) ">"))
+  (alt STRING_LITERAL_QUOTE STRING_LITERAL_SINGLE_QUOTE
+   STRING_LITERAL_LONG_SINGLE_QUOTE STRING_LITERAL_LONG_QUOTE ))
+ (terminal IRIREF "18" (seq "<" (range "^#x00-#x20<>\"{}|^`] | UCHAR)* '>'")))
  (terminal INTEGER "19" (seq (opt (range "+-")) (plus (range "0-9"))))
  (terminal DECIMAL "20"
   (seq (opt (range "+-")) (seq (star (range "0-9")) "." (plus (range "0-9")))))
  (terminal DOUBLE "21"
-  (seq 
+  (seq
    (opt (range "+-"))
-   (alt 
+   (alt
     (seq (plus (range "0-9")) "." (star (range "0-9")) EXPONENT)
     (seq "." (plus (range "0-9")) EXPONENT)
-    (seq (plus (range "0-9")) EXPONENT))))
+    (seq (plus (range "0-9")) EXPONENT)) ))
  (terminal STRING_LITERAL_QUOTE "22"
   (seq "\"" (star (alt (range "^#x22#x5C#xA#xD") ECHAR UCHAR)) "\""))
  (terminal STRING_LITERAL_SINGLE_QUOTE "23"
   (seq "'" (star (alt (range "^#x27#x5C#xA#xD") ECHAR UCHAR)) "'"))
  (terminal STRING_LITERAL_LONG_SINGLE_QUOTE "24"
-  (seq "'''" (star (seq (opt (alt "'" "''")) (alt (range "^'\\") ECHAR UCHAR))) "'''"))
+  (seq "'''" (seq (opt (alt "'" "''")) (range "^'] | ECHAR | UCHAR ))* \"'''\""))))
  (terminal STRING_LITERAL_LONG_QUOTE "25"
-  (seq "\"\"\"" (star (seq (opt (alt "\"" "\"\"")) (alt (range "^\"\\") ECHAR UCHAR))) "\"\"\""))
+  (seq "\"\"\"" (seq (opt (alt "\"" "\"\"")) (range "^\"] | ECHAR | UCHAR ))* '\"\"\"'"))))
  (terminal UCHAR "26"
-  (alt (seq "\\u" HEX HEX HEX HEX) (seq "\\U" HEX HEX HEX HEX HEX HEX HEX HEX)))
+  (alt (seq "u" HEX HEX HEX HEX) (seq "U" HEX HEX HEX HEX HEX HEX HEX HEX)))
  (rule sparqlPrefix "28s" (seq SPARQL_PREFIX PNAME_NS IRIREF))
  (terminal SPARQL_PREFIX "28t"
   (seq (range "Pp") (range "Rr") (range "Ee") (range "Ff") (range "Ii") (range "Xx")))
@@ -61,11 +60,11 @@
  (terminal LANGTAG "144s"
   (seq "@" (plus (range "a-zA-Z")) (star (seq "-" (plus (range "a-zA-Z0-9"))))))
  (terminal EXPONENT "154s" (seq (range "eE") (opt (range "+-")) (plus (range "0-9"))))
- (terminal ECHAR "159s" (seq "\\" (range "tbnrf\\\"'")))
+ (terminal ECHAR "159s" (seq "\\" (range "tbnrf\"'")))
  (terminal WS "161s" (alt (hex "#x20") (hex "#x9") (hex "#xD") (hex "#xA")))
  (terminal ANON "162s" (seq "[" (star WS) "]"))
  (terminal PN_CHARS_BASE "163s"
-  (alt 
+  (alt
    (range "A-Z")
    (range "a-z")
    (range "#x00C0-#x00D6")
@@ -79,24 +78,24 @@
    (range "#x3001-#xD7FF")
    (range "#xF900-#xFDCF")
    (range "#xFDF0-#xFFFD")
-   (range "#x10000-#xEFFFF")))
+   (range "#x10000-#xEFFFF")) )
  (terminal PN_CHARS_U "164s" (alt PN_CHARS_BASE "_"))
  (terminal PN_CHARS "166s"
   (alt PN_CHARS_U "-"
    (range "0-9")
    (hex "#x00B7")
    (range "#x0300-#x036F")
-   (range "#x203F-#x2040")))
+   (range "#x203F-#x2040")) )
  (terminal PN_PREFIX "167s"
   (seq PN_CHARS_BASE (opt (seq (star (alt PN_CHARS ".")) PN_CHARS))))
  (terminal PN_LOCAL "168s"
-  (seq 
+  (seq
    (alt PN_CHARS_U ":" (range "0-9") PLX)
-   (opt (seq (star (alt PN_CHARS "." ":" PLX)) (alt PN_CHARS ":" PLX)))))
+   (opt (seq (star (alt PN_CHARS "." ":" PLX)) (alt PN_CHARS ":" PLX)))) )
  (terminal PLX "169s" (alt PERCENT PN_LOCAL_ESC))
  (terminal PERCENT "170s" (seq "%" HEX HEX))
  (terminal HEX "171s" (alt (range "0-9") (range "A-F") (range "a-f")))
  (terminal PN_LOCAL_ESC "172s"
   (seq "\\"
-   (alt
- "_" "~" "." "-" "!" "$" "&" "'" "(" ")" "*" "+" "," ";" "=" "/" "?" "#" "@" "%"   ))))
+   (alt "_" "~" "." "-" "!" "$" "&" "'" "(" ")" "*" "+" "," ";" "=" "/" "?" "#"
+    "@" "%" )) ))

data/lib/ebnf/base.rb

@@ -119,30 +119,44 @@ module EBNF
     # @param [Hash{Symbol => Object}] options
     # @option options [Boolean, Array] :debug
     #   Output debug information to an array or STDOUT.
+    # @option options [Symbol] :format (:ebnf)
+    #   Format of input, one of :ebnf, or :sxp
     def initialize(input, options = {})
-      @options = options
+      @options = {:format => :ebnf}.merge(options)
       @lineno, @depth, @errors = 1, 0, []
       terminal = false
       @ast = []
 
       input = input.respond_to?(:read) ? input.read : input.to_s
-      scanner = StringScanner.new(input)
-
-      eachRule(scanner) do |r|
-        debug("rule string") {r.inspect}
-        case r
-        when /^@terminals/
-          # Switch mode to parsing terminals
-          terminal = true
-        when /^@pass\s*(.*)$/m
-          # Ignore, as we can't use this for processing the EBNF grammar itself
-        else
-          rule = depth {ruleParts(r)}
-
-          rule.kind = :terminal if terminal # Override after we've parsed @terminals
-          rule.orig = r
-          @ast << rule
+
+      case @options[:format]
+      when :sxp
+        require 'sxp' unless defined?(SXP)
+        @ast = SXP::Reader::Basic.read(input).map {|e| Rule.from_sxp(e)}
+      when :ebnf
+        scanner = StringScanner.new(input)
+
+        eachRule(scanner) do |r|
+          debug("rule string") {r.inspect}
+          case r
+          when /^@terminals/
+            # Switch mode to parsing terminals
+            terminal = true
+          when /^@pass\s*(.*)$/m
+            expr = expression($1).first
+            rule = Rule.new(nil, nil, expr, :kind => :pass)
+            rule.orig = expr
+            @ast << rule
+          else
+            rule = depth {ruleParts(r)}
+
+            rule.kind = :terminal if terminal # Override after we've parsed @terminals
+            rule.orig = r
+            @ast << rule
+          end
         end
+      else
+        raise "unknown input format #{options[:format].inspect}"
       end
     end
 
@@ -158,6 +172,7 @@ module EBNF
     # Write out parsed syntax string as an S-Expression
     # @return [String]
     def to_sxp
+      require 'sxp' unless defined?(SXP)
       SXP::Generator.string(ast.sort_by{|r| r.id.to_f}.map(&:for_sxp))
     end
     def to_s; to_sxp; end

data/lib/ebnf/ll1.rb

@@ -21,12 +21,21 @@ module EBNF
     attr_reader :follow
 
     # Terminal table
+    #
     # The list of terminals used in the grammar.
     #
     # @return [Array<String, Symbol>]
     attr_reader :terminals
 
+    # Pass expression
+    #
+    # A Terminal symbol used for skipping whitespace and comments
+    #
+    # @return [Symbol, String]
+    attr_reader :pass
+
     # Start symbol
+    #
     # The rule which starts the grammar
     #
     # @return [Symbol]
@@ -69,7 +78,6 @@ module EBNF
 
           @ast += comprehensions
           progress("FF.c") {"(#{ittr}) comprehensions #{comprehensions.length}"}
-          #require 'debugger'; breakpoint
           ittr += 1
         end while !comprehensions.empty?
 
@@ -128,7 +136,6 @@ module EBNF
               end
 
               # If there is no comprehension of this rule (meaning, it is a sequence of one non-terminal), then the follows of the non-terminal include the follows of the rule. This handles rules with multiple sequences because it will have a comprehension that includes the last element in the sequence
-              #require 'debugger'; breakpoint if ai.sym == :_predicateObjectList_1 && aj.sym == :_predicateObjectList_7
               if !aj.comp && aj.follow
                 debug("Fo.2.1a") {"(#{ittr}) add follow #{aj.follow.inspect} from #{aj.sym} to #{ai.sym}"}
                 follows += ai.add_follow(aj.follow)
@@ -142,7 +149,6 @@ module EBNF
             end
 
             # Since the rules are of the form wAiw', and we've handled the case which is just Aiw', this leaves those cases that have rules prior to Ai. This basically says that the follows of a rule are added to the follows of the comprehension of the rule
-            #require 'debugger'; breakpoint if aj.sym == :_predicateObjectList_6 && aj.follow
             if aj.comp && aj.follow
               debug("Fo.2.3") {"(#{ittr}) add follow #{aj.follow.inspect} from #{aj.sym} to #{aj.comp.sym}"}
               follows += aj.comp.add_follow(aj.follow)
@@ -175,11 +181,26 @@ module EBNF
           memo[r.sym] = r.follow if r.follow
           memo
         }
-      @terminals = ast.map do |r|
-        (r.first || []) + (r.follow || [])
-      end.flatten.uniq
+      @terminals = ast.map {|r| Array(r.first) + Array(r.follow)}.flatten.uniq
       @terminals = (@terminals - [:_eps, :_eof]).sort_by{|t| t.to_s.sub(/^_/, '')}
 
+      # FIXME: assumes that this is a (seq :PASS), or similar
+      if pass = ast.detect {|r| r.pass?}
+        @pass = pass.expr.last
+      end
+
+      # If a generated terminal is found, this indicates an error, as this version does not automatically generate regular expressions for automatic terminals
+      @terminals.
+        select {|t| t.to_s.start_with?("_")}.
+        reject {|t| t.to_s.start_with?("_pass_")}.  # Concession to backwards compatibility
+        each do |term|
+
+        error("build_tables",
+              "terminal #{term} is automatically generated; " +
+              "regular expressions are not yet generated and parsing " +
+              "is not supported")
+      end
+
       @branch = {}
       @already = []
       @agenda = []
@@ -256,10 +277,18 @@ module EBNF
       end
 
       if rule.alt?
+        # A First/Follow conflict appears when _eps is in the first
+        # of one rule and there is a token in the first and
+        # follow of the same rule
+        if rule.first.include?(:_eps) && !(overlap = ((rule.first & rule.follow) - [:eps])).empty?
+          error("First/Follow Conflict: #{overlap.first.inspect} is both first and follow of #{rule.sym}")
+        end
+
         # Add entries for each alternative, based on the alternative's first/seq
         rule.expr[1..-1].each do |prod|
           prod_rule = find_rule(prod)
           debug("  Alt", prod)
+
           @agenda << prod unless @already.include?(prod) || @agenda.include?(prod)
           if prod == :_empty
             debug("    empty")
@@ -269,9 +298,12 @@ module EBNF
             branchDict[prod] = [prod]
           else
             prod_rule.first.reject{|f| f == :_eps}.each do |f|
+              # A First/First conflict appears when there are two rules having
+              # the same first, so the parser can't know which one to choose.
               if branchDict.has_key?(f)
-                error("First/First Conflict: #{f} is also the condition for #{branchDict[f]}")
+                error("First/First Conflict: #{f.inspect} is also the condition for #{branchDict[f].first}")
               end
+
               debug("   alt") {"[#{f}] => #{prod}"}
               branchDict[f] = [prod]
             end

data/lib/ebnf/ll1/lexer.rb

@@ -47,19 +47,11 @@ module EBNF::LL1
     ESCAPE_CHAR8        = /\\U(?:[0-9A-Fa-f]{8,8})/.freeze    # \UXXXXXXXX
     ECHAR               = /\\./                               # More liberal unescaping
     UCHAR               = /#{ESCAPE_CHAR4}|#{ESCAPE_CHAR8}/.freeze
-    COMMENT             = /#.*/.freeze
-    WS                  = / |\t|\r|\n/m.freeze
-
-    ML_START            = /\'\'\'|\"\"\"/.freeze              # Beginning of terminals that may span lines
 
     ##
-    # @return [Regexp] defines whitespace, defaults to WS
+    # @return [Regexp] defines whitespace, including comments, otherwise whitespace must be explicit in terminals
     attr_reader :whitespace
 
-    ##
-    # @return [Regexp] defines single-line comment, defaults to COMMENT
-    attr_reader :comment
-
     ##
     # Returns a copy of the given `input` string with all `\uXXXX` and
     # `\UXXXXXXXX` Unicode codepoint escape sequences replaced with their
@@ -119,13 +111,11 @@ module EBNF::LL1
     #   Array of symbol, regexp pairs used to match terminals.
     #   If the symbol is nil, it defines a Regexp to match string terminals.
     # @param  [Hash{Symbol => Object}]        options
-    # @option options [Regexp]                :whitespace (WS)
-    #   Regular expression matching the beginning of terminals that may cross newlines
-    # @option options [Regexp]                :comment (COMMENT)
+    # @option options [Regexp]                :whitespace
+    #   Whitespace between tokens, including comments
     def initialize(input = nil, terminals = nil, options = {})
       @options        = options.dup
-      @whitespace     = @options[:whitespace]     || WS
-      @comment        = @options[:comment]        || COMMENT
+      @whitespace     = @options[:whitespace]
       @terminals      = terminals.map do |term|
         term.is_a?(Array) ? Terminal.new(*term) : term
       end
@@ -199,7 +189,7 @@ module EBNF::LL1
         token = match_token
 
         if token.nil?
-          lexme = (scanner.rest.split(/#{@whitespace}|#{@comment}/).first rescue nil) || scanner.rest
+          lexme = (scanner.rest.split(@whitespace || /\s/).first rescue nil) || scanner.rest
           raise Error.new("Invalid token #{lexme[0..100].inspect}",
             :input => scanner.rest[0..100], :token => lexme, :lineno => lineno)
         end
@@ -232,7 +222,7 @@ module EBNF::LL1
     # @return [Token]
     def recover
        until scanner.eos? || tok = match_token
-        if scanner.skip_until(@whitespace).nil? # Skip past current "token"
+        if scanner.skip_until(@whitespace || /\s/m).nil? # Skip past current "token"
           # No whitespace at the end, must be and end of string
           scanner.terminate
         else
@@ -248,14 +238,12 @@ module EBNF::LL1
     attr_reader :scanner
 
     ##
-    # Skip whitespace or comments, as defined through input options or defaults
+    # Skip whitespace, as defined through input options or defaults
     def skip_whitespace
       # skip all white space, but keep track of the current line number
-      while !scanner.eos?
+      while @whitespace && !scanner.eos?
         if matched = scanner.scan(@whitespace)
           @lineno += matched.count("\n")
-        elsif (scanner.scan(@comment))
-          #
         else
           return
         end

data/lib/ebnf/ll1/parser.rb

@@ -219,6 +219,10 @@ module EBNF::LL1
     # @see http://cs.adelaide.edu.au/~charles/lt/Lectures/07-ErrorRecovery.pdf
     def parse(input = nil, start = nil, options = {}, &block)
       @options = options.dup
+      @options[:debug] ||= case
+      when @options[:progress] then 2
+      when @options[:validate] then 1
+      end
       @branch  = options[:branch]
       @first  = options[:first] ||= {}
       @follow  = options[:follow] ||= {}

data/lib/ebnf/parser.rb

@@ -61,7 +61,7 @@ module EBNF
       num_sym, expr = rule.split('::=', 2).map(&:strip)
       num, sym = num_sym.split(']', 2).map(&:strip)
       num = num[1..-1]
-      r = Rule.new(sym && sym.to_sym, num, ebnf(expr).first, :ebnf => self)
+      r = Rule.new(sym && sym.to_sym, num, expression(expr).first, :ebnf => self)
       debug("ruleParts") { r.inspect }
       r
     end
@@ -70,40 +70,44 @@ module EBNF
     # Parse a string into an expression tree and a remaining string
     #
     # @example
-    #     >>> ebnf("a b c")
-    #     ((seq, \[('id', 'a'), ('id', 'b'), ('id', 'c')\]), '')
+    #     >>> expression("a b c")
+    #     ((seq a b c) '')
     #     
-    #     >>> ebnf("a? b+ c*")
-    #     ((seq, \[(opt, ('id', 'a')), (plus, ('id', 'b')), ('*', ('id', 'c'))\]), '')
+    #     >>> expression("a? b+ c*")
+    #     ((seq (opt a) (plus b) (star c)) '')
     #     
-    #     >>> ebnf(" | x xlist")
-    #     ((alt, \[(seq, \[\]), (seq, \[('id', 'x'), ('id', 'xlist')\])\]), '')
+    #     >>> expression(" | x xlist")
+    #     ((alt (seq) (seq x xlist)) '')
     #     
-    #     >>> ebnf("a | (b - c)")
-    #     ((alt, \[('id', 'a'), (diff, \[('id', 'b'), ('id', 'c')\])\]), '')
+    #     >>> expression("a | (b - c)")
+    #     ((alt a (diff b c)) '')
     #     
-    #     >>> ebnf("a b | c d")
-    #     ((alt, \[(seq, \[('id', 'a'), ('id', 'b')\]), (seq, \[('id', 'c'), ('id', 'd')\])\]), '')
+    #     >>> expression("a b | c d")
+    #     ((alt (seq a b) (seq c d)) '')
     #     
-    #     >>> ebnf("a | b | c")
-    #     ((alt, \[('id', 'a'), ('id', 'b'), ('id', 'c')\]), '')
+    #     >>> expression("a | b | c")
+    #     ((alt a b c) '')
     #     
-    #     >>> ebnf("a) b c")
-    #     (('id', 'a'), ' b c')
+    #     >>> expression("a) b c")
+    #     (a ' b c')
     #     
-    #     >>> ebnf("BaseDecl? PrefixDecl*")
-    #     ((seq, \[(opt, ('id', 'BaseDecl')), ('*', ('id', 'PrefixDecl'))\]), '')
+    #     >>> expression("BaseDecl? PrefixDecl*")
+    #     ((seq (opt BaseDecl) (star PrefixDecl)) '')
     #     
-    #     >>> ebnf("NCCHAR1 | diff | [0-9] | #x00B7 | [#x0300-#x036F] | \[#x203F-#x2040\]")
-    #     ((alt, \[('id', 'NCCHAR1'), ("'", diff), (range, '0-9'), (hex, '#x00B7'), (range, '#x0300-#x036F'), (range, '#x203F-#x2040')\]), '')
+    #     >>> expression("NCCHAR1 | diff | [0-9] | #x00B7 | [#x0300-#x036F] | \[#x203F-#x2040\]")
+    #     ((alt NCCHAR1 diff
+    #           (range '0-9')
+    #           (hex '#x00B7')
+    #           (range '#x0300-#x036F')
+    #           (range, '#x203F-#x2040')) '')
     #     
     # @param [String] s
     # @return [Array]
-    def ebnf(s)
-      debug("ebnf") {"(#{s.inspect})"}
+    def expression(s)
+      debug("expression") {"(#{s.inspect})"}
       e, s = depth {alt(s)}
       debug {"=> alt returned #{[e, s].inspect}"}
-      unless s.empty?
+      unless s.to_s.empty?
         t, ss = depth {terminal(s)}
         debug {"=> terminal returned #{[t, ss].inspect}"}
         return [e, ss] if t.is_a?(Array) && t.first == :")"
@@ -114,13 +118,13 @@ module EBNF
     ##
     # Parse alt
     #     >>> alt("a | b | c")
-    #     ((alt, \[('id', 'a'), ('id', 'b'), ('id', 'c')\]), '')
+    #     ((alt a b c) '')
     # @param [String] s
     # @return [Array]
     def alt(s)
       debug("alt") {"(#{s.inspect})"}
       args = []
-      while !s.empty?
+      while !s.to_s.empty?
         e, s = depth {seq(s)}
         debug {"=> seq returned #{[e, s].inspect}"}
         if e.to_s.empty?
@@ -128,7 +132,7 @@ module EBNF
           e = [:seq, []] # empty sequence
         end
         args << e
-        unless s.empty?
+        unless s.to_s.empty?
           t, ss = depth {terminal(s)}
           break unless t[0] == :alt
           s = ss
@@ -141,14 +145,14 @@ module EBNF
     # parse seq
     #
     #     >>> seq("a b c")
-    #     ((seq, \[('id', 'a'), ('id', 'b'), ('id', 'c')\]), '')
+    #     ((seq a b c) '')
     #     
     #     >>> seq("a b? c")
-    #     ((seq, \[('id', 'a'), (opt, ('id', 'b')), ('id', 'c')\]), '')
+    #     ((seq a (opt b) c) '')
     def seq(s)
       debug("seq") {"(#{s.inspect})"}
       args = []
-      while !s.empty?
+      while !s.to_s.empty?
         e, ss = depth {diff(s)}
         debug {"=> diff returned #{[e, ss].inspect}"}
         unless e.to_s.empty?
@@ -171,13 +175,13 @@ module EBNF
     # parse diff
     # 
     #     >>> diff("a - b")
-    #     ((diff, \[('id', 'a'), ('id', 'b')\]), '')
+    #     ((diff a b) '')
     def diff(s)
       debug("diff") {"(#{s.inspect})"}
       e1, s = depth {postfix(s)}
       debug {"=> postfix returned #{[e1, s].inspect}"}
       unless e1.to_s.empty?
-        unless s.empty?
+        unless s.to_s.empty?
           t, ss = depth {terminal(s)}
           debug {"diff #{[t, ss].inspect}"}
           if t.is_a?(Array) && t.first == :diff
@@ -199,16 +203,16 @@ module EBNF
     # parse postfix
     # 
     #     >>> postfix("a b c")
-    #     (('id', 'a'), ' b c')
+    #     (a ' b c')
     #     
     #     >>> postfix("a? b c")
-    #     ((opt, ('id', 'a')), ' b c')
+    #     ((opt, a) ' b c')
     def postfix(s)
       debug("postfix") {"(#{s.inspect})"}
       e, s = depth {primary(s)}
       debug {"=> primary returned #{[e, s].inspect}"}
       return ["", s] if e.to_s.empty?
-      if !s.empty?
+      if !s.to_s.empty?
         t, ss = depth {terminal(s)}
         debug {"=> #{[t, ss].inspect}"}
         if t.is_a?(Array) && [:opt, :star, :plus].include?(t.first)
@@ -222,7 +226,7 @@ module EBNF
     # parse primary
     # 
     #     >>> primary("a b c")
-    #     (('id', 'a'), ' b c')
+    #     (a ' b c')
     def primary(s)
       debug("primary") {"(#{s.inspect})"}
       t, s = depth {terminal(s)}
@@ -232,8 +236,8 @@ module EBNF
       elsif %w(range hex).map(&:to_sym).include?(t.first)
         [t, s]
       elsif t.first == :"("
-        e, s = depth {ebnf(s)}
-        debug {"=> ebnf returned #{[e, s].inspect}"}
+        e, s = depth {expression(s)}
+        debug {"=> expression returned #{[e, s].inspect}"}
         [e, s]
       else
         ["", s]
@@ -248,34 +252,34 @@ module EBNF
     # 
     # @example
     #     >>> terminal("'abc' def")
-    #     (("'", 'abc'), ' def')
+    #     ('abc' ' def')
     #     
     #     >>> terminal("[0-9]")
-    #     ((range, '0-9'), '')
+    #     ((range '0-9') '')
     #     >>> terminal("#x00B7")
-    #     ((hex, '#x00B7'), '')
+    #     ((hex '#x00B7') '')
     #     >>> terminal ("\[#x0300-#x036F\]")
-    #     ((range, '#x0300-#x036F'), '')
+    #     ((range '#x0300-#x036F') '')
     #     >>> terminal("\[^<>'{}|^`\]-\[#x00-#x20\]")
-    #     ((range, "^<>'{}|^`"), '-\[#x00-#x20\]')
+    #     ((range "^<>'{}|^`") '-\[#x00-#x20\]')
     def terminal(s)
       s = s.strip
       case m = s[0,1]
-      when '"', "'"
-        l, s = s[1..-1].split(m, 2)
-        [l, s]
-      when '['
-        l, s = s[1..-1].split(']', 2)
-        [[:range, l], s]
-      when '#'
+      when '"', "'" # STRING1 or STRING2 Terminated by line-end or whitespace
+        l, s = s[1..-1].split(m.rstrip  , 2)
+        [LL1::Lexer.unescape_string(l), s]
+      when '[' # ENUM, RANGE, O_ENUM, or O_RANGE
+        l, s = s[1..-1].split(/(?<=[^\\])\]/, 2)
+        [[:range, LL1::Lexer.unescape_string(l)], s]
+      when '#' # HEX
         s.match(/(#\w+)(.*)$/)
         l, s = $1, $2
         [[:hex, l], s]
-      when /[\w\.]/
+      when /[\w\.]/ # SYMBOL
         s.match(/(\w+)(.*)$/)
         l, s = $1, $2
         [l.to_sym, s]
-      when '@'
+      when '@' # @pass or @terminals
         s.match(/@(#\w+)(.*)$/)
         l, s = $1, $2
         [[:"@", l], s]
@@ -289,7 +293,7 @@ module EBNF
         [[:plus], s[1..-1]]
       when '*'
         [[:star], s[1..-1]]
-      when /[\(\)]/
+      when /[\(\)]/ # '(' or ')'
         [[m.to_sym], s[1..-1]]
       else
         error("terminal", "unrecognized terminal: #{s.inspect}")