nokogiri 1.1.1 → 1.2.0

data/lib/nokogiri/css/generated_tokenizer.rb

@@ -53,94 +53,79 @@ class GeneratedTokenizer < GeneratedParser
       case state
       when nil
         case
-        when (text = ss.scan(/~=/i))
+        when (text = ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*\(\s*/))
+           @rex_tokens.push action { [:FUNCTION, text] }
+
+        when (text = ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*/))
+           @rex_tokens.push action { [:IDENT, text] }
+
+        when (text = ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])+/))
+           @rex_tokens.push action { [:HASH, text] }
+
+        when (text = ss.scan(/[\s\r\n\f]*~=[\s\r\n\f]*/))
            @rex_tokens.push action { [:INCLUDES, text] }
 
-        when (text = ss.scan(/\|=/i))
+        when (text = ss.scan(/[\s\r\n\f]*\|=[\s\r\n\f]*/))
            @rex_tokens.push action { [:DASHMATCH, text] }
 
-        when (text = ss.scan(/\^=/i))
+        when (text = ss.scan(/[\s\r\n\f]*\^=[\s\r\n\f]*/))
            @rex_tokens.push action { [:PREFIXMATCH, text] }
 
-        when (text = ss.scan(/\$=/i))
+        when (text = ss.scan(/[\s\r\n\f]*\$=[\s\r\n\f]*/))
            @rex_tokens.push action { [:SUFFIXMATCH, text] }
 
-        when (text = ss.scan(/\*=/i))
+        when (text = ss.scan(/[\s\r\n\f]*\*=[\s\r\n\f]*/))
            @rex_tokens.push action { [:SUBSTRINGMATCH, text] }
 
-        when (text = ss.scan(/!=/i))
+        when (text = ss.scan(/[\s\r\n\f]*!=[\s\r\n\f]*/))
            @rex_tokens.push action { [:NOT_EQUAL, text] }
 
-        when (text = ss.scan(/[-]?([_a-z]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*\(\s*/i))
-           @rex_tokens.push action { [:FUNCTION, text] }
+        when (text = ss.scan(/[\s\r\n\f]*=[\s\r\n\f]*/))
+           @rex_tokens.push action { [:EQUAL, text] }
 
-        when (text = ss.scan(/@[-]?([_a-z]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*/i))
-           @rex_tokens.push action { [:IDENT, text] }
+        when (text = ss.scan(/[\s\r\n\f]*\)[\s\r\n\f]*/))
+           @rex_tokens.push action { [:RPAREN, text] }
 
-        when (text = ss.scan(/[-]?([_a-z]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*/i))
-           @rex_tokens.push action { [:IDENT, text] }
+        when (text = ss.scan(/[\s\r\n\f]*\[[\s\r\n\f]*/))
+           @rex_tokens.push action { [:LSQUARE, text] }
 
-        when (text = ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/i))
-           @rex_tokens.push action { [:NUMBER, text] }
+        when (text = ss.scan(/[\s\r\n\f]*\][\s\r\n\f]*/))
+           @rex_tokens.push action { [:RSQUARE, text] }
 
-        when (text = ss.scan(/\#([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])+/i))
-           @rex_tokens.push action { [:HASH, text] }
-
-        when (text = ss.scan(/[\s\r\n\f]*\+/i))
+        when (text = ss.scan(/[\s\r\n\f]*\+[\s\r\n\f]*/))
            @rex_tokens.push action { [:PLUS, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*>/i))
+        when (text = ss.scan(/[\s\r\n\f]*>[\s\r\n\f]*/))
            @rex_tokens.push action { [:GREATER, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*,/i))
+        when (text = ss.scan(/[\s\r\n\f]*,[\s\r\n\f]*/))
            @rex_tokens.push action { [:COMMA, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*~/i))
+        when (text = ss.scan(/[\s\r\n\f]*~[\s\r\n\f]*/))
            @rex_tokens.push action { [:TILDE, text] }
 
-        when (text = ss.scan(/\:not\(/i))
+        when (text = ss.scan(/\:not\([\s\r\n\f]*/))
            @rex_tokens.push action { [:NOT, text] }
 
-        when (text = ss.scan(/@[-]?([_a-z]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*/i))
-           @rex_tokens.push action { [:ATKEYWORD, text] }
-
-        when (text = ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)%/i))
-           @rex_tokens.push action { [:PERCENTAGE, text] }
-
-        when (text = ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)[-]?([_a-z]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])([_a-z0-9-]|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*/i))
-           @rex_tokens.push action { [:DIMENSION, text] }
-
-        when (text = ss.scan(/<!--/i))
-           @rex_tokens.push action { [:CDO, text] }
-
-        when (text = ss.scan(/-->/i))
-           @rex_tokens.push action { [:CDC, text] }
+        when (text = ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
+           @rex_tokens.push action { [:NUMBER, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*\/\//i))
+        when (text = ss.scan(/[\s\r\n\f]*\/\/[\s\r\n\f]*/))
            @rex_tokens.push action { [:DOUBLESLASH, text] }
 
-        when (text = ss.scan(/[\s\r\n\f]*\//i))
+        when (text = ss.scan(/[\s\r\n\f]*\/[\s\r\n\f]*/))
            @rex_tokens.push action { [:SLASH, text] }
 
-        when (text = ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/i))
+        when (text = ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
            @rex_tokens.push action {[:UNICODE_RANGE, text] }
 
-        when (text = ss.scan(/\/\*(.|[\r\n])*?\*\//i))
-          ;
-
-        when (text = ss.scan(/[\s\t\r\n\f]+/i))
+        when (text = ss.scan(/[\s\t\r\n\f]+/))
            @rex_tokens.push action { [:S, text] }
 
-        when (text = ss.scan(/[\.*:\[\]=\)]/i))
-           @rex_tokens.push action { [text, text] }
-
-        when (text = ss.scan(/"([^\n\r\f"]|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*"|'([^\n\r\f']|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*'/i))
+        when (text = ss.scan(/"([^\n\r\f"]|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9A-Fa-f])*'/))
            @rex_tokens.push action { [:STRING, text] }
 
-        when (text = ss.scan(/\"([^\n\r\f\"]|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*|([^\n\r\f\']|\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?|\\[^\n\r\f0-9a-f])*/i))
-           @rex_tokens.push action { [:INVALID, text] }
-
-        when (text = ss.scan(/./i))
+        when (text = ss.scan(/./))
            @rex_tokens.push action { [text, text] }
 
         else

data/lib/nokogiri/css/node.rb

@@ -11,9 +11,7 @@ module Nokogiri
         visitor.send(:"visit_#{type.to_s.downcase}", self)
       end
 
-      def to_xpath prefix = nil, visitor = nil
-        prefix ||= '//'
-        visitor ||= XPathVisitor.new
+      def to_xpath prefix = '//', visitor = XPathVisitor.new
         self.preprocess!
         prefix + visitor.accept(self)
       end

data/lib/nokogiri/css/parser.rb

@@ -12,14 +12,6 @@ module Nokogiri
         alias :cache_on? :cache_on
         alias :set_cache :cache_on=
 
-        def parse string
-          new.parse(string)
-        end
-
-        def xpath_for string, options={}
-          new.xpath_for(string, options)
-        end
-
         def [] string
           return unless @cache_on
           @mutex.synchronize { @cache[string] }
@@ -40,6 +32,22 @@ module Nokogiri
           block.call
           @cache_on = tmp
         end
+
+        ###
+        # Parse this CSS selector in +selector+.  Returns an AST.
+        def parse selector
+          @warned ||= false
+          unless @warned
+            $stderr.puts('Nokogiri::CSS::Parser.parse is deprecated, call Nokogiri::CSS.parse()')
+            @warned = true
+          end
+          new.parse selector
+        end
+      end
+
+      def initialize namespaces = {}
+        @namespaces = namespaces
+        super()
       end
       alias :parse :scan_str
 
@@ -47,11 +55,12 @@ module Nokogiri
         v = self.class[string]
         return v if v
 
-        prefix = options[:prefix] || nil
-        visitor = options[:visitor] || nil
-        args = [prefix, visitor]
+        args = [
+          options[:prefix] || '//',
+          options[:visitor] || XPathVisitor.new
+        ]
         self.class[string] = parse(string).map { |ast|
-          ast.to_xpath(prefix, visitor)
+          ast.to_xpath(*args)
         }
       end
 

data/lib/nokogiri/css/parser.y

@@ -1,25 +1,23 @@
 class Nokogiri::CSS::GeneratedParser
 
 token FUNCTION INCLUDES DASHMATCH LBRACE HASH PLUS GREATER S STRING IDENT
-token COMMA URI CDO CDC NUMBER PERCENTAGE LENGTH EMS EXS ANGLE TIME FREQ
-token IMPORTANT_SYM IMPORT_SYM MEDIA_SYM PAGE_SYM CHARSET_SYM DIMENSION
-token PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL SLASH DOUBLESLASH
-token NOT
+token COMMA NUMBER PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL
+token SLASH DOUBLESLASH NOT EQUAL RPAREN LSQUARE RSQUARE
 
 rule
   selector
-    : selector COMMA s_0toN simple_selector_1toN {
+    : selector COMMA simple_selector_1toN {
         result = [val.first, val.last].flatten
       }
     | simple_selector_1toN { result = val.flatten }
     ;
   combinator
-    : PLUS s_0toN { result = :DIRECT_ADJACENT_SELECTOR }
-    | GREATER s_0toN { result = :CHILD_SELECTOR }
-    | TILDE s_0toN { result = :PRECEDING_SELECTOR }
+    : PLUS { result = :DIRECT_ADJACENT_SELECTOR }
+    | GREATER { result = :CHILD_SELECTOR }
+    | TILDE { result = :PRECEDING_SELECTOR }
     | S { result = :DESCENDANT_SELECTOR }
-    | DOUBLESLASH s_0toN { result = :DESCENDANT_SELECTOR }
-    | SLASH s_0toN { result = :CHILD_SELECTOR }
+    | DOUBLESLASH { result = :DESCENDANT_SELECTOR }
+    | SLASH { result = :CHILD_SELECTOR }
     ;
   simple_selector
     : element_name hcap_0toN {
@@ -68,46 +66,75 @@ rule
     : '.' IDENT { result = Node.new(:CLASS_CONDITION, [val[1]]) }
     ;
   element_name
-    : IDENT { result = Node.new(:ELEMENT_NAME, val) }
+    : namespace '|' IDENT {
+        result = Node.new(:ELEMENT_NAME,
+          [[val.first, val.last].compact.join(':')]
+        )
+      }
+    | IDENT {
+        name = @namespaces.key?('xmlns') ? "xmlns:#{val.first}" : val.first
+        result = Node.new(:ELEMENT_NAME, [name])
+      }
     | '*' { result = Node.new(:ELEMENT_NAME, val) }
     ;
+  namespace
+    : IDENT { result = val[0] }
+    |
+    ;
   attrib
-    : '[' s_0toN IDENT s_0toN attrib_val_0or1 ']' {
+    : LSQUARE IDENT attrib_val_0or1 RSQUARE {
         result = Node.new(:ATTRIBUTE_CONDITION,
-          [Node.new(:ELEMENT_NAME, [val[2]])] + (val[4] || [])
+          [Node.new(:ELEMENT_NAME, [val[1]])] + (val[2] || [])
         )
       }
-    | '[' s_0toN function s_0toN attrib_val_0or1 ']' {
+    | LSQUARE function attrib_val_0or1 RSQUARE {
         result = Node.new(:ATTRIBUTE_CONDITION,
-          [val[2]] + (val[4] || [])
+          [val[1]] + (val[2] || [])
         )
       }
-    | '[' s_0toN NUMBER s_0toN ']' {
+    | LSQUARE NUMBER RSQUARE {
         # Non standard, but hpricot supports it.
         result = Node.new(:PSEUDO_CLASS,
-          [Node.new(:FUNCTION, ['nth-child(', val[2]])]
+          [Node.new(:FUNCTION, ['nth-child(', val[1]])]
         )
       }
     ;
   function
-    : FUNCTION ')' {
+    : FUNCTION RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip])
       }
-    | FUNCTION expr ')' {
+    | FUNCTION expr RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
-    | FUNCTION an_plus_b ')' {
+    | FUNCTION an_plus_b RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
-    | NOT expr ')' {
+    | NOT expr RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
     ;
   expr
-    : NUMBER COMMA s_0toN expr { result = [val.first, val.last] }
-    | STRING COMMA s_0toN expr { result = [val.first, val.last] }
+    : NUMBER COMMA expr { result = [val.first, val.last] }
+    | STRING COMMA expr { result = [val.first, val.last] }
+    | IDENT COMMA expr { result = [val.first, val.last] }
     | NUMBER
     | STRING
+    | IDENT                             # even, odd
+      {
+        if val[0] == 'even'
+          val = ["2","n","+","0"]
+          result = Node.new(:AN_PLUS_B, val)
+        elsif val[0] == 'odd'
+          val = ["2","n","+","1"]
+          result = Node.new(:AN_PLUS_B, val)
+        else
+          # This is not CSS standard.  It allows us to support this:
+          # assert_xpath("//a[foo(., @href)]", @parser.parse('a:foo(@href)'))
+          # assert_xpath("//a[foo(., @a, b)]", @parser.parse('a:foo(@a, b)'))
+          # assert_xpath("//a[foo(., a, 10)]", @parser.parse('a:foo(a, 10)'))
+          result = val
+        end
+      }
     ;
   an_plus_b
     : NUMBER IDENT PLUS NUMBER          # 5n+3 -5n+3
@@ -140,18 +167,6 @@ rule
           raise Racc::ParseError, "parse error on IDENT '#{val[1]}'"
         end
       }
-    | IDENT                             # even, odd
-      {
-        if val[0] == 'even'
-          val = ["2","n","+","0"]
-          result = Node.new(:AN_PLUS_B, val)
-        elsif val[0] == 'odd'
-          val = ["2","n","+","1"]
-          result = Node.new(:AN_PLUS_B, val)
-        else
-          raise Racc::ParseError, "parse error on IDENT '#{val[0]}'"
-        end
-      }
     ;
   pseudo
     : ':' function {
@@ -185,12 +200,12 @@ rule
     : HASH { result = Node.new(:ID, val) }
     ;
   attrib_val_0or1
-    : eql_incl_dash s_0toN IDENT s_0toN { result = [val.first, val[2]] }
-    | eql_incl_dash s_0toN STRING s_0toN { result = [val.first, val[2]] }
+    : eql_incl_dash IDENT { result = [val.first, val[1]] }
+    | eql_incl_dash STRING { result = [val.first, val[1]] }
     |
     ;
   eql_incl_dash
-    : '='
+    : EQUAL
     | PREFIXMATCH
     | SUFFIXMATCH
     | SUBSTRINGMATCH
@@ -199,17 +214,13 @@ rule
     | DASHMATCH
     ;
   negation
-    : NOT s_0toN negation_arg s_0toN ')' {
-        result = Node.new(:NOT, [val[2]])
+    : NOT negation_arg RPAREN {
+        result = Node.new(:NOT, [val[1]])
       }
     ;
   negation_arg
     : hcap_1toN
     ;
-  s_0toN
-    : S s_0toN
-    |
-    ;
 end
 
 ---- header

data/lib/nokogiri/css/syntax_error.rb

@@ -1,6 +1,7 @@
+require 'nokogiri/syntax_error'
 module Nokogiri
   module CSS
-    class SyntaxError < ::SyntaxError
+    class SyntaxError < ::Nokogiri::SyntaxError
     end
   end
 end

data/lib/nokogiri/css/tokenizer.rex

@@ -7,56 +7,47 @@ macro
   w         [\s\r\n\f]*
   nonascii  [^\\\\0-\\\\177]
   num       -?([0-9]+|[0-9]*\.[0-9]+)
-  unicode   \\\\\\\\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?
+  unicode   \\\\\\\\\[0-9A-Fa-f]{1,6}(\r\n|[\s\n\r\t\f])?
 
-  escape    {unicode}|\\\\\\\[^\n\r\f0-9a-f]
-  nmchar    [_a-z0-9-]|{nonascii}|{escape}
-  nmstart   [_a-z]|{nonascii}|{escape}
-  ident     [-]?({nmstart})({nmchar})*
+  escape    {unicode}|\\\\\\\[^\n\r\f0-9A-Fa-f]
+  nmchar    [_A-Za-z0-9-]|{nonascii}|{escape}
+  nmstart   [_A-Za-z]|{nonascii}|{escape}
+  ident     [-@]?({nmstart})({nmchar})*
   name      ({nmchar})+
   string1   "([^\n\r\f"]|\\{nl}|{nonascii}|{escape})*"
   string2   '([^\n\r\f']|\\{nl}|{nonascii}|{escape})*'
   string    {string1}|{string2}
-  invalid1  \"([^\n\r\f\\"]|\\{nl}|{nonascii}|{escape})*
-  invalid2  \'([^\n\r\f\\']|\\{nl}|{nonascii}|{escape})*
-  invalid   {invalid1}|{invalid2}
-  Comment   \/\*(.|[\r\n])*?\*\/
 
 rule
 
 # [:state]  pattern  [actions]
 
-            ~=               { [:INCLUDES, text] }
-            \|=              { [:DASHMATCH, text] }
-            \^=              { [:PREFIXMATCH, text] }
-            \$=              { [:SUFFIXMATCH, text] }
-            \*=              { [:SUBSTRINGMATCH, text] }
-            !=               { [:NOT_EQUAL, text] }
             {ident}\(\s*     { [:FUNCTION, text] }
-            @{ident}         { [:IDENT, text] }
             {ident}          { [:IDENT, text] }
-            {num}            { [:NUMBER, text] }
             \#{name}         { [:HASH, text] }
-            {w}\+            { [:PLUS, text] }
-            {w}>             { [:GREATER, text] }
-            {w},             { [:COMMA, text] }
-            {w}~             { [:TILDE, text] }
-            \:not\(          { [:NOT, text] }
-            @{ident}         { [:ATKEYWORD, text] }
-            {num}%           { [:PERCENTAGE, text] }
-            {num}{ident}     { [:DIMENSION, text] }
-            <!--             { [:CDO, text] }
-            -->              { [:CDC, text] }
-            {w}\/\/          { [:DOUBLESLASH, text] }
-            {w}\/            { [:SLASH, text] }
+            {w}~={w}         { [:INCLUDES, text] }
+            {w}\|={w}        { [:DASHMATCH, text] }
+            {w}\^={w}        { [:PREFIXMATCH, text] }
+            {w}\$={w}        { [:SUFFIXMATCH, text] }
+            {w}\*={w}        { [:SUBSTRINGMATCH, text] }
+            {w}!={w}         { [:NOT_EQUAL, text] }
+            {w}={w}          { [:EQUAL, text] }
+            {w}\){w}         { [:RPAREN, text] }
+            {w}\[{w}         { [:LSQUARE, text] }
+            {w}\]{w}         { [:RSQUARE, text] }
+            {w}\+{w}         { [:PLUS, text] }
+            {w}>{w}          { [:GREATER, text] }
+            {w},{w}          { [:COMMA, text] }
+            {w}~{w}          { [:TILDE, text] }
+            \:not\({w}       { [:NOT, text] }
+            {num}            { [:NUMBER, text] }
+            {w}\/\/{w}       { [:DOUBLESLASH, text] }
+            {w}\/{w}         { [:SLASH, text] }
             
             U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?  {[:UNICODE_RANGE, text] }
             
-            {Comment}                    /* ignore comments */
             [\s\t\r\n\f]+    { [:S, text] }
-            [\.*:\[\]=\)]    { [text, text] }
             {string}         { [:STRING, text] }
-            {invalid}        { [:INVALID, text] }
             .                { [text, text] }
 end
 end