nokogiri 1.0.0 → 1.6.8.1

data/lib/nokogiri/css/parser.y

@@ -1,25 +1,23 @@
-class Nokogiri::CSS::GeneratedParser
+class Nokogiri::CSS::Parser
 
 token FUNCTION INCLUDES DASHMATCH LBRACE HASH PLUS GREATER S STRING IDENT
-token COMMA URI CDO CDC NUMBER PERCENTAGE LENGTH EMS EXS ANGLE TIME FREQ
-token IMPORTANT_SYM IMPORT_SYM MEDIA_SYM PAGE_SYM CHARSET_SYM DIMENSION
-token PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL SLASH DOUBLESLASH
-token NOT
+token COMMA NUMBER PREFIXMATCH SUFFIXMATCH SUBSTRINGMATCH TILDE NOT_EQUAL
+token SLASH DOUBLESLASH NOT EQUAL RPAREN LSQUARE RSQUARE HAS
 
 rule
   selector
-    : selector COMMA s_0toN simple_selector_1toN {
+    : selector COMMA simple_selector_1toN {
         result = [val.first, val.last].flatten
       }
-    | simple_selector_1toN { result = val.flatten }
+    | prefixless_combinator_selector { result = val.flatten }
+    | optional_S simple_selector_1toN { result = [val.last].flatten }
     ;
   combinator
-    : PLUS s_0toN { result = :DIRECT_ADJACENT_SELECTOR }
-    | GREATER s_0toN { result = :CHILD_SELECTOR }
-    | TILDE s_0toN { result = :PRECEDING_SELECTOR }
-    | S { result = :DESCENDANT_SELECTOR }
-    | DOUBLESLASH s_0toN { result = :DESCENDANT_SELECTOR }
-    | SLASH s_0toN { result = :CHILD_SELECTOR }
+    : PLUS { result = :DIRECT_ADJACENT_SELECTOR }
+    | GREATER { result = :CHILD_SELECTOR }
+    | TILDE { result = :FOLLOWING_SELECTOR }
+    | DOUBLESLASH { result = :DESCENDANT_SELECTOR }
+    | SLASH { result = :CHILD_SELECTOR }
     ;
   simple_selector
     : element_name hcap_0toN {
@@ -29,10 +27,10 @@ rule
                     Node.new(:CONDITIONAL_SELECTOR, [val.first, val[1]])
                   end
       }
-    | element_name negation {
+    | function
+    | function pseudo {
         result = Node.new(:CONDITIONAL_SELECTOR, val)
       }
-    | function
     | function attrib {
         result = Node.new(:CONDITIONAL_SELECTOR, val)
       }
@@ -42,60 +40,118 @@ rule
         )
       }
     ;
+  prefixless_combinator_selector
+    : combinator simple_selector_1toN {
+        result = Node.new(val.first, [nil, val.last])
+      }
+    ;
   simple_selector_1toN
     : simple_selector combinator simple_selector_1toN {
         result = Node.new(val[1], [val.first, val.last])
       }
+    | simple_selector S simple_selector_1toN {
+        result = Node.new(:DESCENDANT_SELECTOR, [val.first, val.last])
+      }
     | simple_selector
     ;
   class
-    : '.' IDENT { result = Node.new(:CLASS_CONDITION, [val[1]]) }
+    : '.' IDENT { result = Node.new(:CLASS_CONDITION, [unescape_css_identifier(val[1])]) }
     ;
   element_name
-    : IDENT { result = Node.new(:ELEMENT_NAME, val) }
+    : namespaced_ident
     | '*' { result = Node.new(:ELEMENT_NAME, val) }
     ;
+  namespaced_ident
+    : namespace '|' IDENT {
+        result = Node.new(:ELEMENT_NAME,
+          [[val.first, val.last].compact.join(':')]
+        )
+      }
+    | IDENT {
+        name = @namespaces.key?('xmlns') ? "xmlns:#{val.first}" : val.first
+        result = Node.new(:ELEMENT_NAME, [name])
+      }
+    ;
+  namespace
+    : IDENT { result = val[0] }
+    |
+    ;
   attrib
-    : '[' s_0toN IDENT s_0toN attrib_val_0or1 ']' {
+    : LSQUARE attrib_name attrib_val_0or1 RSQUARE {
         result = Node.new(:ATTRIBUTE_CONDITION,
-          [Node.new(:ELEMENT_NAME, [val[2]])] + (val[4] || [])
+          [val[1]] + (val[2] || [])
         )
       }
-    | '[' s_0toN function s_0toN attrib_val_0or1 ']' {
+    | LSQUARE function attrib_val_0or1 RSQUARE {
         result = Node.new(:ATTRIBUTE_CONDITION,
-          [val[2]] + (val[4] || [])
+          [val[1]] + (val[2] || [])
         )
       }
-    | '[' s_0toN NUMBER s_0toN ']' {
+    | LSQUARE NUMBER RSQUARE {
         # Non standard, but hpricot supports it.
         result = Node.new(:PSEUDO_CLASS,
-          [Node.new(:FUNCTION, ['nth-child(', val[2]])]
+          [Node.new(:FUNCTION, ['nth-child(', val[1]])]
         )
       }
     ;
+  attrib_name
+    : namespace '|' IDENT {
+        result = Node.new(:ELEMENT_NAME,
+          [[val.first, val.last].compact.join(':')]
+        )
+      }
+    | IDENT {
+        # Default namespace is not applied to attributes.
+        # So we don't add prefix "xmlns:" as in namespaced_ident.
+        result = Node.new(:ELEMENT_NAME, [val.first])
+      }
+    ;
   function
-    : FUNCTION ')' {
+    : FUNCTION RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip])
       }
-    | FUNCTION expr ')' {
+    | FUNCTION expr RPAREN {
+        result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
+      }
+    | FUNCTION nth RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
-    | FUNCTION an_plus_b ')' {
+    | NOT expr RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
-    | NOT expr ')' {
+    | HAS selector RPAREN {
         result = Node.new(:FUNCTION, [val.first.strip, val[1]].flatten)
       }
     ;
   expr
-    : NUMBER
+    : NUMBER COMMA expr { result = [val.first, val.last] }
+    | STRING COMMA expr { result = [val.first, val.last] }
+    | IDENT COMMA expr { result = [val.first, val.last] }
+    | NUMBER
     | STRING
+    | IDENT                             # even, odd
+      {
+        case val[0]
+        when 'even'
+          result = Node.new(:NTH, ['2','n','+','0'])
+        when 'odd'
+          result = Node.new(:NTH, ['2','n','+','1'])
+        when 'n'
+          result = Node.new(:NTH, ['1','n','+','0'])
+        else
+          # This is not CSS standard.  It allows us to support this:
+          # assert_xpath("//a[foo(., @href)]", @parser.parse('a:foo(@href)'))
+          # assert_xpath("//a[foo(., @a, b)]", @parser.parse('a:foo(@a, b)'))
+          # assert_xpath("//a[foo(., a, 10)]", @parser.parse('a:foo(a, 10)'))
+          result = val
+        end
+      }
     ;
-  an_plus_b
+  nth
     : NUMBER IDENT PLUS NUMBER          # 5n+3 -5n+3
       {
         if val[1] == 'n'
-          result = Node.new(:AN_PLUS_B, val)
+          result = Node.new(:NTH, val)
         else
           raise Racc::ParseError, "parse error on IDENT '#{val[1]}'"
         end
@@ -103,37 +159,31 @@ rule
     | IDENT PLUS NUMBER {               # n+3, -n+3
         if val[0] == 'n'
           val.unshift("1")
-          result = Node.new(:AN_PLUS_B, val)
+          result = Node.new(:NTH, val)
         elsif val[0] == '-n'
           val[0] = 'n'
           val.unshift("-1")
-          result = Node.new(:AN_PLUS_B, val)
+          result = Node.new(:NTH, val)
         else
           raise Racc::ParseError, "parse error on IDENT '#{val[1]}'"
         end
       }
-    | NUMBER IDENT                      # 5n, -5n
-      {
-        if val[1] == 'n'
+    | NUMBER IDENT {                    # 5n, -5n, 10n-1
+        n = val[1]
+        if n[0, 2] == 'n-'
+          val[1] = 'n'
+          val << "-"
+          # b is contained in n as n is the string "n-b"
+          val << n[2, n.size]
+          result = Node.new(:NTH, val)
+        elsif n == 'n'
           val << "+"
           val << "0"
-          result = Node.new(:AN_PLUS_B, val)
+          result = Node.new(:NTH, val)
         else
           raise Racc::ParseError, "parse error on IDENT '#{val[1]}'"
         end
       }
-    | IDENT                             # even, odd
-      {
-        if val[0] == 'even'
-          val = ["2","n","+","0"]
-          result = Node.new(:AN_PLUS_B, val)
-        elsif val[0] == 'odd'
-          val = ["2","n","+","1"]
-          result = Node.new(:AN_PLUS_B, val)
-        else
-          raise Racc::ParseError, "parse error on IDENT '#{val[0]}'"
-        end
-      }
     ;
   pseudo
     : ':' function {
@@ -158,41 +208,54 @@ rule
     | pseudo hcap_1toN {
         result = Node.new(:COMBINATOR, val)
       }
+    | negation hcap_1toN {
+        result = Node.new(:COMBINATOR, val)
+      }
     | attribute_id
     | class
     | attrib
     | pseudo
+    | negation
     ;
   attribute_id
-    : HASH { result = Node.new(:ID, val) }
+    : HASH { result = Node.new(:ID, [unescape_css_identifier(val.first)]) }
     ;
   attrib_val_0or1
-    : eql_incl_dash s_0toN IDENT s_0toN { result = [val.first, val[2]] }
-    | eql_incl_dash s_0toN STRING s_0toN { result = [val.first, val[2]] }
+    : eql_incl_dash IDENT { result = [val.first, val[1]] }
+    | eql_incl_dash STRING { result = [val.first, val[1]] }
     |
     ;
   eql_incl_dash
-    : '='
-    | PREFIXMATCH
-    | SUFFIXMATCH
-    | SUBSTRINGMATCH
-    | NOT_EQUAL
-    | INCLUDES
-    | DASHMATCH
+    : EQUAL           { result = :equal }
+    | PREFIXMATCH     { result = :prefix_match }
+    | SUFFIXMATCH     { result = :suffix_match }
+    | SUBSTRINGMATCH  { result = :substring_match }
+    | NOT_EQUAL       { result = :not_equal }
+    | INCLUDES        { result = :includes }
+    | DASHMATCH       { result = :dash_match }
     ;
   negation
-    : NOT s_0toN negation_arg s_0toN ')' {
-        result = Node.new(:NOT, [val[2]])
+    : NOT negation_arg RPAREN {
+        result = Node.new(:NOT, [val[1]])
       }
     ;
   negation_arg
-    : hcap_1toN
+    : element_name
+    | element_name hcap_1toN
+    | hcap_1toN
     ;
-  s_0toN
-    : S s_0toN
+  optional_S
+    : S
     |
     ;
 end
 
 ---- header
 
+require 'nokogiri/css/parser_extras'
+
+---- inner
+
+def unescape_css_identifier(identifier)
+  identifier.gsub(/\\(?:([^0-9a-fA-F])|([0-9a-fA-F]{1,6})\s?)/){ |m| $1 || [$2.hex].pack('U') }
+end

data/lib/nokogiri/css/parser_extras.rb

@@ -0,0 +1,91 @@
+require 'thread'
+
+module Nokogiri
+  module CSS
+    class Parser < Racc::Parser
+      @cache_on = true
+      @cache    = {}
+      @mutex    = Mutex.new
+
+      class << self
+        # Turn on CSS parse caching
+        attr_accessor :cache_on
+        alias :cache_on? :cache_on
+        alias :set_cache :cache_on=
+
+        # Get the css selector in +string+ from the cache
+        def [] string
+          return unless @cache_on
+          @mutex.synchronize { @cache[string] }
+        end
+
+        # Set the css selector in +string+ in the cache to +value+
+        def []= string, value
+          return value unless @cache_on
+          @mutex.synchronize { @cache[string] = value }
+        end
+
+        # Clear the cache
+        def clear_cache
+          @mutex.synchronize { @cache = {} }
+        end
+
+        # Execute +block+ without cache
+        def without_cache &block
+          tmp = @cache_on
+          @cache_on = false
+          block.call
+          @cache_on = tmp
+        end
+
+        ###
+        # Parse this CSS selector in +selector+.  Returns an AST.
+        def parse selector
+          @warned ||= false
+          unless @warned
+            $stderr.puts('Nokogiri::CSS::Parser.parse is deprecated, call Nokogiri::CSS.parse(), this will be removed August 1st or version 1.4.0 (whichever is first)')
+            @warned = true
+          end
+          new.parse selector
+        end
+      end
+
+      # Create a new CSS parser with respect to +namespaces+
+      def initialize namespaces = {}
+        @tokenizer  = Tokenizer.new
+        @namespaces = namespaces
+        super()
+      end
+
+      def parse string
+        @tokenizer.scan_setup string
+        do_parse
+      end
+
+      def next_token
+        @tokenizer.next_token
+      end
+
+      # Get the xpath for +string+ using +options+
+      def xpath_for string, options={}
+        key = "#{string}#{options[:ns]}#{options[:prefix]}"
+        v = self.class[key]
+        return v if v
+
+        args = [
+          options[:prefix] || '//',
+          options[:visitor] || XPathVisitor.new
+        ]
+        self.class[key] = parse(string).map { |ast|
+          ast.to_xpath(*args)
+        }
+      end
+
+      # On CSS parser error, raise an exception
+      def on_error error_token_id, error_value, value_stack
+        after = value_stack.compact.last
+        raise SyntaxError.new("unexpected '#{error_value}' after '#{after}'")
+      end
+    end
+  end
+end

data/lib/nokogiri/css/syntax_error.rb

@@ -0,0 +1,7 @@
+require 'nokogiri/syntax_error'
+module Nokogiri
+  module CSS
+    class SyntaxError < ::Nokogiri::SyntaxError
+    end
+  end
+end

data/lib/nokogiri/css/tokenizer.rb

@@ -1,9 +1,152 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "lib/nokogiri/css/tokenizer.rex".
+#++
+
 module Nokogiri
-  module CSS
-    class Tokenizer < GeneratedTokenizer
-      def scan(str)
-        scan_evaluate(str)
-      end
+module CSS
+class Tokenizer # :nodoc:
+  require 'strscan'
+
+  class ScanError < StandardError ; end
+
+  attr_reader   :lineno
+  attr_reader   :filename
+  attr_accessor :state
+
+  def scan_setup(str)
+    @ss = StringScanner.new(str)
+    @lineno =  1
+    @state  = nil
+  end
+
+  def action
+    yield
+  end
+
+  def scan_str(str)
+    scan_setup(str)
+    do_parse
+  end
+  alias :scan :scan_str
+
+  def load_file( filename )
+    @filename = filename
+    open(filename, "r") do |f|
+      scan_setup(f.read)
     end
   end
+
+  def scan_file( filename )
+    load_file(filename)
+    do_parse
+  end
+
+
+  def next_token
+    return if @ss.eos?
+    
+    # skips empty actions
+    until token = _next_token or @ss.eos?; end
+    token
+  end
+
+  def _next_token
+    text = @ss.peek(1)
+    @lineno  +=  1  if text == "\n"
+    token = case @state
+    when nil
+      case
+      when (text = @ss.scan(/has\([\s]*/))
+         action { [:HAS, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\([\s]*/))
+         action { [:FUNCTION, text] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
+         action { [:IDENT, text] }
+
+      when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
+         action { [:HASH, text] }
+
+      when (text = @ss.scan(/[\s]*~=[\s]*/))
+         action { [:INCLUDES, text] }
+
+      when (text = @ss.scan(/[\s]*\|=[\s]*/))
+         action { [:DASHMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\^=[\s]*/))
+         action { [:PREFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\$=[\s]*/))
+         action { [:SUFFIXMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*\*=[\s]*/))
+         action { [:SUBSTRINGMATCH, text] }
+
+      when (text = @ss.scan(/[\s]*!=[\s]*/))
+         action { [:NOT_EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*=[\s]*/))
+         action { [:EQUAL, text] }
+
+      when (text = @ss.scan(/[\s]*\)/))
+         action { [:RPAREN, text] }
+
+      when (text = @ss.scan(/\[[\s]*/))
+         action { [:LSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\]/))
+         action { [:RSQUARE, text] }
+
+      when (text = @ss.scan(/[\s]*\+[\s]*/))
+         action { [:PLUS, text] }
+
+      when (text = @ss.scan(/[\s]*>[\s]*/))
+         action { [:GREATER, text] }
+
+      when (text = @ss.scan(/[\s]*,[\s]*/))
+         action { [:COMMA, text] }
+
+      when (text = @ss.scan(/[\s]*~[\s]*/))
+         action { [:TILDE, text] }
+
+      when (text = @ss.scan(/\:not\([\s]*/))
+         action { [:NOT, text] }
+
+      when (text = @ss.scan(/-?([0-9]+|[0-9]*\.[0-9]+)/))
+         action { [:NUMBER, text] }
+
+      when (text = @ss.scan(/[\s]*\/\/[\s]*/))
+         action { [:DOUBLESLASH, text] }
+
+      when (text = @ss.scan(/[\s]*\/[\s]*/))
+         action { [:SLASH, text] }
+
+      when (text = @ss.scan(/U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?/))
+         action {[:UNICODE_RANGE, text] }
+
+      when (text = @ss.scan(/[\s]+/))
+         action { [:S, text] }
+
+      when (text = @ss.scan(/"([^\n\r\f"]|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f']|\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*'/))
+         action { [:STRING, text] }
+
+      when (text = @ss.scan(/./))
+         action { [text, text] }
+
+      else
+        text = @ss.string[@ss.pos .. -1]
+        raise  ScanError, "can not match: '" + text + "'"
+      end  # if
+
+    else
+      raise  ScanError, "undefined state: '" + state.to_s + "'"
+    end  # case state
+    token
+  end  # def _next_token
+
+end # class
+end
 end

data/lib/nokogiri/css/tokenizer.rex

@@ -1,62 +1,54 @@
 module Nokogiri
 module CSS
-class GeneratedTokenizer
+class Tokenizer # :nodoc:
 
 macro
   nl        \n|\r\n|\r|\f
-  w         [\s\r\n\f]*
-  nonascii  [^\\\\0-\\\\177]
+  w         [\s]*
+  nonascii  [^\0-\177]
   num       -?([0-9]+|[0-9]*\.[0-9]+)
-  unicode   \\\\\\\\\[0-9a-f]{1,6}(\r\n|[\s\n\r\t\f])?
+  unicode   \\[0-9A-Fa-f]{1,6}(\r\n|[\s])?
 
-  escape    {unicode}|\\\\\\\[^\n\r\f0-9a-f]
-  nmchar    [_a-z0-9-]|{nonascii}|{escape}
-  nmstart   [_a-z]|{nonascii}|{escape}
-  ident     [-]?({nmstart})({nmchar})*
+  escape    {unicode}|\\[^\n\r\f0-9A-Fa-f]
+  nmchar    [_A-Za-z0-9-]|{nonascii}|{escape}
+  nmstart   [_A-Za-z]|{nonascii}|{escape}
+  ident     [-@]?({nmstart})({nmchar})*
   name      ({nmchar})+
-  string1   "([^\n\r\f"]|\\{nl}|{nonascii}|{escape})*"
-  string2   '([^\n\r\f']|\\{nl}|{nonascii}|{escape})*'
+  string1   "([^\n\r\f"]|{nl}|{nonascii}|{escape})*"
+  string2   '([^\n\r\f']|{nl}|{nonascii}|{escape})*'
   string    {string1}|{string2}
-  invalid1  \"([^\n\r\f\\"]|\\{nl}|{nonascii}|{escape})*
-  invalid2  \'([^\n\r\f\\']|\\{nl}|{nonascii}|{escape})*
-  invalid   {invalid1}|{invalid2}
-  Comment   \/\*(.|[\r\n])*?\*\/
 
 rule
 
 # [:state]  pattern  [actions]
 
-            ~=               { [:INCLUDES, text] }
-            \|=              { [:DASHMATCH, text] }
-            \^=              { [:PREFIXMATCH, text] }
-            \$=              { [:SUFFIXMATCH, text] }
-            \*=              { [:SUBSTRINGMATCH, text] }
-            !=               { [:NOT_EQUAL, text] }
-            {ident}\(\s*     { [:FUNCTION, text] }
-            @{ident}         { [:IDENT, text] }
+            has\({w}         { [:HAS, text] }
+            {ident}\({w}     { [:FUNCTION, text] }
             {ident}          { [:IDENT, text] }
-            {num}            { [:NUMBER, text] }
             \#{name}         { [:HASH, text] }
-            {w}\+            { [:PLUS, text] }
-            {w}>             { [:GREATER, text] }
-            {w},             { [:COMMA, text] }
-            {w}~             { [:TILDE, text] }
-            \:not\(          { [:NOT, text] }
-            @{ident}         { [:ATKEYWORD, text] }
-            {num}%           { [:PERCENTAGE, text] }
-            {num}{ident}     { [:DIMENSION, text] }
-            <!--             { [:CDO, text] }
-            -->              { [:CDC, text] }
-            {w}\/\/          { [:DOUBLESLASH, text] }
-            {w}\/            { [:SLASH, text] }
+            {w}~={w}         { [:INCLUDES, text] }
+            {w}\|={w}        { [:DASHMATCH, text] }
+            {w}\^={w}        { [:PREFIXMATCH, text] }
+            {w}\$={w}        { [:SUFFIXMATCH, text] }
+            {w}\*={w}        { [:SUBSTRINGMATCH, text] }
+            {w}!={w}         { [:NOT_EQUAL, text] }
+            {w}={w}          { [:EQUAL, text] }
+            {w}\)            { [:RPAREN, text] }
+            \[{w}            { [:LSQUARE, text] }
+            {w}\]            { [:RSQUARE, text] }
+            {w}\+{w}         { [:PLUS, text] }
+            {w}>{w}          { [:GREATER, text] }
+            {w},{w}          { [:COMMA, text] }
+            {w}~{w}          { [:TILDE, text] }
+            \:not\({w}       { [:NOT, text] }
+            {num}            { [:NUMBER, text] }
+            {w}\/\/{w}       { [:DOUBLESLASH, text] }
+            {w}\/{w}         { [:SLASH, text] }
             
             U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?  {[:UNICODE_RANGE, text] }
             
-            {Comment}                    /* ignore comments */
-            [\s\t\r\n\f]+    { [:S, text] }
-            [\.*:\[\]=\)]    { [text, text] }
+            [\s]+            { [:S, text] }
             {string}         { [:STRING, text] }
-            {invalid}        { [:INVALID, text] }
             .                { [text, text] }
 end
 end