antisamy 0.2.1 → 0.3.0

data/lib/antisamy/csspool/rsac/sac/selectors/descendant_selector.rb

@@ -1,36 +1,36 @@
-module RSAC
-  module Selectors
-    class DescendantSelector < SimpleSelector
-      attr_accessor :ancestor_selector, :simple_selector
-      alias :ancestor :ancestor_selector
-      alias :selector :simple_selector
-
-      def initialize(ancestor, selector)
-        super(:SAC_DESCENDANT_SELECTOR)
-
-        @ancestor_selector = ancestor
-        @simple_selector = selector
-      end
-
-      def to_css
-        "#{ancestor.to_css} #{selector.to_css}"
-      end
-
-      def to_xpath(prefix=true)
-        "#{ancestor.to_xpath(prefix)}//#{selector.to_xpath(false)}"
-      end
-
-      def specificity
-        ancestor.specificity.zip(selector.specificity).map { |x,y| x + y }
-      end
-
-      def ==(other)
-        super && selector == other.selector && ancestor == other.ancestor
-      end
-
-      def hash
-        [selector, ancestor].hash
-      end
-    end
-  end
-end
+module RSAC
+  module Selectors
+    class DescendantSelector < SimpleSelector
+      attr_accessor :ancestor_selector, :simple_selector
+      alias :ancestor :ancestor_selector
+      alias :selector :simple_selector
+
+      def initialize(ancestor, selector)
+        super(:SAC_DESCENDANT_SELECTOR)
+
+        @ancestor_selector = ancestor
+        @simple_selector = selector
+      end
+
+      def to_css
+        "#{ancestor.to_css} #{selector.to_css}"
+      end
+
+      def to_xpath(prefix=true)
+        "#{ancestor.to_xpath(prefix)}//#{selector.to_xpath(false)}"
+      end
+
+      def specificity
+        ancestor.specificity.zip(selector.specificity).map { |x,y| x + y }
+      end
+
+      def ==(other)
+        super && selector == other.selector && ancestor == other.ancestor
+      end
+
+      def hash
+        [selector, ancestor].hash
+      end
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/selectors/element_selector.rb

@@ -1,35 +1,35 @@
-module RSAC
-  module Selectors
-    class ElementSelector < SimpleSelector
-      attr_reader :local_name
-      alias :name :local_name
-
-      def initialize(name)
-        super(:SAC_ELEMENT_NODE_SELECTOR)
-        @local_name = name
-      end
-
-      def to_css
-        local_name
-      end
-
-      def to_xpath(prefix=true)
-        atoms = [local_name]
-        atoms.unshift("//") if prefix
-        atoms.join
-      end
-
-      def specificity
-        [0, 0, 0, 1]
-      end
-
-      def ==(other)
-        super && name == other.name
-      end
-
-      def hash
-        name.hash
-      end
-    end
-  end
-end
+module RSAC
+  module Selectors
+    class ElementSelector < SimpleSelector
+      attr_reader :local_name
+      alias :name :local_name
+
+      def initialize(name)
+        super(:SAC_ELEMENT_NODE_SELECTOR)
+        @local_name = name
+      end
+
+      def to_css
+        local_name
+      end
+
+      def to_xpath(prefix=true)
+        atoms = [local_name]
+        atoms.unshift("//") if prefix
+        atoms.join
+      end
+
+      def specificity
+        [0, 0, 0, 1]
+      end
+
+      def ==(other)
+        super && name == other.name
+      end
+
+      def hash
+        name.hash
+      end
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/selectors/selector.rb

@@ -1,25 +1,25 @@
-module RSAC
-  module Selectors
-    class Selector
-
-      attr_reader :selector_type
-
-      def initialize(selector_type)
-        @selector_type = selector_type
-      end
-
-      def ==(other)
-        self.class === other && selector_type == other.selector_type
-      end
-
-      def hash
-        selector_type.hash
-      end
-
-      def eql?(other)
-        self == other
-      end
-
-    end
-  end
-end
+module RSAC
+  module Selectors
+    class Selector
+
+      attr_reader :selector_type
+
+      def initialize(selector_type)
+        @selector_type = selector_type
+      end
+
+      def ==(other)
+        self.class === other && selector_type == other.selector_type
+      end
+
+      def hash
+        selector_type.hash
+      end
+
+      def eql?(other)
+        self == other
+      end
+
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/selectors/sibling_selector.rb

@@ -1,35 +1,35 @@
-module RSAC
-  module Selectors
-    class SiblingSelector < SimpleSelector
-      attr_accessor :selector, :sibling_selector
-      alias :sibling :sibling_selector
-
-      def initialize(selector, sibling)
-        super(:SAC_DIRECT_ADJACENT_SELECTOR)
-
-        @selector = selector
-        @sibling_selector = sibling
-      end
-
-      def to_css
-        "#{selector.to_css} + #{sibling.to_css}"
-      end
-
-      def to_xpath(prefix=true)
-        "#{selector.to_xpath(prefix)}/following-sibling::#{sibling.to_xpath(false)}"
-      end
-
-      def specificity
-        selector.specificity.zip(sibling.specificity).map { |x,y| x + y }
-      end
-
-      def ==(other)
-        super && selector == other.selector && sibling == other.sibling
-      end
-
-      def hash
-        [selector, sibling].hash
-      end
-    end
-  end
-end
+module RSAC
+  module Selectors
+    class SiblingSelector < SimpleSelector
+      attr_accessor :selector, :sibling_selector
+      alias :sibling :sibling_selector
+
+      def initialize(selector, sibling)
+        super(:SAC_DIRECT_ADJACENT_SELECTOR)
+
+        @selector = selector
+        @sibling_selector = sibling
+      end
+
+      def to_css
+        "#{selector.to_css} + #{sibling.to_css}"
+      end
+
+      def to_xpath(prefix=true)
+        "#{selector.to_xpath(prefix)}/following-sibling::#{sibling.to_xpath(false)}"
+      end
+
+      def specificity
+        selector.specificity.zip(sibling.specificity).map { |x,y| x + y }
+      end
+
+      def ==(other)
+        super && selector == other.selector && sibling == other.sibling
+      end
+
+      def hash
+        [selector, sibling].hash
+      end
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/selectors/simple_selector.rb

@@ -1,21 +1,21 @@
-module RSAC
-  module Selectors
-    class SimpleSelector < Selector
-      def initialize(selector_type=:SAC_ANY_NODE_SELECTOR)
-        super(selector_type)
-      end
-
-      def to_css
-        '*'
-      end
-
-      def to_xpath
-        "//*"
-      end
-
-      def specificity
-        [0, 0, 0, 0]
-      end
-    end
-  end
-end
+module RSAC
+  module Selectors
+    class SimpleSelector < Selector
+      def initialize(selector_type=:SAC_ANY_NODE_SELECTOR)
+        super(selector_type)
+      end
+
+      def to_css
+        '*'
+      end
+
+      def to_xpath
+        "//*"
+      end
+
+      def specificity
+        [0, 0, 0, 0]
+      end
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/token.rb

@@ -1,25 +1,25 @@
-module RSAC
-  class Token
-    attr_reader :name, :value, :position
-
-    def initialize(name, value, position)
-      @name = name
-      @value = value
-      @position = position
-    end
-
-    def to_racc_token
-      [name, value]
-    end
-  end
-
-  class DelimiterToken < Token
-    def initialize(value, position)
-      super(:delim, value, position)
-    end
-
-    def to_racc_token
-      [value, value]
-    end
-  end
-end
+module RSAC
+  class Token
+    attr_reader :name, :value, :position
+
+    def initialize(name, value, position)
+      @name = name
+      @value = value
+      @position = position
+    end
+
+    def to_racc_token
+      [name, value]
+    end
+  end
+
+  class DelimiterToken < Token
+    def initialize(value, position)
+      super(:delim, value, position)
+    end
+
+    def to_racc_token
+      [value, value]
+    end
+  end
+end

data/lib/antisamy/csspool/rsac/sac/tokenizer.rb

@@ -1,185 +1,185 @@
-require "antisamy/csspool/rsac/sac/lexeme"
-require "antisamy/csspool/rsac/sac/token"
-
-module RSAC
-  class Tokenizer
-    def initialize(&block)
-      @lexemes = []
-      @macros = {}
-
-      # http://www.w3.org/TR/CSS21/syndata.html
-      macro(:h, /([0-9a-f])/ )
-      macro(:nonascii, /([\200-\377])/ )
-      macro(:nl, /(\n|\r\n|\r|\f)/ )
-      macro(:unicode, /(\\#{m(:h)}{1,6}(\r\n|[ \t\r\n\f])?)/ )
-      macro(:escape, /(#{m(:unicode)}|\\[^\r\n\f0-9a-f])/ )
-      macro(:nmstart, /([_a-z]|#{m(:nonascii)}|#{m(:escape)})/ )
-      macro(:nmchar, /([_a-z0-9-]|#{m(:nonascii)}|#{m(:escape)})/ )
-      macro(:string1, /(\"([^\n\r\f\\\"]|\\#{m(:nl)}|#{m(:escape)})*\")/ )
-      macro(:string2, /(\'([^\n\r\f\\']|\\#{m(:nl)}|#{m(:escape)})*\')/ )
-      macro(:invalid1, /(\"([^\n\r\f\\\"]|\\#{m(:nl)}|#{m(:escape)})*)/ )
-      macro(:invalid2, /(\'([^\n\r\f\\']|\\#{m(:nl)}|#{m(:escape)})*)/ )
-      macro(:comment, /(\/\*[^*]*\*+([^\/*][^*]*\*+)*\/)/ )
-      macro(:ident, /(-?#{m(:nmstart)}#{m(:nmchar)}*)/ )
-      macro(:name, /(#{m(:nmchar)}+)/ )
-      macro(:num, /([0-9]+|[0-9]*\.[0-9]+)/ )
-      macro(:string, /(#{m(:string1)}|#{m(:string2)})/ )
-      macro(:invalid, /(#{m(:invalid1)}|#{m(:invalid2)})/ )
-      macro(:url, /(([!#\$%&*-~]|#{m(:nonascii)}|#{m(:escape)})*)/ )
-      macro(:s, /([ \t\r\n\f]+)/ )
-      macro(:w, /(#{m(:s)}?)/ )
-      macro(:A, /(a|\\0{0,4}(41|61)(\r\n|[ \t\r\n\f])?)/ )
-      macro(:C, /(c|\\0{0,4}(43|63)(\r\n|[ \t\r\n\f])?)/ )
-      macro(:D, /(d|\\0{0,4}(44|64)(\r\n|[ \t\r\n\f])?)/ )
-      macro(:E, /(e|\\0{0,4}(45|65)(\r\n|[ \t\r\n\f])?)/ )
-      macro(:G, /(g|\\0{0,4}(47|67)(\r\n|[ \t\r\n\f])?|\\g)/ )
-      macro(:H, /(h|\\0{0,4}(48|68)(\r\n|[ \t\r\n\f])?|\\h)/ )
-      macro(:I, /(i|\\0{0,4}(49|69)(\r\n|[ \t\r\n\f])?|\\i)/ )
-      macro(:K, /(k|\\0{0,4}(4b|6b)(\r\n|[ \t\r\n\f])?|\\k)/ )
-      macro(:M, /(m|\\0{0,4}(4d|6d)(\r\n|[ \t\r\n\f])?|\\m)/ )
-      macro(:N, /(n|\\0{0,4}(4e|6e)(\r\n|[ \t\r\n\f])?|\\n)/ )
-      macro(:O, /(o|\\0{0,4}(51|71)(\r\n|[ \t\r\n\f])?|\\o)/ )
-      macro(:P, /(p|\\0{0,4}(50|70)(\r\n|[ \t\r\n\f])?|\\p)/ )
-      macro(:R, /(r|\\0{0,4}(52|72)(\r\n|[ \t\r\n\f])?|\\r)/ )
-      macro(:S, /(s|\\0{0,4}(53|73)(\r\n|[ \t\r\n\f])?|\\s)/ )
-      macro(:T, /(t|\\0{0,4}(54|74)(\r\n|[ \t\r\n\f])?|\\t)/ )
-      macro(:X, /(x|\\0{0,4}(58|78)(\r\n|[ \t\r\n\f])?|\\x)/ )
-      macro(:Z, /(z|\\0{0,4}(5a|7a)(\r\n|[ \t\r\n\f])?|\\z)/ )
-
-      #token :COMMENT do |patterns|
-      #  patterns << /\/\*[^*]*\*+([^\/*][^*]*\*+)*\//
-      #  patterns << /#{m(:s)}+\/\*[^*]*\*+([^\/*][^*]*\*+)*\//
-      #end
-
-      token(:LBRACE, /#{m(:w)}\{/)
-      token(:PLUS, /#{m(:w)}\+/)
-      token(:GREATER, /#{m(:w)}>/)
-      token(:COMMA, /#{m(:w)},/)
-
-      token(:S, /#{m(:s)}/)
-
-      #token :URI do |patterns|
-      #  patterns << /url\(#{m(:w)}#{m(:string)}#{m(:w)}\)/
-      #  patterns << /url\(#{m(:w)}#{m(:url)}#{m(:w)}\)/
-      #end
-
-      token(:FUNCTION, /#{m(:ident)}\(/)
-      token(:IDENT, /#{m(:ident)}/)
-
-      token(:CDO, /<!--/)
-      token(:CDC, /-->/)
-      token(:INCLUDES, /~=/)
-      token(:DASHMATCH, /\|=/)
-      #token(:STRING, /#{m(:string)}/)
-      token(:INVALID, /#{m(:invalid)}/)
-      token(:HASH, /##{m(:name)}/)
-      token(:IMPORT_SYM, /@#{m(:I)}#{m(:M)}#{m(:P)}#{m(:O)}#{m(:R)}#{m(:T)}/)
-      token(:PAGE_SYM, /@#{m(:P)}#{m(:A)}#{m(:G)}#{m(:E)}/)
-      token(:MEDIA_SYM, /@#{m(:M)}#{m(:E)}#{m(:D)}#{m(:I)}#{m(:A)}/)
-      token(:CHARSET_SYM, /@#{m(:C)}#{m(:H)}#{m(:A)}#{m(:R)}#{m(:S)}#{m(:E)}#{m(:T)}/)
-      token(:IMPORTANT_SYM, /!(#{m(:w)}|#{m(:comment)})*#{m(:I)}#{m(:M)}#{m(:P)}#{m(:O)}#{m(:R)}#{m(:T)}#{m(:A)}#{m(:N)}#{m(:T)}/)
-      token(:EMS, /#{m(:num)}#{m(:E)}#{m(:M)}/)
-      token(:EXS, /#{m(:num)}#{m(:E)}#{m(:X)}/)
-
-      token :LENGTH do |patterns|
-        patterns << /#{m(:num)}#{m(:P)}#{m(:X)}/
-        patterns << /#{m(:num)}#{m(:C)}#{m(:M)}/
-        patterns << /#{m(:num)}#{m(:M)}#{m(:M)}/
-        patterns << /#{m(:num)}#{m(:I)}#{m(:N)}/
-        patterns << /#{m(:num)}#{m(:P)}#{m(:T)}/
-        patterns << /#{m(:num)}#{m(:P)}#{m(:C)}/
-      end
-
-      token :ANGLE do |patterns|
-        patterns << /#{m(:num)}#{m(:D)}#{m(:E)}#{m(:G)}/
-        patterns << /#{m(:num)}#{m(:R)}#{m(:A)}#{m(:D)}/
-        patterns << /#{m(:num)}#{m(:G)}#{m(:R)}#{m(:A)}#{m(:D)}/
-      end
-
-      token :TIME do |patterns|
-        patterns << /#{m(:num)}#{m(:M)}#{m(:S)}/
-        patterns << /#{m(:num)}#{m(:S)}/
-      end
-
-      token :FREQ do |patterns|
-        patterns << /#{m(:num)}#{m(:H)}#{m(:Z)}/
-        patterns << /#{m(:num)}#{m(:K)}#{m(:H)}#{m(:Z)}/
-      end
-
-      token(:DIMENSION, /#{m(:num)}#{m(:ident)}/)
-      token(:PERCENTAGE, /#{m(:num)}%/)
-      token(:NUMBER, /#{m(:num)}/)
-
-
-      yield self if block_given?
-    end
-
-    def tokenize(input_data)
-      tokens = []
-      pos = 0
-
-      comment_pattern = /\/\*.*?\*\//m
-      comments = input_data.scan(comment_pattern)
-      non_comments = input_data.split(comment_pattern)
-
-      # Handle a small edge case, if our CSS is *only* comments,
-      # the split, zip, scan trick won't work
-      if non_comments.length == 0
-        tokens = comments.map { |x| Token.new(:COMMENT, x, nil) }
-      else
-        non_comments.zip(comments).each do |non_comment, comment|
-          non_comment.split(/url\([^\)]*\)/m).zip(
-          non_comment.scan(/url\([^\)]*\)/m)
-          ).each do |non_url, url|
-            non_url.split(/"[^"]*"|'[^']*'/m).zip(
-            non_url.scan(/"[^"]*"|'[^']*'/m)
-            ).each do |non_string, quoted_string|
-              if non_string.length > 0 && non_string =~ /\A\s*\Z/m
-                tokens << Token.new(:S, non_string, nil)
-              else
-                non_string.split(/[ \t\r\n\f]*(?![{}+>]*)/m).zip(
-                non_string.scan(/[ \t\r\n\f]*(?![{}+>]*)/m)
-                ).each do |string, whitespace|
-                  until string.empty?
-                    token = nil
-                    @lexemes.each do |lexeme|
-                      match = lexeme.pattern.match(string)
-                      if match
-                        token = Token.new(lexeme.name, match.to_s, pos)
-                        break
-                      end
-                    end
-
-                    token ||= DelimiterToken.new(/^./.match(string).to_s, pos)
-
-                    tokens << token
-                    string = string.slice(Range.new(token.value.length, -1))
-                    pos += token.value.length
-                  end
-                  tokens << Token.new(:S, whitespace, nil) if whitespace
-                end
-              end
-              tokens << Token.new(:STRING, quoted_string, nil) if quoted_string
-            end
-            tokens << Token.new(:URI, url, nil) if url
-          end
-          tokens << Token.new(:COMMENT, comment, nil) if comment
-        end
-      end
-
-      tokens
-    end
-
-    private
-
-    def token(name, pattern=nil, &block)
-      @lexemes << Lexeme.new(name, pattern, &block)
-    end
-
-    def macro(name, regex=nil)
-      regex ? @macros[name] = regex : @macros[name].source
-    end
-
-    alias :m :macro
-  end
-end
+require "antisamy/csspool/rsac/sac/lexeme"
+require "antisamy/csspool/rsac/sac/token"
+
+module RSAC
+  class Tokenizer
+    def initialize(&block)
+      @lexemes = []
+      @macros = {}
+
+      # http://www.w3.org/TR/CSS21/syndata.html
+      macro(:h, /([0-9a-f])/ )
+      macro(:nonascii, /([\200-\377])/ )
+      macro(:nl, /(\n|\r\n|\r|\f)/ )
+      macro(:unicode, /(\\#{m(:h)}{1,6}(\r\n|[ \t\r\n\f])?)/ )
+      macro(:escape, /(#{m(:unicode)}|\\[^\r\n\f0-9a-f])/ )
+      macro(:nmstart, /([_a-z]|#{m(:nonascii)}|#{m(:escape)})/ )
+      macro(:nmchar, /([_a-z0-9-]|#{m(:nonascii)}|#{m(:escape)})/ )
+      macro(:string1, /(\"([^\n\r\f\\\"]|\\#{m(:nl)}|#{m(:escape)})*\")/ )
+      macro(:string2, /(\'([^\n\r\f\\']|\\#{m(:nl)}|#{m(:escape)})*\')/ )
+      macro(:invalid1, /(\"([^\n\r\f\\\"]|\\#{m(:nl)}|#{m(:escape)})*)/ )
+      macro(:invalid2, /(\'([^\n\r\f\\']|\\#{m(:nl)}|#{m(:escape)})*)/ )
+      macro(:comment, /(\/\*[^*]*\*+([^\/*][^*]*\*+)*\/)/ )
+      macro(:ident, /(-?#{m(:nmstart)}#{m(:nmchar)}*)/ )
+      macro(:name, /(#{m(:nmchar)}+)/ )
+      macro(:num, /([0-9]+|[0-9]*\.[0-9]+)/ )
+      macro(:string, /(#{m(:string1)}|#{m(:string2)})/ )
+      macro(:invalid, /(#{m(:invalid1)}|#{m(:invalid2)})/ )
+      macro(:url, /(([!#\$%&*-~]|#{m(:nonascii)}|#{m(:escape)})*)/ )
+      macro(:s, /([ \t\r\n\f]+)/ )
+      macro(:w, /(#{m(:s)}?)/ )
+      macro(:A, /(a|\\0{0,4}(41|61)(\r\n|[ \t\r\n\f])?)/ )
+      macro(:C, /(c|\\0{0,4}(43|63)(\r\n|[ \t\r\n\f])?)/ )
+      macro(:D, /(d|\\0{0,4}(44|64)(\r\n|[ \t\r\n\f])?)/ )
+      macro(:E, /(e|\\0{0,4}(45|65)(\r\n|[ \t\r\n\f])?)/ )
+      macro(:G, /(g|\\0{0,4}(47|67)(\r\n|[ \t\r\n\f])?|\\g)/ )
+      macro(:H, /(h|\\0{0,4}(48|68)(\r\n|[ \t\r\n\f])?|\\h)/ )
+      macro(:I, /(i|\\0{0,4}(49|69)(\r\n|[ \t\r\n\f])?|\\i)/ )
+      macro(:K, /(k|\\0{0,4}(4b|6b)(\r\n|[ \t\r\n\f])?|\\k)/ )
+      macro(:M, /(m|\\0{0,4}(4d|6d)(\r\n|[ \t\r\n\f])?|\\m)/ )
+      macro(:N, /(n|\\0{0,4}(4e|6e)(\r\n|[ \t\r\n\f])?|\\n)/ )
+      macro(:O, /(o|\\0{0,4}(51|71)(\r\n|[ \t\r\n\f])?|\\o)/ )
+      macro(:P, /(p|\\0{0,4}(50|70)(\r\n|[ \t\r\n\f])?|\\p)/ )
+      macro(:R, /(r|\\0{0,4}(52|72)(\r\n|[ \t\r\n\f])?|\\r)/ )
+      macro(:S, /(s|\\0{0,4}(53|73)(\r\n|[ \t\r\n\f])?|\\s)/ )
+      macro(:T, /(t|\\0{0,4}(54|74)(\r\n|[ \t\r\n\f])?|\\t)/ )
+      macro(:X, /(x|\\0{0,4}(58|78)(\r\n|[ \t\r\n\f])?|\\x)/ )
+      macro(:Z, /(z|\\0{0,4}(5a|7a)(\r\n|[ \t\r\n\f])?|\\z)/ )
+
+      #token :COMMENT do |patterns|
+      #  patterns << /\/\*[^*]*\*+([^\/*][^*]*\*+)*\//
+      #  patterns << /#{m(:s)}+\/\*[^*]*\*+([^\/*][^*]*\*+)*\//
+      #end
+
+      token(:LBRACE, /#{m(:w)}\{/)
+      token(:PLUS, /#{m(:w)}\+/)
+      token(:GREATER, /#{m(:w)}>/)
+      token(:COMMA, /#{m(:w)},/)
+
+      token(:S, /#{m(:s)}/)
+
+      #token :URI do |patterns|
+      #  patterns << /url\(#{m(:w)}#{m(:string)}#{m(:w)}\)/
+      #  patterns << /url\(#{m(:w)}#{m(:url)}#{m(:w)}\)/
+      #end
+
+      token(:FUNCTION, /#{m(:ident)}\(/)
+      token(:IDENT, /#{m(:ident)}/)
+
+      token(:CDO, /<!--/)
+      token(:CDC, /-->/)
+      token(:INCLUDES, /~=/)
+      token(:DASHMATCH, /\|=/)
+      #token(:STRING, /#{m(:string)}/)
+      token(:INVALID, /#{m(:invalid)}/)
+      token(:HASH, /##{m(:name)}/)
+      token(:IMPORT_SYM, /@#{m(:I)}#{m(:M)}#{m(:P)}#{m(:O)}#{m(:R)}#{m(:T)}/)
+      token(:PAGE_SYM, /@#{m(:P)}#{m(:A)}#{m(:G)}#{m(:E)}/)
+      token(:MEDIA_SYM, /@#{m(:M)}#{m(:E)}#{m(:D)}#{m(:I)}#{m(:A)}/)
+      token(:CHARSET_SYM, /@#{m(:C)}#{m(:H)}#{m(:A)}#{m(:R)}#{m(:S)}#{m(:E)}#{m(:T)}/)
+      token(:IMPORTANT_SYM, /!(#{m(:w)}|#{m(:comment)})*#{m(:I)}#{m(:M)}#{m(:P)}#{m(:O)}#{m(:R)}#{m(:T)}#{m(:A)}#{m(:N)}#{m(:T)}/)
+      token(:EMS, /#{m(:num)}#{m(:E)}#{m(:M)}/)
+      token(:EXS, /#{m(:num)}#{m(:E)}#{m(:X)}/)
+
+      token :LENGTH do |patterns|
+        patterns << /#{m(:num)}#{m(:P)}#{m(:X)}/
+        patterns << /#{m(:num)}#{m(:C)}#{m(:M)}/
+        patterns << /#{m(:num)}#{m(:M)}#{m(:M)}/
+        patterns << /#{m(:num)}#{m(:I)}#{m(:N)}/
+        patterns << /#{m(:num)}#{m(:P)}#{m(:T)}/
+        patterns << /#{m(:num)}#{m(:P)}#{m(:C)}/
+      end
+
+      token :ANGLE do |patterns|
+        patterns << /#{m(:num)}#{m(:D)}#{m(:E)}#{m(:G)}/
+        patterns << /#{m(:num)}#{m(:R)}#{m(:A)}#{m(:D)}/
+        patterns << /#{m(:num)}#{m(:G)}#{m(:R)}#{m(:A)}#{m(:D)}/
+      end
+
+      token :TIME do |patterns|
+        patterns << /#{m(:num)}#{m(:M)}#{m(:S)}/
+        patterns << /#{m(:num)}#{m(:S)}/
+      end
+
+      token :FREQ do |patterns|
+        patterns << /#{m(:num)}#{m(:H)}#{m(:Z)}/
+        patterns << /#{m(:num)}#{m(:K)}#{m(:H)}#{m(:Z)}/
+      end
+
+      token(:DIMENSION, /#{m(:num)}#{m(:ident)}/)
+      token(:PERCENTAGE, /#{m(:num)}%/)
+      token(:NUMBER, /#{m(:num)}/)
+
+
+      yield self if block_given?
+    end
+
+    def tokenize(input_data)
+      tokens = []
+      pos = 0
+
+      comment_pattern = /\/\*.*?\*\//m
+      comments = input_data.scan(comment_pattern)
+      non_comments = input_data.split(comment_pattern)
+
+      # Handle a small edge case, if our CSS is *only* comments,
+      # the split, zip, scan trick won't work
+      if non_comments.length == 0
+        tokens = comments.map { |x| Token.new(:COMMENT, x, nil) }
+      else
+        non_comments.zip(comments).each do |non_comment, comment|
+          non_comment.split(/url\([^\)]*\)/m).zip(
+          non_comment.scan(/url\([^\)]*\)/m)
+          ).each do |non_url, url|
+            non_url.split(/"[^"]*"|'[^']*'/m).zip(
+            non_url.scan(/"[^"]*"|'[^']*'/m)
+            ).each do |non_string, quoted_string|
+              if non_string.length > 0 && non_string =~ /\A\s*\Z/m
+                tokens << Token.new(:S, non_string, nil)
+              else
+                non_string.split(/[ \t\r\n\f]*(?![{}+>]*)/m).zip(
+                non_string.scan(/[ \t\r\n\f]*(?![{}+>]*)/m)
+                ).each do |string, whitespace|
+                  until string.empty?
+                    token = nil
+                    @lexemes.each do |lexeme|
+                      match = lexeme.pattern.match(string)
+                      if match
+                        token = Token.new(lexeme.name, match.to_s, pos)
+                        break
+                      end
+                    end
+
+                    token ||= DelimiterToken.new(/^./.match(string).to_s, pos)
+
+                    tokens << token
+                    string = string.slice(Range.new(token.value.length, -1))
+                    pos += token.value.length
+                  end
+                  tokens << Token.new(:S, whitespace, nil) if whitespace
+                end
+              end
+              tokens << Token.new(:STRING, quoted_string, nil) if quoted_string
+            end
+            tokens << Token.new(:URI, url, nil) if url
+          end
+          tokens << Token.new(:COMMENT, comment, nil) if comment
+        end
+      end
+
+      tokens
+    end
+
+    private
+
+    def token(name, pattern=nil, &block)
+      @lexemes << Lexeme.new(name, pattern, &block)
+    end
+
+    def macro(name, regex=nil)
+      regex ? @macros[name] = regex : @macros[name].source
+    end
+
+    alias :m :macro
+  end
+end