crass 0.0.2 → 0.1.0

data/lib/crass/scanner.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require 'strscan'
 
 module Crass
 
@@ -17,13 +18,12 @@ module Crass
     # position, not a byte position, so it accounts for multi-byte characters.
     attr_accessor :pos
 
-    # The string being scanned.
-    attr_reader :string
-
     # Creates a Scanner instance for the given _input_ string or IO instance.
     def initialize(input)
-      @string = input.is_a?(IO) ? input.read : input.to_s
-      @chars  = @string.chars.to_a
+      string = input.is_a?(IO) ? input.read : input.to_s
+
+      @chars   = string.chars.to_a
+      @scanner = StringScanner.new(string)
 
       reset
     end
@@ -31,20 +31,20 @@ module Crass
     # Consumes the next character and returns it, advancing the pointer, or
     # an empty string if the end of the string has been reached.
     def consume
-      @current = @chars[@pos] || ''
-      @pos += 1 if @current
-      @current
+      if @pos == @len
+        ''
+      else
+        @pos += 1
+        @current = @scanner.getch
+      end
     end
 
     # Consumes the rest of the string and returns it, advancing the pointer to
     # the end of the string. Returns an empty string is the end of the string
     # has already been reached.
     def consume_rest
-      rest     = @string[@pos..@len] || ''
-      @current = rest[-1] || ''
-      @pos     = @len
-
-      rest
+      @pos = @len
+      @scanner.rest
     end
 
     # Returns `true` if the end of the string has been reached, `false`
@@ -62,7 +62,7 @@ module Crass
     # Returns the substring between {#marker} and {#pos}, without altering the
     # pointer.
     def marked
-      if result = @chars[@marker...@pos]
+      if result = @chars[@marker, @pos - @marker]
         result.join('')
       else
         ''
@@ -73,24 +73,21 @@ module Crass
     # doesn't consume them. The number of characters returned may be less than
     # _length_ if the end of the string is reached.
     def peek(length = 1)
-      if result = @chars[@pos, length]
-        result.join('')
-      else
-        ''
-      end
+      @scanner.peek(length)
     end
 
     # Moves the pointer back one character without changing the value of
     # {#current}. The next call to {#consume} will re-consume the current
     # character.
     def reconsume
+      @scanner.unscan
       @pos -= 1 if @pos > 0
     end
 
     # Resets the pointer to the beginning of the string.
     def reset
       @current = nil
-      @len     = @string.length
+      @len     = @chars.size
       @marker  = 0
       @pos     = 0
     end
@@ -99,29 +96,30 @@ module Crass
     # matched substring will be returned and the pointer will be advanced.
     # Otherwise, `nil` will be returned.
     def scan(pattern)
-      match = pattern.match(@string, @pos)
-      return nil if match.nil? || match.begin(0) != @pos
-
-      @pos     = match.end(0)
-      @current = @chars[@pos - 1]
+      if match = @scanner.scan(pattern)
+        @pos     += match.size
+        @current  = @chars[@pos - 1]
+      end
 
-      match[0]
+      match
     end
 
     # Scans the string until the _pattern_ is matched. Returns the substring up
     # to and including the end of the match, and advances the pointer. If there
     # is no match, `nil` is returned and the pointer is not advanced.
     def scan_until(pattern)
-      start = @pos
-      match = pattern.match(@string, @pos)
-
-      return nil if match.nil?
-
-      @pos     = match.end(0)
-      @current = @chars[@pos - 1]
+      if match = @scanner.scan_until(pattern)
+        @pos     += match.size
+        @current  = @chars[@pos - 1]
+      end
 
-      @string[start...@pos]
+      match
     end
   end
 
+  # Returns the string being scanned.
+  def string
+    @scanner.string
+  end
+
 end

data/lib/crass/tokenizer.rb

@@ -32,7 +32,6 @@ module Crass
 
     RE_UNICODE_RANGE_START = /\+(?:[0-9A-Fa-f]|\?)/
     RE_UNICODE_RANGE_END   = /-[0-9A-Fa-f]/
-    RE_URL_QUOTE           = /["']/
     RE_WHITESPACE          = /[\n\u0009\u0020]+/
 
     # -- Class Methods ---------------------------------------------------------
@@ -67,47 +66,47 @@ module Crass
     #
     # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-token0
     def consume
-      return token(:eof) if @s.eos?
+      return nil if @s.eos?
 
       @s.mark
-      return token(:whitespace) if @s.scan(RE_WHITESPACE)
+      return create_token(:whitespace) if @s.scan(RE_WHITESPACE)
 
-      case char = @s.consume
-      when '"'
+      char = @s.consume
+
+      case char.to_sym
+      when :'"'
         consume_string('"')
 
-      when '#'
+      when :'#'
         if @s.peek =~ RE_NAME || valid_escape?
-          value = consume_name
-
-          token(:hash,
+          create_token(:hash,
             :type  => start_identifier? ? :id : :unrestricted,
-            :value => value)
+            :value => consume_name)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '$'
+      when :'$'
         if @s.peek == '='
           @s.consume
-          token(:suffix_match)
+          create_token(:suffix_match)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when "'"
+      when :"'"
         consume_string("'")
 
-      when '('
-        token(:'(')
+      when :'('
+        create_token(:'(')
 
-      when ')'
-        token(:')')
+      when :')'
+        create_token(:')')
 
-      when '*'
+      when :*
         if @s.peek == '='
           @s.consume
-          token(:substring_match)
+          create_token(:substring_match)
 
         elsif @options[:preserve_hacks] && @s.peek =~ RE_NAME_START
           # NON-STANDARD: IE * hack
@@ -115,121 +114,117 @@ module Crass
           consume_ident
 
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '+'
-        if start_number?
+      when :+
+        if start_number?(char + @s.peek(2))
           @s.reconsume
           consume_numeric
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when ','
-        token(:comma)
+      when :','
+        create_token(:comma)
 
-      when '-'
-        if start_number?
+      when :-
+        if start_number?(char + @s.peek(2))
           @s.reconsume
           consume_numeric
-        elsif start_identifier?
+        elsif start_identifier?(char + @s.peek(2))
           @s.reconsume
           consume_ident
         elsif @s.peek(2) == '->'
           @s.consume
           @s.consume
-          token(:cdc)
+          create_token(:cdc)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '.'
-        if start_number?
+      when :'.'
+        if start_number?(char + @s.peek(2))
           @s.reconsume
           consume_numeric
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '/'
+      when :/
         if @s.peek == '*'
           @s.consume
 
           if text = @s.scan_until(RE_COMMENT_CLOSE)
             text.slice!(-2, 2)
           else
-            text = @s.rest
+            text = @s.consume_rest
           end
 
           if @options[:preserve_comments]
-            token(:comment, :value => text)
+            create_token(:comment, :value => text)
           else
             consume
           end
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when ':'
-        token(:colon)
+      when :':'
+        create_token(:colon)
 
-      when ';'
-        token(:semicolon)
+      when :';'
+        create_token(:semicolon)
 
-      when '<'
+      when :<
         if @s.peek(3) == '!--'
           @s.consume
           @s.consume
           @s.consume
 
-          token(:cdo)
+          create_token(:cdo)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '@'
+      when :'@'
         if start_identifier?
-          token(:at_keyword, :value => consume_name)
+          create_token(:at_keyword, :value => consume_name)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '['
-        token(:'[')
+      when :'['
+        create_token(:'[')
 
-      when '\\'
+      when :'\\'
         if valid_escape?(char + @s.peek)
           @s.reconsume
           consume_ident
         else
-          token(:delim,
+          create_token(:delim,
             :error => true,
             :value => char)
         end
 
-      when ']'
-        token(:']')
+      when :']'
+        create_token(:']')
 
-      when '^'
+      when :'^'
         if @s.peek == '='
           @s.consume
-          token(:prefix_match)
+          create_token(:prefix_match)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '{'
-        token(:'{')
+      when :'{'
+        create_token(:'{')
 
-      when '}'
-        token(:'}')
-
-      when RE_DIGIT
-        @s.reconsume
-        consume_numeric
+      when :'}'
+        create_token(:'}')
 
-      when 'U', 'u'
+      when :U, :u
         if @s.peek(2) =~ RE_UNICODE_RANGE_START
           @s.consume
           consume_unicode_range
@@ -238,58 +233,68 @@ module Crass
           consume_ident
         end
 
-      when RE_NAME_START
-        @s.reconsume
-        consume_ident
-
-      when '|'
+      when :|
         case @s.peek
         when '='
           @s.consume
-          token(:dash_match)
+          create_token(:dash_match)
 
         when '|'
           @s.consume
-          token(:column)
+          create_token(:column)
 
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
-      when '~'
+      when :~
         if @s.peek == '='
           @s.consume
-          token(:include_match)
+          create_token(:include_match)
         else
-          token(:delim, :value => char)
+          create_token(:delim, :value => char)
         end
 
       else
-        token(:delim, :value => char)
+        case char
+        when RE_DIGIT
+          @s.reconsume
+          consume_numeric
+
+        when RE_NAME_START
+          @s.reconsume
+          consume_ident
+
+        else
+          create_token(:delim, :value => char)
+        end
       end
     end
 
     # Consumes the remnants of a bad URL and returns the consumed text.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-the-remnants-of-a-bad-url0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-the-remnants-of-a-bad-url
     def consume_bad_url
       text = ''
 
-      while true
-        return text if @s.eos?
-
-        if valid_escape?
+      until @s.eos?
+        if valid_escape?(@s.current + @s.peek)
+          text << consume_escaped
+        elsif valid_escape?
+          @s.consume
           text << consume_escaped
         else
           char = @s.consume
 
           if char == ')'
-            return text
+            break
           else
             text << char
           end
         end
       end
+
+      text
     end
 
     # Consumes an escaped code point and returns its unescaped value.
@@ -325,33 +330,39 @@ module Crass
 
     # Consumes an ident-like token and returns it.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-an-ident-like-token0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-an-ident-like-token
     def consume_ident
       value = consume_name
 
-      if value.downcase == 'url' && @s.peek == '('
+      if @s.peek == '('
         @s.consume
-        consume_url
-      elsif @s.peek == '('
-        @s.consume
-        token(:function, :value => value)
+
+        if value.downcase == 'url'
+          consume_url
+        else
+          create_token(:function, :value => value)
+        end
       else
-        token(:ident, :value => value)
+        create_token(:ident, :value => value)
       end
     end
 
     # Consumes a name and returns it.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-name0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-name
     def consume_name
       result = ''
 
-      while char = @s.peek
-        if char =~ RE_NAME
-          result << @s.consume
+      while true
+        if match = @s.scan(RE_NAME)
+          result << match
+          next
+        end
 
-        elsif char == '\\' && valid_escape?
-          result << @s.consume
+        char = @s.peek
+
+        if char == '\\' && valid_escape?
+          @s.consume
           result << consume_escaped
 
         # NON-STANDARD: IE * hack
@@ -391,12 +402,12 @@ module Crass
 
     # Consumes a numeric token and returns it.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-numeric-token0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-numeric-token
     def consume_numeric
       number = consume_number
 
       if start_identifier?
-        token(:dimension,
+        create_token(:dimension,
           :repr  => number[0],
           :type  => number[2],
           :unit  => consume_name,
@@ -405,12 +416,13 @@ module Crass
       elsif @s.peek == '%'
         @s.consume
 
-        token(:percentage,
+        create_token(:percentage,
           :repr  => number[0],
+          :type  => number[2],
           :value => number[1])
 
       else
-        token(:number,
+        create_token(:number,
           :repr  => number[0],
           :type  => number[2],
           :value => number[1])
@@ -420,16 +432,18 @@ module Crass
     # Consumes a string token that ends at the given character, and returns the
     # token.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-string-token0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-string-token
     def consume_string(ending)
       value = ''
 
-      while char = @s.consume
-        case char
-        when ending then break
+      until @s.eos?
+        case char = @s.consume
+        when ending
+          break
 
         when "\n"
-          return token(:bad_string,
+          @s.reconsume
+          return create_token(:bad_string,
             :error => true,
             :value => value)
 
@@ -443,7 +457,7 @@ module Crass
             @s.consume
 
           else
-            value += consume_escaped
+            value << consume_escaped
           end
 
         else
@@ -451,15 +465,15 @@ module Crass
         end
       end
 
-      token(:string, :value => value)
+      create_token(:string, :value => value)
     end
 
     # Consumes a Unicode range token and returns it. Assumes the initial "u+" or
     # "U+" has already been consumed.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-unicode-range-token0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-unicode-range-token
     def consume_unicode_range
-      value = @s.scan(RE_HEX)
+      value = @s.scan(RE_HEX) || ''
 
       while value.length < 6
         break unless @s.peek == '?'
@@ -471,37 +485,43 @@ module Crass
       if value.include?('?')
         range[:start] = value.gsub('?', '0').hex
         range[:end]   = value.gsub('?', 'F').hex
-        return token(:unicode_range, range)
+        return create_token(:unicode_range, range)
       end
 
       range[:start] = value.hex
 
       if @s.peek(2) =~ RE_UNICODE_RANGE_END
-        range[:value] << @s.consume << end_value = @s.scan(RE_HEX)
-        range[:end] = end_value.hex
+        @s.consume
+        range[:end] = (@s.scan(RE_HEX) || '').hex
       else
         range[:end] = range[:start]
       end
 
-      token(:unicode_range, range)
+      create_token(:unicode_range, range)
     end
 
     # Consumes a URL token and returns it. Assumes the original "url(" has
     # already been consumed.
     #
-    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-url-token0
+    # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#consume-a-url-token
     def consume_url
       value = ''
 
       @s.scan(RE_WHITESPACE)
-      return token(:url, :value => value) if @s.eos?
+
+      if @s.eos?
+        return create_token(:url, :value => value)
+      end
 
       # Quoted URL.
-      if @s.peek =~ RE_URL_QUOTE
+      next_char = @s.peek
+
+      if next_char == "'" || next_char == '"'
         string = consume_string(@s.consume)
 
         if string[:node] == :bad_string
-          return token(:bad_url, :value => string[:value] + consume_bad_url)
+          return create_token(:bad_url,
+            :value => string[:value] + consume_bad_url)
         end
 
         value = string[:value]
@@ -509,16 +529,17 @@ module Crass
 
         if @s.eos? || @s.peek == ')'
           @s.consume
-          return token(:url, :value => value)
+          return create_token(:url, :value => value)
         else
-          return token(:bad_url, :value => value + consume_bad_url)
+          return create_token(:bad_url, :value => value + consume_bad_url)
         end
       end
 
       # Unquoted URL.
-      while !@s.eos?
+      until @s.eos?
         case char = @s.consume
-        when ')' then break
+        when ')'
+          break
 
         when RE_WHITESPACE
           @s.scan(RE_WHITESPACE)
@@ -527,19 +548,19 @@ module Crass
             @s.consume
             break
           else
-            return token(:bad_url, :value => value + consume_bad_url)
+            return create_token(:bad_url, :value => value + consume_bad_url)
           end
 
         when '"', "'", '(', RE_NON_PRINTABLE
-          return token(:bad_url,
+          return create_token(:bad_url,
             :error => true,
             :value => value + consume_bad_url)
 
         when '\\'
-          if valid_escape?
+          if valid_escape?(char + @s.peek)
             value << consume_escaped
           else
-            return token(:bad_url,
+            return create_token(:bad_url,
               :error => true,
               :value => value + consume_bad_url
             )
@@ -550,7 +571,7 @@ module Crass
         end
       end
 
-      token(:url, :value => value)
+      create_token(:url, :value => value)
     end
 
     # Converts a valid CSS number string into a number and returns the number.
@@ -571,6 +592,15 @@ module Crass
       s * (i + f * 10**-d) * 10**(t * e)
     end
 
+    # Creates and returns a new token with the given _properties_.
+    def create_token(type, properties = {})
+      {
+        :node => type,
+        :pos  => @s.marker,
+        :raw  => @s.marked
+      }.merge!(properties)
+    end
+
     # Preprocesses _input_ to prepare it for the tokenizer.
     #
     # http://www.w3.org/TR/2013/WD-css-syntax-3-20130919/#input-preprocessing
@@ -630,25 +660,14 @@ module Crass
       end
     end
 
-    # Creates and returns a new token with the given _properties_.
-    def token(type, properties = {})
-      {
-        :node => type,
-        :pos  => @s.marker,
-        :raw  => @s.marked
-      }.merge!(properties)
-    end
-
     # Tokenizes the input stream and returns an array of tokens.
     def tokenize
       @s.reset
 
       tokens = []
-      token  = consume
 
-      while token && token[:node] != :eof
+      while token = consume
         tokens << token
-        token = consume
       end
 
       tokens