csspool 2.0.0 → 3.0.0

data/lib/csspool/css/parser.y

@@ -0,0 +1,347 @@
+class CSSPool::CSS::Parser
+
+token CHARSET_SYM IMPORT_SYM STRING SEMI IDENT S COMMA LBRACE RBRACE STAR HASH
+token LSQUARE RSQUARE EQUAL INCLUDES DASHMATCH RPAREN FUNCTION GREATER PLUS
+token SLASH NUMBER MINUS LENGTH PERCENTAGE EMS EXS ANGLE TIME FREQ URI
+token IMPORTANT_SYM MEDIA_SYM
+
+rule
+  document
+    : { @handler.start_document }
+      stylesheet
+      { @handler.end_document }
+    ;
+  stylesheet
+    : charset stylesheet
+    | import stylesheet
+    | charset
+    | import
+    | body
+    ;
+  charset
+    : CHARSET_SYM STRING SEMI { @handler.charset interpret_string(val[1]), {} }
+    ;
+  import
+    : IMPORT_SYM import_location medium SEMI {
+        @handler.import_style [val[2]].flatten, val[1]
+      }
+    | IMPORT_SYM import_location SEMI {
+        @handler.import_style [], val[1]
+      }
+    ;
+  import_location
+    : import_location S
+    | STRING { result = Terms::String.new interpret_string val.first }
+    | URI { result = Terms::URI.new interpret_uri val.first }
+    ;
+  medium
+    : medium COMMA IDENT {
+        result = [val.first, Terms::Ident.new(interpret_identifier val.last)]
+      }
+    | IDENT {
+        result = Terms::Ident.new interpret_identifier val.first
+      }
+    ;
+  body
+    : ruleset body
+    | media body
+    | ruleset
+    | media
+    ;
+  media
+    : start_media body RBRACE { @handler.end_media val.first }
+    ;
+  start_media
+    : MEDIA_SYM medium LBRACE {
+        result = [val[1]].flatten
+        @handler.start_media result
+      }
+    | MEDIA_SYM LBRACE { result = [] }
+    ;
+  ruleset
+    : start_selector declarations RBRACE {
+        @handler.end_selector val.first
+      }
+    | start_selector RBRACE {
+        @handler.end_selector val.first
+      }
+    ;
+  start_selector
+    : S start_selector { result = val.last }
+    | selectors LBRACE {
+        @handler.start_selector val.first
+      }
+    ;
+  selectors
+    : selector COMMA selectors
+      {
+        # FIXME: should always garantee array
+        sel = Selector.new(val.first, {})
+        result = [sel, val[2]].flatten
+      }
+    | selector
+      {
+        result = [Selector.new(val.first, {})]
+      }
+    ;
+  selector
+    : simple_selector combinator selector
+      {
+        val = val.flatten
+        val[2].combinator = val.delete_at 1
+        result = val
+      }
+    | simple_selector
+    ;
+  combinator
+    : S       { result = :s }
+    | GREATER { result = :> }
+    | PLUS    { result = :+ }
+    ;
+  simple_selector
+    : element_name hcap {
+        selector = val.first
+        selector.additional_selectors = val.last
+        result = [selector]
+      }
+    | element_name { result = val }
+    | hcap
+      {
+        ss = Selectors::Simple.new nil, nil
+        ss.additional_selectors = val.flatten
+        result = [ss]
+      }
+    ;
+  element_name
+    : IDENT { result = Selectors::Type.new interpret_identifier val.first }
+    | STAR  { result = Selectors::Universal.new val.first }
+    ;
+  hcap
+    : hash        { result = val }
+    | class       { result = val }
+    | attrib      { result = val }
+    | pseudo      { result = val }
+    | hash hcap   { result = val.flatten }
+    | class hcap  { result = val.flatten }
+    | attrib hcap { result = val.flatten }
+    | pseudo hcap { result = val.flatten }
+    ;
+  hash
+    : HASH {
+        result = Selectors::Id.new interpret_identifier val.first.sub(/^#/, '')
+      }
+  class
+    : '.' IDENT {
+        result = Selectors::Class.new interpret_identifier val.last
+      }
+    ;
+  attrib
+    : LSQUARE IDENT EQUAL IDENT RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_identifier(val[3]),
+          Selectors::Attribute::EQUALS
+        )
+      }
+    | LSQUARE IDENT EQUAL STRING RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_string(val[3]),
+          Selectors::Attribute::EQUALS
+        )
+      }
+    | LSQUARE IDENT INCLUDES STRING RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_string(val[3]),
+          Selectors::Attribute::INCLUDES
+        )
+      }
+    | LSQUARE IDENT INCLUDES IDENT RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_identifier(val[3]),
+          Selectors::Attribute::INCLUDES
+        )
+      }
+    | LSQUARE IDENT DASHMATCH IDENT RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_identifier(val[3]),
+          Selectors::Attribute::DASHMATCH
+        )
+      }
+    | LSQUARE IDENT DASHMATCH STRING RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          interpret_string(val[3]),
+          Selectors::Attribute::DASHMATCH
+        )
+      }
+    | LSQUARE IDENT RSQUARE {
+        result = Selectors::Attribute.new(
+          interpret_identifier(val[1]),
+          nil,
+          Selectors::Attribute::SET
+        )
+      }
+    ;
+  pseudo
+    : ':' IDENT {
+        result = Selectors::PseudoClass.new(
+          interpret_identifier(val[1])
+        )
+      }
+    | ':' FUNCTION RPAREN {
+        result = Selectors::PseudoClass.new(
+          interpret_identifier(val[1].sub(/\($/, '')),
+          ''
+        )
+      }
+    | ':' FUNCTION IDENT RPAREN {
+        result = Selectors::PseudoClass.new(
+          interpret_identifier(val[1].sub(/\($/, '')),
+          interpret_identifier(val[2])
+        )
+      }
+    ;
+  declarations
+    : declaration declarations
+    | declaration
+    ;
+  declaration
+    : property ':' expr prio SEMI
+      { @handler.property val.first, val[2], val[3] }
+    | property ':' S expr prio SEMI
+      { @handler.property val.first, val[3], val[4] }
+    ;
+  prio
+    : IMPORTANT_SYM { result = true }
+    |               { result = false }
+    ;
+  property
+    : IDENT { result = interpret_identifier val[0] }
+    ;
+  operator
+    : COMMA
+    | SLASH
+    | EQUAL
+    ;
+  expr
+    : term operator expr {
+        result = [val.first, val.last].flatten
+        val.last.first.operator = val[1]
+      }
+    | term expr { result = val.flatten }
+    | term      { result = val }
+    ;
+  term
+    : ident
+    | numeric
+    | string
+    | uri
+    | hexcolor
+    | function
+    ;
+  function
+    : function S { result = val.first }
+    | FUNCTION expr RPAREN {
+        name = interpret_identifier val.first.sub(/\($/, '')
+        if name == 'rgb'
+          result = Terms::Rgb.new(*val[1])
+        else
+          result = Terms::Function.new name, val[1]
+        end
+      }
+    ;
+  hexcolor
+    : hexcolor S { result = val.first }
+    | HASH { result = Terms::Hash.new val.first.sub(/^#/, '') }
+    ;
+  uri
+    : uri S { result = val.first }
+    | URI { result = Terms::URI.new interpret_uri val.first }
+  string
+    : string S { result = val.first }
+    | STRING { result = Terms::String.new interpret_string val.first }
+    ;
+  numeric
+    : unary_operator numeric {
+        result = val[1]
+        val[1].unary_operator = val.first
+      }
+    | NUMBER {
+        result = Terms::Number.new numeric val.first
+      }
+    | PERCENTAGE {
+        result = Terms::Number.new numeric(val.first), nil, '%'
+      }
+    | LENGTH {
+        unit    = val.first.gsub(/[\s\d.]/, '')
+        result = Terms::Number.new numeric(val.first), nil, unit
+      }
+    | EMS {
+        result = Terms::Number.new numeric(val.first), nil, 'em'
+      }
+    | EXS {
+        result = Terms::Number.new numeric(val.first), nil, 'ex'
+      }
+    | ANGLE {
+        unit    = val.first.gsub(/[\s\d.]/, '')
+        result = Terms::Number.new numeric(val.first), nil, unit
+      }
+    | TIME {
+        unit    = val.first.gsub(/[\s\d.]/, '')
+        result = Terms::Number.new numeric(val.first), nil, unit
+      }
+    | FREQ {
+        unit    = val.first.gsub(/[\s\d.]/, '')
+        result = Terms::Number.new numeric(val.first), nil, unit
+      }
+    ;
+  unary_operator
+    : MINUS { result = :minus }
+    | PLUS  { result = :plus }
+    ;
+  ident
+    : ident S { result = val.first }
+    | IDENT { result = Terms::Ident.new interpret_identifier val.first }
+    ;
+
+---- inner
+
+def numeric thing
+  thing = thing.gsub(/[^\d.]/, '')
+  Integer(thing) rescue Float(thing)
+end
+
+def interpret_identifier s
+  interpret_escapes s
+end
+
+def interpret_uri s
+  interpret_escapes s.match(/^url\((.*)\)$/mu)[1].strip.match(/^(['"]?)((?:\\.|.)*)\1$/mu)[2]
+end
+
+def interpret_string s
+  interpret_escapes s.match(/^(['"])((?:\\.|.)*)\1$/mu)[2]
+end
+
+def interpret_escapes s
+  token_exp = /\\([0-9a-fA-F]{1,6}(?:\r\n|\s)?)|\\(.)|(.)/mu
+  characters = s.scan(token_exp).map do |u_escape, i_escape, ident|
+    if u_escape
+      code = u_escape.chomp.to_i 16
+      code = 0xFFFD if code > 0x10FFFF
+      [code].pack('U')
+    elsif i_escape
+      if i_escape == "\n"
+        ''
+      else
+        i_escape
+      end
+    else
+      ident
+    end
+  end.join ''
+end

data/lib/csspool/css/rule_set.rb

@@ -1,11 +1,16 @@
 module CSSPool
   module CSS
-    class RuleSet < Struct.new(:selectors, :declarations, :media)
-      include CSSPool::Visitable
+    class RuleSet < CSSPool::Node
+      attr_accessor :selectors
+      attr_accessor :declarations
+      attr_accessor :media
 
       def initialize selectors, declarations = [], media = []
+        @selectors    = selectors
+        @declarations = declarations
+        @media        = media
+
         selectors.each { |sel| sel.rule_set = self }
-        super
       end
     end
   end

data/lib/csspool/css/tokenizer.rb

@@ -0,0 +1,228 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "lib/csspool/css/tokenizer.rex".
+#++
+
+module CSSPool
+module CSS
+class Tokenizer < Parser
+  require 'strscan'
+
+  class ScanError < StandardError ; end
+
+  attr_reader   :lineno
+  attr_reader   :filename
+  attr_accessor :state
+
+  def scan_setup(str)
+    @ss = StringScanner.new(str)
+    @lineno =  1
+    @state  = nil
+  end
+
+  def action
+    yield
+  end
+
+  def scan_str(str)
+    scan_setup(str)
+    do_parse
+  end
+  alias :scan :scan_str
+
+  def load_file( filename )
+    @filename = filename
+    open(filename, "r") do |f|
+      scan_setup(f.read)
+    end
+  end
+
+  def scan_file( filename )
+    load_file(filename)
+    do_parse
+  end
+
+
+  def next_token
+    return if @ss.eos?
+    
+    # skips empty actions
+    until token = _next_token or @ss.eos?; end
+    token
+  end
+
+  def _next_token
+    text = @ss.peek(1)
+    @lineno  +=  1  if text == "\n"
+    token = case @state
+    when nil
+      case
+      when (text = @ss.scan(/url\([\s]*("([^\n\r\f\\"]|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')[\s]*\)/i))
+         action { [:URI, st(text)] }
+
+      when (text = @ss.scan(/url\([\s]*([!#\$%&*-~]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*[\s]*\)/i))
+         action { [:URI, st(text)] }
+
+      when (text = @ss.scan(/U\+[0-9a-fA-F?]{1,6}(-[0-9a-fA-F]{1,6})?/i))
+         action {[:UNICODE_RANGE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\/\*(.|[\s]*)*?\*\/[\s]*/i))
+         action { next_token }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f]|[.])*\(\s*/i))
+         action { [:FUNCTION, st(text)] }
+
+      when (text = @ss.scan(/[\s]*@import[\s]*/i))
+         action { [:IMPORT_SYM, st(text)] }
+
+      when (text = @ss.scan(/[\s]*@page[\s]*/i))
+         action { [:PAGE_SYM, st(text)] }
+
+      when (text = @ss.scan(/[\s]*@charset[\s]*/i))
+         action { [:CHARSET_SYM, st(text)] }
+
+      when (text = @ss.scan(/[\s]*@media[\s]*/i))
+         action { [:MEDIA_SYM, st(text)] }
+
+      when (text = @ss.scan(/[\s]*!([\s]*|[\s]*\/\*(.|[\s]*)*?\*\/[\s]*)important[\s]*/i))
+         action { [:IMPORTANT_SYM, st(text)] }
+
+      when (text = @ss.scan(/[-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/i))
+         action { [:IDENT, st(text)] }
+
+      when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/i))
+         action { [:HASH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*~=[\s]*/i))
+         action { [:INCLUDES, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\|=[\s]*/i))
+         action { [:DASHMATCH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\^=[\s]*/i))
+         action { [:PREFIXMATCH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\$=[\s]*/i))
+         action { [:SUFFIXMATCH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\*=[\s]*/i))
+         action { [:SUBSTRINGMATCH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*!=[\s]*/i))
+         action { [:NOT_EQUAL, st(text)] }
+
+      when (text = @ss.scan(/[\s]*=[\s]*/i))
+         action { [:EQUAL, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\)/i))
+         action { [:RPAREN, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\[[\s]*/i))
+         action { [:LSQUARE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\]/i))
+         action { [:RSQUARE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\+[\s]*/i))
+         action { [:PLUS, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\{[\s]*/i))
+         action { [:LBRACE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\}[\s]*/i))
+         action { [:RBRACE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*>[\s]*/i))
+         action { [:GREATER, st(text)] }
+
+      when (text = @ss.scan(/[\s]*,[\s]*/i))
+         action { [:COMMA, st(',')] }
+
+      when (text = @ss.scan(/[\s]*;[\s]*/i))
+         action { [:SEMI, st(';')] }
+
+      when (text = @ss.scan(/\*/i))
+         action { [:STAR, st(text)] }
+
+      when (text = @ss.scan(/[\s]*~[\s]*/i))
+         action { [:TILDE, st(text)] }
+
+      when (text = @ss.scan(/\:not\([\s]*/i))
+         action { [:NOT, st(text)]  }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)em[\s]*/i))
+         action { [:EMS, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)ex[\s]*/i))
+         action { [:EXS, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(px|cm|mm|in|pt|pc)[\s]*/i))
+         action { [:LENGTH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(deg|rad|grad)[\s]*/i))
+         action { [:ANGLE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(ms|s)[\s]*/i))
+         action { [:TIME, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[k]?hz[\s]*/i))
+         action { [:FREQ, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)%[\s]*/i))
+         action { [:PERCENTAGE, st(text)] }
+
+      when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[\s]*/i))
+         action { [:NUMBER, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\/\/[\s]*/i))
+         action { [:DOUBLESLASH, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\/[\s]*/i))
+         action { [:SLASH, st('/')] }
+
+      when (text = @ss.scan(/<!--/i))
+         action { [:CDO, st(text)] }
+
+      when (text = @ss.scan(/-->/i))
+         action { [:CDC, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\-(?![-@]?([_A-Za-z]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*)[\s]*/i))
+         action { [:MINUS, st(text)] }
+
+      when (text = @ss.scan(/[\s]*\+[\s]*/i))
+         action { [:PLUS, st(text)] }
+
+      when (text = @ss.scan(/[\s]+/i))
+         action { [:S, st(text)] }
+
+      when (text = @ss.scan(/("([^\n\r\f\\"]|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')/i))
+         action { [:STRING, st(text)] }
+
+      when (text = @ss.scan(/("([^\n\r\f\\"]|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*|'([^\n\r\f\\']|\\\n|\r\n|\r|\f|[^\0-\177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*)/i))
+         action { [:INVALID, st(text)] }
+
+      when (text = @ss.scan(/./i))
+         action { [st(text), st(text)] }
+
+      else
+        text = @ss.string[@ss.pos .. -1]
+        raise  ScanError, "can not match: '" + text + "'"
+      end  # if
+
+    else
+      raise  ScanError, "undefined state: '" + state.to_s + "'"
+    end  # case state
+    token
+  end  # def _next_token
+
+def st o
+  @st ||= Hash.new { |h,k| h[k] = k }
+  @st[o]
+end # class
+
+end
+end
+end
+
+# vim: syntax=lex

data/lib/csspool/css/tokenizer.rex

@@ -0,0 +1,96 @@
+module CSSPool
+module CSS
+class Tokenizer < Parser
+
+macro
+  nl        \n|\r\n|\r|\f
+  w         [\s]*
+  nonascii  [^\0-\177]
+  num       ([0-9]*\.[0-9]+|[0-9]+)
+  unicode   \\[0-9A-Fa-f]{1,6}(\r\n|[\s])?
+
+  escape    {unicode}|\\[^\n\r\f0-9A-Fa-f]
+  nmchar    [_A-Za-z0-9-]|{nonascii}|{escape}
+  nmstart   [_A-Za-z]|{nonascii}|{escape}
+  ident     [-@]?({nmstart})({nmchar})*
+  func      [-@]?({nmstart})({nmchar}|[.])*
+  name      ({nmchar})+
+  string1   "([^\n\r\f\\"]|\\{nl}|{nonascii}|{escape})*"
+  string2   '([^\n\r\f\\']|\\{nl}|{nonascii}|{escape})*'
+  string    ({string1}|{string2})
+  invalid1  "([^\n\r\f\\"]|\\{nl}|{nonascii}|{escape})*
+  invalid2  '([^\n\r\f\\']|\\{nl}|{nonascii}|{escape})*
+  invalid   ({invalid1}|{invalid2})
+  comment   \/\*(.|{w})*?\*\/
+
+rule
+
+# [:state]  pattern  [actions]
+
+            url\({w}{string}{w}\) { [:URI, st(text)] }
+            url\({w}([!#\$%&*-~]|{nonascii}|{escape})*{w}\) { [:URI, st(text)] }
+            U\+[0-9a-fA-F?]{1,6}(-[0-9a-fA-F]{1,6})?  {[:UNICODE_RANGE, st(text)] }
+            {w}{comment}{w}  { next_token }
+
+            {func}\(\s*      { [:FUNCTION, st(text)] }
+            {w}@import{w}    { [:IMPORT_SYM, st(text)] }
+            {w}@page{w}      { [:PAGE_SYM, st(text)] }
+            {w}@charset{w}   { [:CHARSET_SYM, st(text)] }
+            {w}@media{w}     { [:MEDIA_SYM, st(text)] }
+            {w}!({w}|{w}{comment}{w})important{w}  { [:IMPORTANT_SYM, st(text)] }
+            {ident}          { [:IDENT, st(text)] }
+            \#{name}         { [:HASH, st(text)] }
+            {w}~={w}         { [:INCLUDES, st(text)] }
+            {w}\|={w}        { [:DASHMATCH, st(text)] }
+            {w}\^={w}        { [:PREFIXMATCH, st(text)] }
+            {w}\$={w}        { [:SUFFIXMATCH, st(text)] }
+            {w}\*={w}        { [:SUBSTRINGMATCH, st(text)] }
+            {w}!={w}         { [:NOT_EQUAL, st(text)] }
+            {w}={w}          { [:EQUAL, st(text)] }
+            {w}\)            { [:RPAREN, st(text)] }
+            {w}\[{w}         { [:LSQUARE, st(text)] }
+            {w}\]            { [:RSQUARE, st(text)] }
+            {w}\+{w}         { [:PLUS, st(text)] }
+            {w}\{{w}         { [:LBRACE, st(text)] }
+            {w}\}{w}         { [:RBRACE, st(text)] }
+            {w}>{w}          { [:GREATER, st(text)] }
+            {w},{w}          { [:COMMA, st(',')] }
+            {w};{w}          { [:SEMI, st(';')] }
+            \*               { [:STAR, st(text)] }
+            {w}~{w}          { [:TILDE, st(text)] }
+            \:not\({w}       { [:NOT, st(text)]  }
+            {w}{num}em{w}    { [:EMS, st(text)] }
+            {w}{num}ex{w}    { [:EXS, st(text)] }
+
+            {w}{num}(px|cm|mm|in|pt|pc){w}    { [:LENGTH, st(text)] }
+            {w}{num}(deg|rad|grad){w} { [:ANGLE, st(text)] }
+            {w}{num}(ms|s){w} { [:TIME, st(text)] }
+            {w}{num}[k]?hz{w} { [:FREQ, st(text)] }
+
+            {w}{num}%{w}     { [:PERCENTAGE, st(text)] }
+            {w}{num}{w}      { [:NUMBER, st(text)] }
+            {w}\/\/{w}       { [:DOUBLESLASH, st(text)] }
+            {w}\/{w}         { [:SLASH, st('/')] }
+            <!--             { [:CDO, st(text)] }
+            -->              { [:CDC, st(text)] }
+            {w}\-(?!{ident}){w}   { [:MINUS, st(text)] }
+            {w}\+{w}         { [:PLUS, st(text)] }
+            
+            
+            [\s]+            { [:S, st(text)] }
+            {string}         { [:STRING, st(text)] }
+            {invalid}        { [:INVALID, st(text)] }
+            .                { [st(text), st(text)] }
+
+inner
+
+def st o
+  @st ||= Hash.new { |h,k| h[k] = k }
+  @st[o]
+end
+
+end
+end
+end
+
+# vim: syntax=lex

data/lib/csspool/css.rb

@@ -1,3 +1,4 @@
+require 'csspool/css/tokenizer'
 require 'csspool/css/charset'
 require 'csspool/css/import_rule'
 require 'csspool/css/media'