xaiml 0.1.0

data/vendor/bundle/ruby/2.5.0/gems/coderay-1.1.2/lib/coderay/scanners/clojure.rb

@@ -0,0 +1,217 @@
+# encoding: utf-8
+module CodeRay
+  module Scanners
+    
+    # Clojure scanner by Licenser.
+    class Clojure < Scanner
+      
+      register_for :clojure
+      file_extension 'clj'
+      
+      SPECIAL_FORMS = %w[
+        def if do let quote var fn loop recur throw try catch monitor-enter monitor-exit .
+        new 
+      ]  # :nodoc:
+      
+      CORE_FORMS = %w[
+        + - -> ->> .. / * <= < = == >= > accessor aclone add-classpath add-watch
+        agent agent-error agent-errors aget alength alias all-ns alter alter-meta!
+        alter-var-root amap ancestors and apply areduce array-map aset aset-boolean
+        aset-byte aset-char aset-double aset-float aset-int aset-long aset-short
+        assert assoc assoc! assoc-in associative? atom await await-for bases bean
+        bigdec bigint binding bit-and bit-and-not bit-clear bit-flip bit-not bit-or
+        bit-set bit-shift-left bit-shift-right bit-test bit-xor boolean boolean-array
+        booleans bound-fn bound-fn* bound? butlast byte byte-array bytes case cast char
+        char-array char-escape-string char-name-string char? chars class class?
+        clear-agent-errors clojure-version coll? comment commute comp comparator
+        compare compare-and-set! compile complement concat cond condp conj conj!
+        cons constantly construct-proxy contains? count counted? create-ns
+        create-struct cycle dec decimal? declare definline defmacro defmethod defmulti
+        defn defn- defonce defprotocol defrecord defstruct deftype delay delay?
+        deliver denominator deref derive descendants disj disj! dissoc dissoc!
+        distinct distinct? doall doc dorun doseq dosync dotimes doto double
+        double-array doubles drop drop-last drop-while empty empty? ensure
+        enumeration-seq error-handler error-mode eval even? every? extend
+        extend-protocol extend-type extenders extends? false? ffirst file-seq
+        filter find find-doc find-ns find-var first float float-array float?
+        floats flush fn fn? fnext for force format future future-call future-cancel
+        future-cancelled? future-done? future? gen-class gen-interface gensym get
+        get-in get-method get-proxy-class get-thread-bindings get-validator hash
+        hash-map hash-set identical? identity if-let if-not ifn? import in-ns
+        inc init-proxy instance? int int-array integer? interleave intern
+        interpose into into-array ints io! isa? iterate iterator-seq juxt key
+        keys keyword keyword? last lazy-cat lazy-seq let letfn line-seq list list*
+        list? load load-file load-reader load-string loaded-libs locking long
+        long-array longs loop macroexpand macroexpand-1 make-array make-hierarchy
+        map map? mapcat max max-key memfn memoize merge merge-with meta methods
+        min min-key mod name namespace neg? newline next nfirst nil? nnext not
+        not-any? not-empty not-every? not= ns ns-aliases ns-imports ns-interns
+        ns-map ns-name ns-publics ns-refers ns-resolve ns-unalias ns-unmap nth
+        nthnext num number? numerator object-array odd? or parents partial
+        partition pcalls peek persistent! pmap pop pop! pop-thread-bindings
+        pos? pr pr-str prefer-method prefers print print-namespace-doc
+        print-str printf println println-str prn prn-str promise proxy
+        proxy-mappings proxy-super push-thread-bindings pvalues quot rand
+        rand-int range ratio? rationalize re-find re-groups re-matcher
+        re-matches re-pattern re-seq read read-line read-string reduce ref
+        ref-history-count ref-max-history ref-min-history ref-set refer
+        refer-clojure reify release-pending-sends rem remove remove-all-methods
+        remove-method remove-ns remove-watch repeat repeatedly replace replicate
+        require reset! reset-meta! resolve rest restart-agent resultset-seq
+        reverse reversible? rseq rsubseq satisfies? second select-keys send
+        send-off seq seq? seque sequence sequential? set set-error-handler!
+        set-error-mode! set-validator! set? short short-array shorts
+        shutdown-agents slurp some sort sort-by sorted-map sorted-map-by
+        sorted-set sorted-set-by sorted? special-form-anchor special-symbol?
+        split-at split-with str string? struct struct-map subs subseq subvec
+        supers swap! symbol symbol? sync syntax-symbol-anchor take take-last
+        take-nth take-while test the-ns thread-bound? time to-array to-array-2d
+        trampoline transient tree-seq true? type unchecked-add unchecked-dec
+        unchecked-divide unchecked-inc unchecked-multiply unchecked-negate
+        unchecked-remainder unchecked-subtract underive update-in update-proxy
+        use val vals var-get var-set var? vary-meta vec vector vector-of vector?
+        when when-first when-let when-not while with-bindings with-bindings*
+        with-in-str with-local-vars with-meta with-open with-out-str
+        with-precision xml-seq zero? zipmap 
+      ]  # :nodoc:
+      
+      PREDEFINED_CONSTANTS = %w[
+        true false nil *1 *2 *3 *agent* *clojure-version* *command-line-args*
+        *compile-files* *compile-path* *e *err* *file* *flush-on-newline*
+        *in* *ns* *out* *print-dup* *print-length* *print-level* *print-meta*
+        *print-readably* *read-eval* *warn-on-reflection*
+      ]  # :nodoc:
+      
+      IDENT_KIND = WordList.new(:ident).
+        add(SPECIAL_FORMS, :keyword).
+        add(CORE_FORMS, :keyword).
+        add(PREDEFINED_CONSTANTS, :predefined_constant)
+      
+      KEYWORD_NEXT_TOKEN_KIND = WordList.new(nil).
+        add(%w[ def defn defn- definline defmacro defmulti defmethod defstruct defonce declare ], :function).
+        add(%w[ ns ], :namespace).
+        add(%w[ defprotocol defrecord ], :class)
+      
+      BASIC_IDENTIFIER = /[a-zA-Z$%*\/_+!?&<>\-=]=?[a-zA-Z0-9$&*+!\/_?<>\-\#]*/
+      IDENTIFIER = /(?!-\d)(?:(?:#{BASIC_IDENTIFIER}\.)*#{BASIC_IDENTIFIER}(?:\/#{BASIC_IDENTIFIER})?\.?)|\.\.?/
+      SYMBOL = /::?#{IDENTIFIER}/o
+      DIGIT = /\d/
+      DIGIT10 = DIGIT
+      DIGIT16 = /[0-9a-f]/i
+      DIGIT8 = /[0-7]/
+      DIGIT2 = /[01]/
+      RADIX16 = /\#x/i
+      RADIX8 = /\#o/i
+      RADIX2 = /\#b/i
+      RADIX10 = /\#d/i
+      EXACTNESS = /#i|#e/i
+      SIGN = /[\+-]?/
+      EXP_MARK = /[esfdl]/i
+      EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
+      SUFFIX = /#{EXP}?/
+      PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
+      PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
+      PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
+      PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
+      UINT10 = /#{DIGIT10}+#*/
+      UINT16 = /#{DIGIT16}+#*/
+      UINT8 = /#{DIGIT8}+#*/
+      UINT2 = /#{DIGIT2}+#*/
+      DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
+      UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
+      UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
+      UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
+      UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
+      REAL10 = /#{SIGN}#{UREAL10}/
+      REAL16 = /#{SIGN}#{UREAL16}/
+      REAL8 = /#{SIGN}#{UREAL8}/
+      REAL2 = /#{SIGN}#{UREAL2}/
+      IMAG10 = /i|#{UREAL10}i/
+      IMAG16 = /i|#{UREAL16}i/
+      IMAG8 = /i|#{UREAL8}i/
+      IMAG2 = /i|#{UREAL2}i/
+      COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
+      COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
+      COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
+      COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
+      NUM10 = /#{PREFIX10}?#{COMPLEX10}/
+      NUM16 = /#{PREFIX16}#{COMPLEX16}/
+      NUM8 = /#{PREFIX8}#{COMPLEX8}/
+      NUM2 = /#{PREFIX2}#{COMPLEX2}/
+      NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
+      
+    protected
+      
+      def scan_tokens encoder, options
+        
+        state = :initial
+        kind = nil
+        
+        until eos?
+          
+          case state
+          when :initial
+            if match = scan(/ \s+ | \\\n | , /x)
+              encoder.text_token match, :space
+            elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|[@\^]/)
+              encoder.text_token match, :operator
+            elsif match = scan(/;.*/)
+              encoder.text_token match, :comment  # TODO: recognize (comment ...) too
+            elsif match = scan(/\#?\\(?:newline|space|.?)/)
+              encoder.text_token match, :char
+            elsif match = scan(/\#[ft]/)
+              encoder.text_token match, :predefined_constant
+            elsif match = scan(/#{IDENTIFIER}/o)
+              kind = IDENT_KIND[match]
+              encoder.text_token match, kind
+              if rest? && kind == :keyword
+                if kind = KEYWORD_NEXT_TOKEN_KIND[match]
+                  encoder.text_token match, :space if match = scan(/\s+/o)
+                  encoder.text_token match, kind if match = scan(/#{IDENTIFIER}/o)
+                end
+              end
+            elsif match = scan(/#{SYMBOL}/o)
+              encoder.text_token match, :symbol
+            elsif match = scan(/\./)
+              encoder.text_token match, :operator
+            elsif match = scan(/ \# \^ #{IDENTIFIER} /ox)
+              encoder.text_token match, :type
+            elsif match = scan(/ (\#)? " /x)
+              state = self[1] ? :regexp : :string
+              encoder.begin_group state
+              encoder.text_token match, :delimiter
+            elsif match = scan(/#{NUM}/o) and not matched.empty?
+              encoder.text_token match, match[/[.e\/]/i] ? :float : :integer
+            else
+              encoder.text_token getch, :error
+            end
+            
+          when :string, :regexp
+            if match = scan(/[^"\\]+|\\.?/)
+              encoder.text_token match, :content
+            elsif match = scan(/"/)
+              encoder.text_token match, :delimiter
+              encoder.end_group state
+              state = :initial
+            else
+              raise_inspect "else case \" reached; %p not handled." % peek(1),
+                encoder, state
+            end
+            
+          else
+            raise 'else case reached'
+            
+          end
+          
+        end
+        
+        if [:string, :regexp].include? state
+          encoder.end_group state
+        end
+        
+        encoder
+        
+      end
+    end
+  end
+end

data/vendor/bundle/ruby/2.5.0/gems/coderay-1.1.2/lib/coderay/scanners/cpp.rb

@@ -0,0 +1,217 @@
+module CodeRay
+module Scanners
+
+  # Scanner for C++.
+  #
+  # Aliases: +cplusplus+, c++
+  class CPlusPlus < Scanner
+
+    register_for :cpp
+    file_extension 'cpp'
+    title 'C++'
+
+    #-- http://www.cppreference.com/wiki/keywords/start
+    KEYWORDS = [
+      'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
+      'case', 'catch', 'class', 'compl', 'const_cast',
+      'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
+      'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
+      'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
+      'sizeof', 'static_assert', 'static_cast', 'struct', 'switch',
+      'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
+      'while', 'xor', 'xor_eq',
+    ]  # :nodoc:
+
+    PREDEFINED_TYPES = [
+      'bool', 'char', 'char16_t', 'char32_t', 'double', 'float',
+      'int', 'long', 'short', 'signed', 'unsigned',
+      'wchar_t', 'string',
+    ]  # :nodoc:
+    PREDEFINED_CONSTANTS = [
+      'false', 'true',
+      'EOF', 'NULL', 'nullptr'
+    ]  # :nodoc:
+    PREDEFINED_VARIABLES = [
+      'this',
+    ]  # :nodoc:
+    DIRECTIVES = [
+      'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit',
+      'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator',
+      'override', 'private', 'protected', 'public', 'register', 'static',
+      'thread_local', 'using', 'virtual', 'void', 'volatile',
+    ]  # :nodoc:
+
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
+      add(PREDEFINED_VARIABLES, :local_variable).
+      add(DIRECTIVES, :directive).
+      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
+
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+
+  protected
+
+    def scan_tokens encoder, options
+
+      state = :initial
+      label_expected = true
+      case_expected = false
+      label_expected_before_preproc_line = nil
+      in_preproc_line = false
+
+      until eos?
+
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            if in_preproc_line && match != "\\\n" && match.index(?\n)
+              in_preproc_line = false
+              label_expected = label_expected_before_preproc_line
+            end
+            encoder.text_token match, :space
+
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            encoder.text_token match, :comment
+
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            encoder.text_token match, :comment
+
+          elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+            label_expected = match =~ /[;\{\}]/
+            if case_expected
+              label_expected = true if match == ':'
+              case_expected = false
+            end
+            encoder.text_token match, :operator
+
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = IDENT_KIND[match]
+            if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+              kind = :label
+              match << matched
+            else
+              label_expected = false
+              if kind == :keyword
+                case match
+                when 'class'
+                  state = :class_name_expected
+                when 'case', 'default'
+                  case_expected = true
+                end
+              end
+            end
+            encoder.text_token match, kind
+
+          elsif match = scan(/\$/)
+            encoder.text_token match, :ident
+
+          elsif match = scan(/L?"/)
+            encoder.begin_group :string
+            if match[0] == ?L
+              encoder.text_token match, 'L', :modifier
+              match = '"'
+            end
+            state = :string
+            encoder.text_token match, :delimiter
+
+          elsif match = scan(/#[ \t]*(\w*)/)
+            encoder.text_token match, :preprocessor
+            in_preproc_line = true
+            label_expected_before_preproc_line = label_expected
+            state = :include_expected if self[1] == 'include'
+
+          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+            label_expected = false
+            encoder.text_token match, :char
+
+          elsif match = scan(/0[xX][0-9A-Fa-f]+/)
+            label_expected = false
+            encoder.text_token match, :hex
+
+          elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
+            label_expected = false
+            encoder.text_token match, :octal
+
+          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            encoder.text_token match, :integer
+
+          elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+            label_expected = false
+            encoder.text_token match, :float
+
+          else
+            encoder.text_token getch, :error
+
+          end
+
+        when :string
+          if match = scan(/[^\\"]+/)
+            encoder.text_token match, :content
+          elsif match = scan(/"/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
+            state = :initial
+            label_expected = false
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group :string
+            encoder.text_token match, :error unless match.empty?
+            state = :initial
+            label_expected = false
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
+          end
+
+        when :include_expected
+          if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            encoder.text_token match, :include
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            encoder.text_token match, :space
+            state = :initial if match.index ?\n
+
+          else
+            state = :initial
+
+          end
+
+        when :class_name_expected
+          if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            encoder.text_token match, :class
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            encoder.text_token match, :space
+
+          else
+            encoder.text_token getch, :error
+            state = :initial
+
+          end
+
+        else
+          raise_inspect 'Unknown state', encoder
+
+        end
+
+      end
+
+      if state == :string
+        encoder.end_group :string
+      end
+
+      encoder
+    end
+
+  end
+
+end
+end

data/vendor/bundle/ruby/2.5.0/gems/coderay-1.1.2/lib/coderay/scanners/css.rb

@@ -0,0 +1,196 @@
+module CodeRay
+module Scanners
+
+  class CSS < Scanner
+    
+    register_for :css
+    
+    KINDS_NOT_LOC = [
+      :comment,
+      :class, :pseudo_class, :tag,
+      :id, :directive,
+      :key, :value, :operator, :color, :float, :string,
+      :error, :important, :type,
+    ]  # :nodoc:
+    
+    module RE  # :nodoc:
+      Hex = /[0-9a-fA-F]/
+      Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]/
+      NMStart = /[_a-zA-Z]/
+      String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/  # TODO: buggy regexp
+      String = /#{String1}|#{String2}/
+      
+      HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
+      
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/
+      Name = /#{NMChar}+/
+      Ident = /-?#{NMStart}#{NMChar}*/
+      AtKeyword = /@#{Ident}/
+      Percentage = /#{Num}%/
+      
+      reldimensions = %w[em ex px]
+      absdimensions = %w[in cm mm pt pc]
+      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg]))
+      
+      Dimension = /#{Num}#{Unit}/
+      
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/
+      
+      Id = /(?!#{HexColor}\b(?!-))##{Name}/
+      Class = /\.#{Name}/
+      PseudoClass = /::?#{Ident}/
+      AttributeSelector = /\[[^\]]*\]?/
+    end
+    
+  protected
+    
+    def setup
+      @state = :initial
+      @value_expected = false
+    end
+    
+    def scan_tokens encoder, options
+      states = Array(options[:state] || @state).dup
+      value_expected = @value_expected
+      
+      until eos?
+        
+        if match = scan(/\s+/)
+          encoder.text_token match, :space
+          
+        elsif case states.last
+          when :initial, :media
+            if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
+              encoder.text_token match, :tag
+              next
+            elsif match = scan(RE::Class)
+              encoder.text_token match, :class
+              next
+            elsif match = scan(RE::Id)
+              encoder.text_token match, :id
+              next
+            elsif match = scan(RE::PseudoClass)
+              encoder.text_token match, :pseudo_class
+              next
+            elsif match = scan(RE::AttributeSelector)
+              # TODO: Improve highlighting inside of attribute selectors.
+              encoder.text_token match[0,1], :operator
+              encoder.text_token match[1..-2], :attribute_name if match.size > 2
+              encoder.text_token match[-1,1], :operator if match[-1] == ?]
+              next
+            elsif match = scan(/@media/)
+              encoder.text_token match, :directive
+              states.push :media_before_name
+              next
+            end
+          
+          when :block
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
+              if value_expected
+                encoder.text_token match, :value
+              else
+                encoder.text_token match, :key
+              end
+              next
+            end
+            
+          when :media_before_name
+            if match = scan(RE::Ident)
+              encoder.text_token match, :type
+              states[-1] = :media_after_name
+              next
+            end
+          
+          when :media_after_name
+            if match = scan(/\{/)
+              encoder.text_token match, :operator
+              states[-1] = :media
+              next
+            end
+          
+          else
+            #:nocov:
+            raise_inspect 'Unknown state', encoder
+            #:nocov:
+            
+          end
+          
+        elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
+          encoder.text_token match, :comment
+          
+        elsif match = scan(/\{/)
+          value_expected = false
+          encoder.text_token match, :operator
+          states.push :block
+          
+        elsif match = scan(/\}/)
+          value_expected = false
+          encoder.text_token match, :operator
+          if states.last == :block || states.last == :media
+            states.pop
+          end
+          
+        elsif match = scan(/#{RE::String}/o)
+          encoder.begin_group :string
+          encoder.text_token match[0, 1], :delimiter
+          encoder.text_token match[1..-2], :content if match.size > 2
+          encoder.text_token match[-1, 1], :delimiter if match.size >= 2
+          encoder.end_group :string
+          
+        elsif match = scan(/#{RE::Function}/o)
+          encoder.begin_group :function
+          start = match[/^\w+\(/]
+          encoder.text_token start, :delimiter
+          if match[-1] == ?)
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
+            encoder.text_token ')', :delimiter
+          else
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
+          end
+          encoder.end_group :function
+          
+        elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          encoder.text_token match, :float
+          
+        elsif match = scan(/#{RE::HexColor}/o)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/! *important/)
+          encoder.text_token match, :important
+          
+        elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
+          encoder.text_token match, :color
+          
+        elsif match = scan(RE::AtKeyword)
+          encoder.text_token match, :directive
+          
+        elsif match = scan(/ [+>~:;,.=()\/] /x)
+          if match == ':'
+            value_expected = true
+          elsif match == ';'
+            value_expected = false
+          end
+          encoder.text_token match, :operator
+          
+        else
+          encoder.text_token getch, :error
+          
+        end
+        
+      end
+      
+      if options[:keep_state]
+        @state = states
+        @value_expected = value_expected
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end