@tbela99/css-parser 0.0.1-alpha5 → 0.0.1-rc2

package/dist/lib/parser/parse.js

@@ -1,16 +1,23 @@
-import { isWhiteSpace, isDigit, isPseudo, isAtKeyword, isFunction, isNumber, isDimension, parseDimension, isPercentage, isIdent, isHash, isNewLine, isIdentStart, isHexColor } from './utils/syntax.js';
+import { isPseudo, isAtKeyword, isFunction, isNumber, isDimension, parseDimension, isPercentage, isIdent, isHexColor, isHash, isIdentStart } from './utils/syntax.js';
 import { renderToken } from '../renderer/render.js';
 import { COLORS_NAMES } from '../renderer/utils/color.js';
-import { deduplicate } from './deduplicate.js';
+import { minify, combinators } from '../ast/minify.js';
+import { tokenize } from './tokenize.js';
 
 const urlTokenMatcher = /^(["']?)[a-zA-Z0-9_/.-][a-zA-Z0-9_/:.#?-]+(\1)$/;
 const funcLike = ['Start-parens', 'Func', 'UrlFunc', 'Pseudo-class-func'];
+/**
+ *
+ * @param iterator
+ * @param opt
+ */
 async function parse(iterator, opt = {}) {
+    const startTime = performance.now();
     const errors = [];
     const options = {
         src: '',
         sourcemap: false,
-        compress: false,
+        minify: true,
         nestingRules: false,
         resolveImport: false,
         resolveUrls: false,
@@ -20,172 +27,28 @@ async function parse(iterator, opt = {}) {
     if (options.resolveImport) {
         options.resolveUrls = true;
     }
-    let ind = -1;
-    let lin = 1;
-    let col = 0;
-    const tokens = [];
     const src = options.src;
     const stack = [];
     const ast = {
         typ: "StyleSheet",
         chi: []
     };
-    const position = {
-        ind: Math.max(ind, 0),
-        lin: lin,
-        col: Math.max(col, 1)
-    };
-    let value;
-    let buffer = '';
-    let total = iterator.length;
-    let bytesIn = total;
+    let tokens = [];
     let map = new Map;
+    let bytesIn = 0;
     let context = ast;
     if (options.sourcemap) {
         ast.loc = {
             sta: {
-                ind: ind,
-                lin: lin,
-                col: col
+                ind: 0,
+                lin: 1,
+                col: 1
             },
             src: ''
         };
     }
-    function getType(val) {
-        if (val === '') {
-            throw new Error('empty string?');
-        }
-        if (val == ':') {
-            return { typ: 'Colon' };
-        }
-        if (val == ')') {
-            return { typ: 'End-parens' };
-        }
-        if (val == '(') {
-            return { typ: 'Start-parens' };
-        }
-        if (val == '=') {
-            return { typ: 'Delim', val };
-        }
-        if (val == ';') {
-            return { typ: 'Semi-colon' };
-        }
-        if (val == ',') {
-            return { typ: 'Comma' };
-        }
-        if (val == '<') {
-            return { typ: 'Lt' };
-        }
-        if (val == '>') {
-            return { typ: 'Gt' };
-        }
-        if (isPseudo(val)) {
-            return val.endsWith('(') ? {
-                typ: 'Pseudo-class-func',
-                val: val.slice(0, -1),
-                chi: []
-            }
-                : {
-                    typ: 'Pseudo-class',
-                    val
-                };
-        }
-        if (isAtKeyword(val)) {
-            return {
-                typ: 'At-rule',
-                val: val.slice(1)
-            };
-        }
-        if (isFunction(val)) {
-            val = val.slice(0, -1);
-            return {
-                typ: val == 'url' ? 'UrlFunc' : 'Func',
-                val,
-                chi: []
-            };
-        }
-        if (isNumber(val)) {
-            return {
-                typ: 'Number',
-                val
-            };
-        }
-        if (isDimension(val)) {
-            return parseDimension(val);
-        }
-        if (isPercentage(val)) {
-            return {
-                typ: 'Perc',
-                val: val.slice(0, -1)
-            };
-        }
-        if (val == 'currentColor') {
-            return {
-                typ: 'Color',
-                val,
-                kin: 'lit'
-            };
-        }
-        if (isIdent(val)) {
-            return {
-                typ: 'Iden',
-                val
-            };
-        }
-        if (val.charAt(0) == '#' && isHash(val)) {
-            return {
-                typ: 'Hash',
-                val
-            };
-        }
-        if ('"\''.includes(val.charAt(0))) {
-            return {
-                typ: 'Unclosed-string',
-                val
-            };
-        }
-        return {
-            typ: 'Literal',
-            val
-        };
-    }
-    // consume and throw away
-    function consume(open, close) {
-        let count = 1;
-        let chr;
-        while (true) {
-            chr = next();
-            if (chr == '\\') {
-                if (peek() === '') {
-                    break;
-                }
-                continue;
-            }
-            else if (chr == '/' && peek() == '*') {
-                next();
-                while (true) {
-                    chr = next();
-                    if (chr === '') {
-                        break;
-                    }
-                    if (chr == '*' && peek() == '/') {
-                        next();
-                        break;
-                    }
-                }
-            }
-            else if (chr == close) {
-                count--;
-            }
-            else if (chr == open) {
-                count++;
-            }
-            if (chr === '' || count == 0) {
-                break;
-            }
-        }
-    }
-    async function parseNode(tokens) {
+    async function parseNode(results) {
+        let tokens = results.map(mapToken);
         let i;
         let loc;
         for (i = 0; i < tokens.length; i++) {
@@ -283,12 +146,12 @@ async function parse(iterator, opt = {}) {
                             // @ts-ignore
                             const root = await options.load(url, options.src).then((src) => {
                                 return parse(src, Object.assign({}, options, {
-                                    compress: false,
+                                    minify: false,
                                     // @ts-ignore
                                     src: options.resolve(url, options.src).absolute
                                 }));
                             });
-                            bytesIn += root.bytesIn;
+                            bytesIn += root.stats.bytesIn;
                             if (root.ast.chi.length > 0) {
                                 context.chi.push(...root.ast.chi);
                             }
@@ -334,20 +197,17 @@ async function parse(iterator, opt = {}) {
             // rule
             if (delim.typ == 'Block-start') {
                 const position = map.get(tokens[0]);
-                if (context.typ == 'Rule') {
-                    if (tokens[0]?.typ == 'Iden') {
-                        errors.push({ action: 'drop', message: 'invalid nesting rule', location: { src, ...position } });
-                        return null;
-                    }
-                }
                 const uniq = new Map;
-                parseTokens(tokens, 'Rule', { compress: options.compress }).reduce((acc, curr, index, array) => {
+                parseTokens(tokens, { minify: options.minify }).reduce((acc, curr, index, array) => {
                     if (curr.typ == 'Whitespace') {
-                        if (array[index - 1]?.val == '+' || array[index + 1]?.val == '+') {
+                        if (array[index - 1]?.typ == 'Gt' ||
+                            array[index + 1]?.typ == 'Gt' ||
+                            combinators.includes(array[index - 1]?.val) ||
+                            combinators.includes(array[index + 1]?.val)) {
                             return acc;
                         }
                     }
-                    let t = renderToken(curr, { compress: true });
+                    let t = renderToken(curr, { minify: true });
                     if (t == ',') {
                         acc.push([]);
                     }
@@ -390,7 +250,7 @@ async function parse(iterator, opt = {}) {
                     }
                     if (tokens[i].typ == 'Colon') {
                         name = tokens.slice(0, i);
-                        value = parseTokens(tokens.slice(i + 1), 'Declaration', {
+                        value = parseTokens(tokens.slice(i + 1), {
                             parseColor: true,
                             src: options.src,
                             resolveUrls: options.resolveUrls,
@@ -416,11 +276,19 @@ async function parse(iterator, opt = {}) {
                     }
                 }
                 if (value == null) {
-                    errors.push({ action: 'drop', message: 'invalid declaration', location: { src, ...position } });
+                    errors.push({
+                        action: 'drop',
+                        message: 'invalid declaration',
+                        location: { src, ...position }
+                    });
                     return null;
                 }
                 if (value.length == 0) {
-                    errors.push({ action: 'drop', message: 'invalid declaration', location: { src, ...position } });
+                    errors.push({
+                        action: 'drop',
+                        message: 'invalid declaration',
+                        location: { src, ...position }
+                    });
                     return null;
                 }
                 const node = {
@@ -434,7 +302,11 @@ async function parse(iterator, opt = {}) {
                     node.val.shift();
                 }
                 if (node.val.length == 0) {
-                    errors.push({ action: 'drop', message: 'invalid declaration', location: { src, ...position } });
+                    errors.push({
+                        action: 'drop',
+                        message: 'invalid declaration',
+                        location: { src, ...position }
+                    });
                     return null;
                 }
                 // @ts-ignore
@@ -443,517 +315,220 @@ async function parse(iterator, opt = {}) {
             }
         }
     }
-    function peek(count = 1) {
-        if (count == 1) {
-            return iterator.charAt(ind + 1);
-        }
-        return iterator.slice(ind + 1, ind + count + 1);
-    }
-    function prev(count = 1) {
-        if (count == 1) {
-            return ind == 0 ? '' : iterator.charAt(ind - 1);
-        }
-        return iterator.slice(ind - 1 - count, ind - 1);
-    }
-    function next(count = 1) {
-        let char = '';
-        while (count-- > 0 && ind < total) {
-            const codepoint = iterator.charCodeAt(++ind);
-            if (isNaN(codepoint)) {
-                return char;
-            }
-            char += iterator.charAt(ind);
-            if (isNewLine(codepoint)) {
-                lin++;
-                col = 0;
-            }
-            else {
-                col++;
-            }
-        }
-        return char;
-    }
-    function pushToken(token) {
-        tokens.push(token);
-        map.set(token, { ...position });
-        position.ind = ind;
-        position.lin = lin;
-        position.col = col == 0 ? 1 : col;
+    function mapToken(token) {
+        const node = getTokenType(token.token, token.hint);
+        map.set(node, token.position);
+        return node;
     }
-    function consumeWhiteSpace() {
-        let count = 0;
-        while (isWhiteSpace(iterator.charAt(count + ind + 1).charCodeAt(0))) {
-            count++;
-        }
-        next(count);
-        return count;
-    }
-    function consumeString(quoteStr) {
-        const quote = quoteStr;
-        let value;
-        let hasNewLine = false;
-        if (buffer.length > 0) {
-            pushToken(getType(buffer));
-            buffer = '';
+    const iter = tokenize(iterator);
+    let item;
+    while (true) {
+        item = iter.next().value;
+        if (item == null) {
+            break;
         }
-        buffer += quoteStr;
-        while (ind < total) {
-            value = peek();
-            if (ind >= total) {
-                pushToken({ typ: hasNewLine ? 'Bad-string' : 'Unclosed-string', val: buffer });
-                break;
-            }
-            if (value == '\\') {
-                const sequence = peek(6);
-                let escapeSequence = '';
-                let codepoint;
-                let i;
-                for (i = 1; i < sequence.length; i++) {
-                    codepoint = sequence.charCodeAt(i);
-                    if (codepoint == 0x20 ||
-                        (codepoint >= 0x61 && codepoint <= 0x66) ||
-                        (codepoint >= 0x41 && codepoint <= 0x46) ||
-                        (codepoint >= 0x30 && codepoint <= 0x39)) {
-                        escapeSequence += sequence[i];
-                        if (codepoint == 0x20) {
-                            break;
-                        }
-                        continue;
-                    }
-                    break;
-                }
-                // not hex or new line
+        tokens.push(item);
+        bytesIn = item.bytesIn;
+        if (item.token == ';' || item.token == '{') {
+            let node = await parseNode(tokens);
+            if (node != null) {
+                stack.push(node);
                 // @ts-ignore
-                if (i == 1 && !isNewLine(codepoint)) {
-                    buffer += sequence[i];
-                    next(2);
-                    continue;
-                }
-                if (escapeSequence.trimEnd().length > 0) {
-                    const codepoint = Number(`0x${escapeSequence.trimEnd()}`);
-                    if (codepoint == 0 ||
-                        // leading surrogate
-                        (0xD800 <= codepoint && codepoint <= 0xDBFF) ||
-                        // trailing surrogate
-                        (0xDC00 <= codepoint && codepoint <= 0xDFFF)) {
-                        buffer += String.fromCodePoint(0xFFFD);
+                context = node;
+            }
+            else if (item.token == '{') {
+                // node == null
+                // consume and throw away until the closing '}' or EOF
+                let inBlock = 1;
+                do {
+                    item = iter.next().value;
+                    if (item == null) {
+                        break;
                     }
-                    else {
-                        buffer += String.fromCodePoint(codepoint);
+                    if (item.token == '{') {
+                        inBlock++;
                     }
-                    next(escapeSequence.length + 1);
-                    continue;
-                }
-                // buffer += value;
-                if (ind >= total) {
-                    // drop '\\' at the end
-                    pushToken(getType(buffer));
-                    break;
-                }
-                buffer += next(2);
-                continue;
-            }
-            if (value == quote) {
-                buffer += value;
-                pushToken({ typ: hasNewLine ? 'Bad-string' : 'String', val: buffer });
-                next();
-                // i += value.length;
-                buffer = '';
-                break;
-            }
-            if (isNewLine(value.charCodeAt(0))) {
-                hasNewLine = true;
-            }
-            if (hasNewLine && value == ';') {
-                pushToken({ typ: 'Bad-string', val: buffer });
-                buffer = '';
-                break;
-            }
-            buffer += value;
-            // i += value.length;
-            next();
-        }
-    }
-    while (ind < total) {
-        value = next();
-        if (ind >= total) {
-            if (buffer.length > 0) {
-                pushToken(getType(buffer));
-                buffer = '';
+                    else if (item.token == '}') {
+                        inBlock--;
+                    }
+                } while (inBlock != 0);
             }
-            break;
+            tokens = [];
+            map = new Map;
         }
-        if (isWhiteSpace(value.charCodeAt(0))) {
-            if (buffer.length > 0) {
-                pushToken(getType(buffer));
-                buffer = '';
-            }
-            while (ind < total) {
-                value = next();
-                if (ind >= total) {
-                    break;
-                }
-                if (!isWhiteSpace(value.charCodeAt(0))) {
-                    break;
-                }
-            }
-            pushToken({ typ: 'Whitespace' });
-            buffer = '';
-            if (ind >= total) {
-                break;
+        else if (item.token == '}') {
+            await parseNode(tokens);
+            const previousNode = stack.pop();
+            // @ts-ignore
+            context = stack[stack.length - 1] || ast;
+            // @ts-ignore
+            if (options.removeEmpty && previousNode != null && previousNode.chi.length == 0 && context.chi[context.chi.length - 1] == previousNode) {
+                context.chi.pop();
             }
+            tokens = [];
+            map = new Map;
         }
-        switch (value) {
-            case '/':
-                if (buffer.length > 0 && tokens.at(-1)?.typ == 'Whitespace') {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                    if (peek() != '*') {
-                        pushToken(getType(value));
-                        break;
-                    }
-                }
-                buffer += value;
-                if (peek() == '*') {
-                    buffer += '*';
-                    // i++;
-                    next();
-                    while (ind < total) {
-                        value = next();
-                        if (ind >= total) {
-                            pushToken({
-                                typ: 'Bad-comment', val: buffer
-                            });
-                            break;
-                        }
-                        if (value == '\\') {
-                            buffer += value;
-                            value = next();
-                            if (ind >= total) {
-                                pushToken({
-                                    typ: 'Bad-comment',
-                                    val: buffer
-                                });
-                                break;
-                            }
-                            buffer += value;
-                            continue;
-                        }
-                        if (value == '*') {
-                            buffer += value;
-                            value = next();
-                            if (ind >= total) {
-                                pushToken({
-                                    typ: 'Bad-comment', val: buffer
-                                });
-                                break;
-                            }
-                            buffer += value;
-                            if (value == '/') {
-                                pushToken({ typ: 'Comment', val: buffer });
-                                buffer = '';
-                                break;
-                            }
-                        }
-                        else {
-                            buffer += value;
-                        }
-                    }
-                }
-                break;
-            case '<':
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                buffer += value;
-                value = next();
-                if (ind >= total) {
-                    break;
-                }
-                if (peek(3) == '!--') {
-                    while (ind < total) {
-                        value = next();
-                        if (ind >= total) {
-                            break;
-                        }
-                        buffer += value;
-                        if (value == '>' && prev(2) == '--') {
-                            pushToken({
-                                typ: 'CDOCOMM',
-                                val: buffer
-                            });
-                            buffer = '';
-                            break;
-                        }
-                    }
-                }
-                if (ind >= total) {
-                    pushToken({ typ: 'BADCDO', val: buffer });
-                    buffer = '';
-                }
-                break;
-            case '\\':
-                value = next();
-                // EOF
-                if (ind + 1 >= total) {
-                    // end of stream ignore \\
-                    pushToken(getType(buffer));
-                    buffer = '';
-                    break;
-                }
-                buffer += value;
-                break;
-            case '"':
-            case "'":
-                consumeString(value);
-                break;
-            case '~':
-            case '|':
-                if (tokens.at(-1)?.typ == 'Whitespace') {
-                    tokens.pop();
-                }
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                buffer += value;
-                value = next();
-                if (ind >= total) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                    break;
-                }
-                if (value == '=') {
-                    buffer += value;
-                    pushToken({
-                        typ: buffer[0] == '~' ? 'Includes' : 'Dash-matches',
-                        val: buffer
-                    });
-                    buffer = '';
-                    break;
-                }
-                pushToken(getType(buffer));
-                while (isWhiteSpace(value.charCodeAt(0))) {
-                    value = next();
-                }
-                buffer = value;
-                break;
-            case '>':
-                if (buffer !== '') {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                if (tokens[tokens.length - 1]?.typ == 'Whitespace') {
-                    tokens.pop();
-                }
-                pushToken({ typ: 'Gt' });
-                consumeWhiteSpace();
-                break;
-            case '.':
-                const codepoint = peek().charCodeAt(0);
-                if (!isDigit(codepoint) && buffer !== '') {
-                    pushToken(getType(buffer));
-                    buffer = value;
-                    break;
-                }
-                buffer += value;
-                break;
-            case '+':
-            case ':':
-            case ',':
-            case '=':
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                if (value == ':' && ':' == peek()) {
-                    buffer += value + next();
-                    break;
-                }
-                pushToken(getType(value));
-                buffer = '';
-                if (value == '+' && isWhiteSpace(peek().charCodeAt(0))) {
-                    pushToken(getType(next()));
-                }
-                while (isWhiteSpace(peek().charCodeAt(0))) {
-                    next();
-                }
-                break;
-            case ')':
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                pushToken({ typ: 'End-parens' });
-                break;
-            case '(':
-                if (buffer.length == 0) {
-                    pushToken({ typ: 'Start-parens' });
-                }
-                else {
-                    buffer += value;
-                    pushToken(getType(buffer));
-                    buffer = '';
-                    const token = tokens[tokens.length - 1];
-                    if (token.typ == 'UrlFunc') {
-                        // consume either string or url token
-                        let whitespace = '';
-                        value = peek();
-                        while (isWhiteSpace(value.charCodeAt(0))) {
-                            whitespace += value;
-                        }
-                        if (whitespace.length > 0) {
-                            next(whitespace.length);
-                        }
-                        value = peek();
-                        if (value == '"' || value == "'") {
-                            consumeString(next());
-                            let token = tokens[tokens.length - 1];
-                            if (['String', 'Literal'].includes(token.typ) && urlTokenMatcher.test(token.val)) {
-                                if (token.val.slice(1, 6) != 'data:') {
-                                    if (token.typ == 'String') {
-                                        token.val = token.val.slice(1, -1);
-                                    }
-                                    // @ts-ignore
-                                    token.typ = 'Url-token';
-                                }
-                            }
-                            break;
-                        }
-                        else {
-                            buffer = '';
-                            do {
-                                let cp = value.charCodeAt(0);
-                                // EOF -
-                                if (cp == null) {
-                                    pushToken({ typ: 'Bad-url-token', val: buffer });
-                                    break;
-                                }
-                                // ')'
-                                if (cp == 0x29 || cp == null) {
-                                    if (buffer.length == 0) {
-                                        pushToken({ typ: 'Bad-url-token', val: '' });
-                                    }
-                                    else {
-                                        pushToken({ typ: 'Url-token', val: buffer });
-                                    }
-                                    if (cp != null) {
-                                        pushToken(getType(next()));
-                                    }
-                                    break;
-                                }
-                                if (isWhiteSpace(cp)) {
-                                    whitespace = next();
-                                    while (true) {
-                                        value = peek();
-                                        cp = value.charCodeAt(0);
-                                        if (isWhiteSpace(cp)) {
-                                            whitespace += value;
-                                            continue;
-                                        }
-                                        break;
-                                    }
-                                    if (cp == null || cp == 0x29) {
-                                        continue;
-                                    }
-                                    // bad url token
-                                    buffer += next(whitespace.length);
-                                    do {
-                                        value = peek();
-                                        cp = value.charCodeAt(0);
-                                        if (cp == null || cp == 0x29) {
-                                            break;
-                                        }
-                                        buffer += next();
-                                    } while (true);
-                                    pushToken({ typ: 'Bad-url-token', val: buffer });
-                                    continue;
-                                }
-                                buffer += next();
-                                value = peek();
-                            } while (true);
-                            buffer = '';
-                        }
-                    }
-                }
-                break;
-            case '[':
-            case ']':
-            case '{':
-            case '}':
-            case ';':
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                pushToken(getBlockType(value));
-                let node = null;
-                if (value == '{' || value == ';') {
-                    node = await parseNode(tokens);
-                    if (node != null) {
-                        stack.push(node);
-                        // @ts-ignore
-                        context = node;
-                    }
-                    else if (value == '{') {
-                        // node == null
-                        // consume and throw away until the closing '}' or EOF
-                        consume('{', '}');
-                    }
-                    tokens.length = 0;
-                    map.clear();
-                }
-                else if (value == '}') {
-                    await parseNode(tokens);
-                    const previousNode = stack.pop();
-                    // @ts-ignore
-                    context = stack[stack.length - 1] || ast;
-                    // @ts-ignore
-                    if (options.removeEmpty && previousNode != null && previousNode.chi.length == 0 && context.chi[context.chi.length - 1] == previousNode) {
-                        context.chi.pop();
-                    }
-                    tokens.length = 0;
-                    map.clear();
-                    buffer = '';
-                }
-                break;
-            case '!':
-                if (buffer.length > 0) {
-                    pushToken(getType(buffer));
-                    buffer = '';
-                }
-                const important = peek(9);
-                if (important == 'important') {
-                    if (tokens[tokens.length - 1]?.typ == 'Whitespace') {
-                        tokens.pop();
-                    }
-                    pushToken({ typ: 'Important' });
-                    next(9);
-                    buffer = '';
-                    break;
-                }
-                buffer = '!';
-                break;
-            default:
-                buffer += value;
-                break;
-        }
-    }
-    if (buffer.length > 0) {
-        pushToken(getType(buffer));
     }
     if (tokens.length > 0) {
         await parseNode(tokens);
     }
-    if (options.compress) {
+    const endParseTime = performance.now();
+    if (options.minify) {
         if (ast.chi.length > 0) {
-            deduplicate(ast, options, true);
+            minify(ast, options, true);
         }
     }
-    return { ast, errors, bytesIn };
+    const endTime = performance.now();
+    return {
+        ast, errors, stats: {
+            bytesIn,
+            parse: `${(endParseTime - startTime).toFixed(2)}ms`,
+            minify: `${(endTime - endParseTime).toFixed(2)}ms`,
+            total: `${(endTime - startTime).toFixed(2)}ms`
+        }
+    };
+}
+function parseString(src, options = { location: false }) {
+    return [...tokenize(src)].map(t => {
+        const token = getTokenType(t.token, t.hint);
+        if (options.location) {
+            Object.assign(token, { loc: t.position });
+        }
+        return token;
+    });
+}
+function getTokenType(val, hint) {
+    if (val === '' && hint == null) {
+        throw new Error('empty string?');
+    }
+    if (hint != null) {
+        return ([
+            'Whitespace', 'Semi-colon', 'Colon', 'Block-start',
+            'Block-start', 'Attr-start', 'Attr-end', 'Start-parens', 'End-parens',
+            'Comma', 'Gt', 'Lt'
+        ].includes(hint) ? { typ: hint } : { typ: hint, val });
+    }
+    if (val == ' ') {
+        return { typ: 'Whitespace' };
+    }
+    if (val == ';') {
+        return { typ: 'Semi-colon' };
+    }
+    if (val == '{') {
+        return { typ: 'Block-start' };
+    }
+    if (val == '}') {
+        return { typ: 'Block-end' };
+    }
+    if (val == '[') {
+        return { typ: 'Attr-start' };
+    }
+    if (val == ']') {
+        return { typ: 'Attr-end' };
+    }
+    if (val == ':') {
+        return { typ: 'Colon' };
+    }
+    if (val == ')') {
+        return { typ: 'End-parens' };
+    }
+    if (val == '(') {
+        return { typ: 'Start-parens' };
+    }
+    if (val == '=') {
+        return { typ: 'Delim', val };
+    }
+    if (val == ';') {
+        return { typ: 'Semi-colon' };
+    }
+    if (val == ',') {
+        return { typ: 'Comma' };
+    }
+    if (val == '<') {
+        return { typ: 'Lt' };
+    }
+    if (val == '>') {
+        return { typ: 'Gt' };
+    }
+    if (isPseudo(val)) {
+        return val.endsWith('(') ? {
+            typ: 'Pseudo-class-func',
+            val: val.slice(0, -1),
+            chi: []
+        }
+            : {
+                typ: 'Pseudo-class',
+                val
+            };
+    }
+    if (isAtKeyword(val)) {
+        return {
+            typ: 'At-rule',
+            val: val.slice(1)
+        };
+    }
+    if (isFunction(val)) {
+        val = val.slice(0, -1);
+        return {
+            typ: val == 'url' ? 'UrlFunc' : 'Func',
+            val,
+            chi: []
+        };
+    }
+    if (isNumber(val)) {
+        return {
+            typ: 'Number',
+            val
+        };
+    }
+    if (isDimension(val)) {
+        return parseDimension(val);
+    }
+    if (isPercentage(val)) {
+        return {
+            typ: 'Perc',
+            val: val.slice(0, -1)
+        };
+    }
+    const v = val.toLowerCase();
+    if (v == 'currentcolor' || val == 'transparent' || v in COLORS_NAMES) {
+        return {
+            typ: 'Color',
+            val,
+            kin: 'lit'
+        };
+    }
+    if (isIdent(val)) {
+        return {
+            typ: 'Iden',
+            val
+        };
+    }
+    if (val.charAt(0) == '#' && isHexColor(val)) {
+        return {
+            typ: 'Color',
+            val,
+            kin: 'hex'
+        };
+    }
+    if (val.charAt(0) == '#' && isHash(val)) {
+        return {
+            typ: 'Hash',
+            val
+        };
+    }
+    if ('"\''.includes(val.charAt(0))) {
+        return {
+            typ: 'Unclosed-string',
+            val
+        };
+    }
+    return {
+        typ: 'Literal',
+        val
+    };
 }
-function parseTokens(tokens, nodeType, options = {}) {
+function parseTokens(tokens, options = {}) {
     for (let i = 0; i < tokens.length; i++) {
         const t = tokens[i];
         if (t.typ == 'Whitespace' && ((i == 0 ||
@@ -1007,7 +582,7 @@ function parseTokens(tokens, nodeType, options = {}) {
                 if (t.chi.length > 1) {
                     /*(<AttrToken>t).chi =*/
                     // @ts-ignore
-                    parseTokens(t.chi, t.typ, options);
+                    parseTokens(t.chi, t.typ);
                 }
                 // @ts-ignore
                 t.chi.forEach(val => {
@@ -1119,8 +694,8 @@ function parseTokens(tokens, nodeType, options = {}) {
             // @ts-ignore
             if (t.chi.length > 0) {
                 // @ts-ignore
-                parseTokens(t.chi, t.typ, options);
-                if (t.typ == 'Pseudo-class-func' && t.val == ':is' && options.compress) {
+                parseTokens(t.chi, t.typ);
+                if (t.typ == 'Pseudo-class-func' && t.val == ':is' && options.minify) {
                     //
                     const count = t.chi.filter(t => t.typ != 'Comment').length;
                     if (count == 1 ||
@@ -1158,23 +733,5 @@ function parseTokens(tokens, nodeType, options = {}) {
     }
     return tokens;
 }
-function getBlockType(chr) {
-    if (chr == ';') {
-        return { typ: 'Semi-colon' };
-    }
-    if (chr == '{') {
-        return { typ: 'Block-start' };
-    }
-    if (chr == '}') {
-        return { typ: 'Block-end' };
-    }
-    if (chr == '[') {
-        return { typ: 'Attr-start' };
-    }
-    if (chr == ']') {
-        return { typ: 'Attr-end' };
-    }
-    throw new Error(`unhandled token: '${chr}'`);
-}
 
-export { parse };
+export { parse, parseString, urlTokenMatcher };