@tbela99/css-parser 1.4.2 → 1.4.3

package/dist/lib/parser/tokenize.js

@@ -1,81 +1,147 @@
-import { EnumToken } from '../ast/types.js';
-import '../ast/minify.js';
-import '../ast/walk.js';
-import './parse.js';
-import './utils/config.js';
-import { isWhiteSpace, isIdentStart, isIdent, isNewLine, isDigit } from '../syntax/syntax.js';
-import '../syntax/color/utils/constants.js';
-import '../renderer/sourcemap/lib/encode.js';
+import { EnumToken, ColorType } from '../ast/types.js';
+import { definedPropertySettings, wildCardFuncs, whenElseFunc, transformFunctions, mathFuncs, colorsFunc, timingFunc, supportFunc, generalEnclosedFunc, timelineFunc, imageFunc, gridTemplateFunc, urlFunc, containerFunc } from '../syntax/constants.js';
+import { isDigit, isPseudo, isIdent, isNumber, isWhiteSpace, isNewLine, isHexColor, isHash, isPercentage, parseDimension } from '../syntax/syntax.js';
 
+const SymbolsMapTokens = {
+    "+": EnumToken.Plus,
+    "=": EnumToken.DelimTokenType,
+    "|": EnumToken.Pipe,
+    "||": EnumToken.ColumnCombinatorTokenType,
+    "|=": EnumToken.DashMatchTokenType,
+    "&": EnumToken.NestingSelectorTokenType,
+    "*": EnumToken.Star,
+    "*=": EnumToken.ContainMatchTokenType,
+    "~": EnumToken.Tilda,
+    "~=": EnumToken.IncludeMatchTokenType,
+    "^=": EnumToken.StartMatchTokenType,
+    "$=": EnumToken.EndMatchTokenType,
+    ",": EnumToken.Comma,
+    ":": EnumToken.ColonTokenType,
+    "::": EnumToken.DoubleColonTokenType,
+    ";": EnumToken.SemiColonTokenType,
+    "(": EnumToken.StartParensTokenType,
+    ")": EnumToken.EndParensTokenType,
+    "[": EnumToken.AttrStartTokenType,
+    "]": EnumToken.AttrEndTokenType,
+    "{": EnumToken.BlockStartTokenType,
+    "}": EnumToken.BlockEndTokenType,
+    "<=": EnumToken.LteTokenType,
+    ">": EnumToken.GtTokenType,
+    ">=": EnumToken.GteTokenType,
+    " ": EnumToken.Whitespace,
+    "\t": EnumToken.Whitespace,
+    "\r": EnumToken.Whitespace,
+    "\n": EnumToken.Whitespace,
+    "\f": EnumToken.Whitespace,
+    ...containerFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.ContainerFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...urlFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.UrlFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...gridTemplateFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.GridTemplateFuncTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...imageFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.ImageFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...timelineFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.TimelineFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...generalEnclosedFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.GeneralEnclosedFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...supportFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.SupportsFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...timingFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.TimingFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...colorsFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.ColorFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...mathFuncs.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.MathFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...transformFunctions.reduce((acc, curr) => {
+        acc[curr.toLowerCase() + "("] = EnumToken.TransformFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...whenElseFunc.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.WhenElseFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+    ...wildCardFuncs.reduce((acc, curr) => {
+        acc[curr + "("] = EnumToken.WildCardFunctionTokenDefType;
+        return acc;
+    }, Object.create(null)),
+};
+// do not capture the value
+const hintsEnum = new Set([
+    EnumToken.CommaTokenType,
+    EnumToken.ImportantTokenType,
+    EnumToken.SemiColonTokenType,
+    EnumToken.BlockStartTokenType,
+    EnumToken.BlockEndTokenType,
+    EnumToken.StartParensTokenType,
+    EnumToken.EndParensTokenType,
+    EnumToken.ColonTokenType,
+    EnumToken.EOFTokenType,
+]);
 var TokenMap;
 (function (TokenMap) {
+    TokenMap[TokenMap["EXCLAMATION"] = 33] = "EXCLAMATION";
     TokenMap[TokenMap["SLASH"] = 47] = "SLASH";
-    TokenMap[TokenMap["AMPERSAND"] = 38] = "AMPERSAND";
     TokenMap[TokenMap["LOWERTHAN"] = 60] = "LOWERTHAN";
     TokenMap[TokenMap["HASH"] = 35] = "HASH";
     TokenMap[TokenMap["REVERSE_SOLIDUS"] = 92] = "REVERSE_SOLIDUS";
     TokenMap[TokenMap["DOUBLE_QUOTE"] = 34] = "DOUBLE_QUOTE";
     TokenMap[TokenMap["SINGLE_QUOTE"] = 39] = "SINGLE_QUOTE";
-    // ^
-    TokenMap[TokenMap["CIRCUMFLEX"] = 94] = "CIRCUMFLEX";
-    TokenMap[TokenMap["TILDA"] = 126] = "TILDA";
-    TokenMap[TokenMap["PIPE"] = 124] = "PIPE";
-    TokenMap[TokenMap["DOLLAR"] = 36] = "DOLLAR";
-    TokenMap[TokenMap["GREATER_THAN"] = 62] = "GREATER_THAN";
     TokenMap[TokenMap["DOT"] = 46] = "DOT";
-    TokenMap[TokenMap["PLUS"] = 43] = "PLUS";
+    TokenMap[TokenMap["AT"] = 64] = "AT";
+    TokenMap[TokenMap["PIPE"] = 124] = "PIPE";
+    TokenMap[TokenMap["EQUALS"] = 61] = "EQUALS";
+    TokenMap[TokenMap["AMPERSAND"] = 38] = "AMPERSAND";
     TokenMap[TokenMap["STAR"] = 42] = "STAR";
-    TokenMap[TokenMap["COLON"] = 58] = "COLON";
+    TokenMap[TokenMap["TILDA"] = 126] = "TILDA";
+    TokenMap[TokenMap["CARET"] = 94] = "CARET";
+    TokenMap[TokenMap["DOLLAR"] = 36] = "DOLLAR";
     TokenMap[TokenMap["COMMA"] = 44] = "COMMA";
-    TokenMap[TokenMap["EQUAL"] = 61] = "EQUAL";
-    TokenMap[TokenMap["CLOSE_PAREN"] = 41] = "CLOSE_PAREN";
-    TokenMap[TokenMap["OPEN_PAREN"] = 40] = "OPEN_PAREN";
-    TokenMap[TokenMap["OPEN_BRACKET"] = 91] = "OPEN_BRACKET";
-    TokenMap[TokenMap["CLOSE_BRACKET"] = 93] = "CLOSE_BRACKET";
-    TokenMap[TokenMap["OPEN_CURLY_BRACE"] = 123] = "OPEN_CURLY_BRACE";
-    TokenMap[TokenMap["CLOSE_CURLY_BRACE"] = 125] = "CLOSE_CURLY_BRACE";
+    TokenMap[TokenMap["COLON"] = 58] = "COLON";
     TokenMap[TokenMap["SEMICOLON"] = 59] = "SEMICOLON";
-    TokenMap[TokenMap["EXCLAMATION"] = 33] = "EXCLAMATION";
-    TokenMap[TokenMap["AT"] = 64] = "AT";
+    TokenMap[TokenMap["LEFT_PARENTHESIS"] = 40] = "LEFT_PARENTHESIS";
+    TokenMap[TokenMap["RIGHT_PARENTHESIS"] = 41] = "RIGHT_PARENTHESIS";
+    TokenMap[TokenMap["LEFT_BRACKETS"] = 91] = "LEFT_BRACKETS";
+    TokenMap[TokenMap["RIGHT_BRACKETS"] = 93] = "RIGHT_BRACKETS";
+    TokenMap[TokenMap["LEFT_BRACE"] = 123] = "LEFT_BRACE";
+    TokenMap[TokenMap["RIGHT_BRACE"] = 125] = "RIGHT_BRACE";
+    TokenMap[TokenMap["PLUS"] = 43] = "PLUS";
+    TokenMap[TokenMap["MINUS"] = 45] = "MINUS";
+    TokenMap[TokenMap["GREATERTHAN"] = 62] = "GREATERTHAN";
 })(TokenMap || (TokenMap = {}));
-function consumeWhiteSpace(parseInfo) {
-    let count = 0;
-    while (isWhiteSpace(parseInfo.stream.charCodeAt(count + parseInfo.currentPosition.ind + 1))) {
-        count++;
-    }
-    next(parseInfo, count);
-    return count;
-}
-function pushToken(token, parseInfo, hint) {
-    const result = {
-        token,
-        len: parseInfo.currentPosition.ind - parseInfo.position.ind - 1,
-        hint,
-        sta: { ...parseInfo.position },
-        end: { ...parseInfo.currentPosition },
-        bytesIn: parseInfo.currentPosition.ind + 1
-    };
-    parseInfo.position.ind = parseInfo.currentPosition.ind;
-    parseInfo.position.lin = parseInfo.currentPosition.lin;
-    parseInfo.position.col = Math.max(parseInfo.currentPosition.col, 1);
-    if (result.end.col == 0) {
-        result.end.col = 1;
-    }
-    return result;
-}
-function* consumeString(quoteStr, buffer, parseInfo) {
+function consumeString(quoteStr, buffer, parseInfo) {
     const quote = quoteStr;
     let value;
-    let hasNewLine = false;
+    const result = [];
     if (buffer.length > 0) {
-        yield pushToken(buffer, parseInfo);
-        buffer = '';
+        result.push(yieldResult(buffer, parseInfo));
+        buffer = "";
     }
     buffer += quoteStr;
-    while (value = peek(parseInfo)) {
-        if (value == '\\') {
+    while ((value = parseInfo.stream.charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1))) {
+        if (value == "\\") {
             const sequence = peek(parseInfo, 6);
-            let escapeSequence = '';
+            let escapeSequence = "";
             let codepoint;
             let i;
             for (i = 1; i < sequence.length; i++) {
@@ -96,15 +162,21 @@ function* consumeString(quoteStr, buffer, parseInfo) {
                 const codepoint = parseInt(escapeSequence, 16);
                 if (codepoint == 0 ||
                     // leading surrogate
-                    (0xD800 <= codepoint && codepoint <= 0xDBFF) ||
+                    (0xd800 <= codepoint && codepoint <= 0xdbff) ||
                     // trailing surrogate
-                    (0xDC00 <= codepoint && codepoint <= 0xDFFF)) {
-                    buffer += String.fromCodePoint(0xFFFD);
+                    (0xdc00 <= codepoint && codepoint <= 0xdfff)) {
+                    buffer += String.fromCodePoint(0xfffd);
                 }
                 else {
                     buffer += String.fromCodePoint(codepoint);
                 }
-                next(parseInfo, escapeSequence.length + 1 + (isWhiteSpace(peek(parseInfo)?.charCodeAt(0)) ? 1 : 0));
+                next(parseInfo, escapeSequence.length +
+                    1 +
+                    (isWhiteSpace(parseInfo.stream
+                        .charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1)
+                        ?.charCodeAt(0))
+                        ? 1
+                        : 0));
                 continue;
             }
             buffer += next(parseInfo, 2);
@@ -112,34 +184,125 @@ function* consumeString(quoteStr, buffer, parseInfo) {
         }
         if (value == quote) {
             buffer += value;
-            yield pushToken(buffer, parseInfo, hasNewLine ? EnumToken.BadStringTokenType : EnumToken.StringTokenType);
+            result.push(yieldResult(buffer, parseInfo, 
+            /* hasNewLine ? EnumToken.BadStringTokenType : */ EnumToken.StringTokenType));
             next(parseInfo);
-            buffer = '';
-            return;
+            buffer = "";
+            return result;
         }
         if (isNewLine(value.charCodeAt(0))) {
-            hasNewLine = true;
-        }
-        if (hasNewLine && value == ';') {
-            yield pushToken(buffer + value, parseInfo, EnumToken.BadStringTokenType);
-            buffer = '';
-            next(parseInfo);
-            break;
+            result.push(yieldResult(buffer + next(parseInfo), parseInfo, EnumToken.BadStringTokenType));
+            return result;
         }
         buffer += value;
         next(parseInfo);
     }
-    if (hasNewLine) {
-        yield pushToken(buffer, parseInfo, EnumToken.BadStringTokenType);
+    // EOF - 'Unclosed-string' fixed
+    result.push(yieldResult(buffer + quote, parseInfo, EnumToken.StringTokenType));
+    return result;
+}
+function getTokenType(val, hint) {
+    let token = null;
+    let dimension;
+    if (hint != null) {
+        token = hintsEnum.has(hint) ? { typ: hint } : { typ: hint, val };
     }
     else {
-        // EOF - 'Unclosed-string' fixed
-        yield pushToken(buffer + quote, parseInfo, EnumToken.StringTokenType);
+        let slice = val.slice(1);
+        if (val.charAt(0) == "@" && isIdent(slice)) {
+            token = {
+                typ: EnumToken.AtRuleTokenType,
+                nam: slice,
+            };
+        }
+        else if (val.charAt(0) == "." && isIdent(slice)) {
+            token = {
+                typ: EnumToken.ClassSelectorTokenType,
+                val,
+            };
+        }
+        else if (val.charAt(0) == "#") {
+            if (isHexColor(val)) {
+                token = {
+                    typ: EnumToken.ColorTokenType,
+                    val: val,
+                    kin: ColorType.HEX,
+                };
+            }
+            else if (isHash(val)) {
+                token = {
+                    typ: EnumToken.HashTokenType,
+                    val: val,
+                };
+            }
+        }
+        else if ("\"'".includes(val.charAt(0))) {
+            token = {
+                typ: EnumToken.UnclosedStringTokenType,
+                val: val,
+            };
+        }
+        else if (isNumber(val)) {
+            token =
+                val[0] === "-" || val[0] === "+"
+                    ? {
+                        typ: EnumToken.NumberTokenType,
+                        sign: val[0],
+                        val: +val,
+                    }
+                    : {
+                        typ: EnumToken.NumberTokenType,
+                        val: +val,
+                    };
+        }
+        else if (isPercentage(val)) {
+            token = {
+                typ: EnumToken.PercentageTokenType,
+                val: +val.slice(0, -1),
+            };
+        }
+        else if ((dimension = parseDimension(val))) {
+            token = dimension;
+        }
+        else if (isIdent(val)) {
+            token = {
+                typ: val.startsWith("--") ? EnumToken.DashedIdenTokenType : EnumToken.IdenTokenType,
+                val,
+            };
+        }
     }
+    if (token == null) {
+        token = {
+            typ: EnumToken.LiteralTokenType,
+            val,
+        };
+    }
+    return token;
+}
+function yieldResult(val, parseInfo, hint) {
+    const token = getTokenType(val, hint);
+    Object.defineProperty(token, "loc", {
+        ...definedPropertySettings,
+        enumerable: false,
+        value: {
+            src: parseInfo.src,
+            sta: { ...parseInfo.position },
+            end: { ...parseInfo.currentPosition },
+        },
+    });
+    parseInfo.position.ind = parseInfo.currentPosition.ind;
+    parseInfo.position.lin = parseInfo.currentPosition.lin;
+    parseInfo.position.col = parseInfo.currentPosition.col;
+    return { token, bytesIn: parseInfo.currentPosition.ind + 1 };
 }
 function match(parseInfo, input) {
-    const position = parseInfo.currentPosition.ind - parseInfo.offset;
-    return parseInfo.stream.slice(position + 1, position + input.length + 1) == input;
+    let position = parseInfo.currentPosition.ind - parseInfo.offset;
+    for (let i = 0; i < input.length; i++) {
+        if (parseInfo.stream[position + i + 1] != input.charAt(i)) {
+            return false;
+        }
+    }
+    return true;
 }
 function peek(parseInfo, count = 1) {
     if (count == 1) {
@@ -148,25 +311,26 @@ function peek(parseInfo, count = 1) {
     const position = parseInfo.currentPosition.ind - parseInfo.offset;
     return parseInfo.stream.slice(position + 1, position + count + 1);
 }
-function prev(parseInfo) {
-    return parseInfo.offset == parseInfo.currentPosition.ind ? parseInfo.buffer.slice(-1) : parseInfo.stream.charAt(parseInfo.currentPosition.ind - parseInfo.offset - 1);
-}
 function next(parseInfo, count = 1) {
-    let char = '';
-    let chr = '';
     let position = parseInfo.currentPosition.ind - parseInfo.offset;
-    while (count-- && (chr = parseInfo.stream.charAt(position + 1))) {
-        char += chr;
-        const codepoint = parseInfo.stream.charCodeAt(++position);
-        ++parseInfo.currentPosition.ind;
-        if (isNewLine(codepoint)) {
+    let char = count == 1 ? parseInfo.stream.charAt(position + 1) : parseInfo.stream.slice(position + 1, position + 1 + count);
+    let i = 0;
+    for (; i < char.length; i++) {
+        const codepoint = char[i].charCodeAt(0);
+        if (codepoint == 0xa ||
+            codepoint == 0xb ||
+            codepoint == 0xc ||
+            codepoint == 0xd ||
+            codepoint == 0x2028 ||
+            codepoint == 0x2029) {
             parseInfo.currentPosition.lin++;
-            parseInfo.currentPosition.col = 0;
+            parseInfo.currentPosition.col = 1;
         }
         else {
             parseInfo.currentPosition.col++;
         }
     }
+    parseInfo.currentPosition.ind += char.length;
     return char;
 }
 /**
@@ -174,382 +338,350 @@ function next(parseInfo, count = 1) {
  * @param parseInfo
  * @param yieldEOFToken
  */
-function* tokenize(parseInfo, yieldEOFToken = true) {
+function tokenize(parseInfo, yieldEOFToken = true) {
+    if (typeof parseInfo == "string") {
+        parseInfo = {
+            stream: parseInfo,
+            buffer: "",
+            src: "",
+            offset: 0,
+            time: 0,
+            position: { ind: 0, lin: 1, col: 0 },
+            currentPosition: { ind: -1, lin: 1, col: 0 },
+        };
+    }
     let value;
+    let nextValue;
     let buffer = parseInfo.buffer;
     let charCode;
-    parseInfo.buffer = '';
-    while (value = next(parseInfo)) {
+    let nextCharCode;
+    const startTime = performance.now();
+    const result = [];
+    parseInfo.buffer = "";
+    while ((value = next(parseInfo))) {
+        nextValue = parseInfo.stream.charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1);
         charCode = value.charCodeAt(0);
-        if (isWhiteSpace(charCode)) {
-            if (buffer.length > 0) {
-                yield pushToken(buffer, parseInfo);
-                buffer = '';
-            }
-            buffer += value;
-            while (value = next(parseInfo)) {
-                charCode = value.charCodeAt(0);
-                if (!isWhiteSpace(charCode)) {
-                    break;
-                }
-                buffer += value;
-            }
-            yield pushToken(buffer, parseInfo, EnumToken.WhitespaceTokenType);
-            buffer = '';
-        }
+        nextCharCode = nextValue.charCodeAt(0);
         switch (charCode) {
-            case 47 /* TokenMap.SLASH */:
+            case 61 /* TokenMap.EQUALS */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                    if (!match(parseInfo, '*')) {
-                        yield pushToken(value, parseInfo);
-                        break;
-                    }
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                buffer += value;
-                if (match(parseInfo, '*')) {
-                    buffer += next(parseInfo);
-                    while (value = next(parseInfo)) {
-                        if (value == '*') {
-                            buffer += value;
-                            if (match(parseInfo, '/')) {
-                                yield pushToken(buffer + next(parseInfo), parseInfo, EnumToken.CommentTokenType);
-                                buffer = '';
-                                break;
-                            }
-                        }
-                        else {
-                            buffer += value;
-                        }
-                    }
+                result.push(yieldResult(value, parseInfo, EnumToken.DelimTokenType));
+                break;
+            // '+' or '-'
+            case 43 /* TokenMap.PLUS */:
+            case 45 /* TokenMap.MINUS */:
+                if (charCode === 43 /* TokenMap.PLUS */ && !isNumber(nextValue)) {
                     if (buffer.length > 0) {
-                        yield pushToken(buffer, parseInfo, EnumToken.BadCommentTokenType);
-                        buffer = '';
+                        result.push(yieldResult(buffer, parseInfo));
+                        buffer = "";
                     }
+                    result.push(yieldResult(value, parseInfo, SymbolsMapTokens[value]));
+                    break;
                 }
+                buffer += value;
                 break;
-            case 38 /* TokenMap.AMPERSAND */:
+            // '{'
+            case 123 /* TokenMap.LEFT_BRACE */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                yield pushToken(value, parseInfo);
+                result.push(yieldResult(value, parseInfo, EnumToken.BlockStartTokenType));
                 break;
-            case 60 /* TokenMap.LOWERTHAN */:
+            // '}'
+            case 125 /* TokenMap.RIGHT_BRACE */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                if (match(parseInfo, '=')) {
-                    yield pushToken(value + next(parseInfo), parseInfo, EnumToken.LteTokenType);
-                    break;
-                }
-                buffer += value;
-                if (match(parseInfo, '!--')) {
-                    buffer += next(parseInfo, 3);
-                    while (value = next(parseInfo)) {
-                        buffer += value;
-                        if (value == '-' && match(parseInfo, '->')) {
-                            break;
-                        }
-                    }
-                    if (value === '') {
-                        yield pushToken(buffer, parseInfo, EnumToken.BadCdoTokenType);
+                result.push(yieldResult(value, parseInfo, EnumToken.BlockEndTokenType));
+                break;
+            // '('
+            case 40 /* TokenMap.LEFT_PARENTHESIS */:
+                if (buffer.length > 0) {
+                    if (buffer[0] === ":" && isPseudo(buffer)) {
+                        result.push(yieldResult(buffer, parseInfo, EnumToken.PseudoClassFunctionTokenDefType));
+                        buffer = "";
+                        break;
                     }
-                    else {
-                        yield pushToken(buffer + next(parseInfo, 2), parseInfo, EnumToken.CDOCOMMTokenType);
+                    else if (isIdent(buffer)) {
+                        result.push(yieldResult(buffer, parseInfo, buffer.startsWith("--")
+                            ? EnumToken.CustomFunctionTokenDefType
+                            : (SymbolsMapTokens[buffer.toLowerCase() + "("] ?? EnumToken.FunctionTokenDefType)));
+                        buffer = "";
+                        break;
                     }
-                    buffer = '';
                 }
+                result.push(yieldResult(value, parseInfo, EnumToken.StartParensTokenType));
+                buffer = "";
                 break;
-            case 35 /* TokenMap.HASH */:
+            // ')'
+            case 41 /* TokenMap.RIGHT_PARENTHESIS */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                buffer += value;
+                result.push(yieldResult(value, parseInfo, EnumToken.EndParensTokenType));
                 break;
-            case 92 /* TokenMap.REVERSE_SOLIDUS */:
-                // EOF
-                if (!(value = next(parseInfo))) {
-                    // end of stream ignore \\
-                    if (buffer.length > 0) {
-                        yield pushToken(buffer, parseInfo);
-                        buffer = '';
-                    }
-                    break;
+            // '['
+            case 91 /* TokenMap.LEFT_BRACKETS */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                buffer += prev(parseInfo) + value;
+                result.push(yieldResult(value, parseInfo, EnumToken.AttrStartTokenType));
                 break;
-            case 39 /* TokenMap.SINGLE_QUOTE */:
-            case 34 /* TokenMap.DOUBLE_QUOTE */:
-                yield* consumeString(value, buffer, parseInfo);
-                buffer = '';
+            // ']'
+            case 93 /* TokenMap.RIGHT_BRACKETS */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                result.push(yieldResult(value, parseInfo, EnumToken.AttrEndTokenType));
                 break;
-            case 94 /* TokenMap.CIRCUMFLEX */:
-            case 126 /* TokenMap.TILDA */:
-            case 124 /* TokenMap.PIPE */:
-            case 36 /* TokenMap.DOLLAR */:
+            case 59 /* TokenMap.SEMICOLON */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                if (charCode == 124 /* TokenMap.PIPE */) {
-                    if (match(parseInfo, '|')) {
-                        yield pushToken(value + next(parseInfo), parseInfo, EnumToken.ColumnCombinatorTokenType);
-                    }
-                    else if (match(parseInfo, '=')) {
-                        buffer += next(parseInfo);
-                        yield pushToken(buffer, parseInfo);
-                    }
-                    else {
-                        yield pushToken('|', parseInfo);
-                    }
-                    buffer = '';
-                    continue;
+                result.push(yieldResult(value, parseInfo, EnumToken.SemiColonTokenType));
+                break;
+            case 58 /* TokenMap.COLON */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
+                if (nextCharCode == 58 /* TokenMap.COLON */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.DoubleColonTokenType));
+                    break;
+                }
+                result.push(yieldResult(value, parseInfo, EnumToken.ColonTokenType));
+                break;
+            // \n \r \f \v \t space
+            case 0x9:
+            case 0x20:
+            case 0xa:
+            case 0xb:
+            case 0xc:
+            case 0xd:
+            case 0x2028:
+            case 0x2029:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
                 buffer += value;
-                if (!(value = peek(parseInfo))) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                    break;
+                nextCharCode = parseInfo.stream
+                    .charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1)
+                    .charCodeAt(0);
+                while (nextCharCode == 0x20 ||
+                    (nextCharCode >= 0x9 && nextCharCode <= 0xd) ||
+                    nextCharCode == 0x2028 ||
+                    nextCharCode == 0x2029) {
+                    value += next(parseInfo);
+                    nextCharCode = parseInfo.stream
+                        .charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1)
+                        .charCodeAt(0);
+                }
+                result.push(yieldResult(value, parseInfo, EnumToken.WhitespaceTokenType));
+                buffer = "";
+                break;
+            case 44 /* TokenMap.COMMA */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                // ~=
-                // ^=
-                // $=
-                // |=
-                if (match(parseInfo, '=')) {
-                    next(parseInfo);
-                    switch (buffer.charCodeAt(0)) {
-                        case 126 /* TokenMap.TILDA */:
-                            yield pushToken(buffer, parseInfo, EnumToken.IncludeMatchTokenType);
-                            break;
-                        case 94 /* TokenMap.CIRCUMFLEX */:
-                            yield pushToken(buffer, parseInfo, EnumToken.StartMatchTokenType);
-                            break;
-                        case 36 /* TokenMap.DOLLAR */:
-                            yield pushToken(buffer, parseInfo, EnumToken.EndMatchTokenType);
-                            break;
-                        case 124 /* TokenMap.PIPE */:
-                            yield pushToken(buffer, parseInfo, EnumToken.DashMatchTokenType);
-                            break;
-                    }
-                    buffer = '';
+                result.push(yieldResult(value, parseInfo, EnumToken.CommaTokenType));
+                break;
+            case 36 /* TokenMap.DOLLAR */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                if (nextCharCode == 61 /* TokenMap.EQUALS */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.EndMatchTokenType));
                     break;
                 }
-                yield pushToken(buffer, parseInfo);
-                buffer = '';
+                buffer += value;
                 break;
-            case 62 /* TokenMap.GREATER_THAN */:
-                if (buffer !== '') {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                }
-                if (match(parseInfo, '=')) {
-                    yield pushToken(value + next(parseInfo), parseInfo, EnumToken.GteTokenType);
+            case 126 /* TokenMap.TILDA */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                else {
-                    yield pushToken(value, parseInfo, EnumToken.GtTokenType);
+                if (nextCharCode == 61 /* TokenMap.EQUALS */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.IncludeMatchTokenType));
+                    break;
                 }
-                consumeWhiteSpace(parseInfo);
+                result.push(yieldResult(value, parseInfo, EnumToken.Tilda));
+                buffer = "";
                 break;
-            case 46 /* TokenMap.DOT */:
-                const codepoint = peek(parseInfo).charCodeAt(0);
-                if (!isDigit(codepoint) && buffer !== '') {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = value;
+            // case '^':
+            case 94 /* TokenMap.CARET */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                if (nextCharCode == 61 /* TokenMap.EQUALS */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.StartMatchTokenType));
                     break;
                 }
                 buffer += value;
                 break;
-            case 58 /* TokenMap.COLON */:
-            case 43 /* TokenMap.PLUS */:
             case 42 /* TokenMap.STAR */:
-            case 44 /* TokenMap.COMMA */:
-            case 61 /* TokenMap.EQUAL */:
-                if (buffer.length > 0 && buffer != ':') {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                }
-                const val = peek(parseInfo);
-                if (val == '=') {
-                    next(parseInfo);
-                    yield pushToken(value + val, parseInfo, EnumToken.ContainMatchTokenType);
-                    break;
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                if (value == ':') {
-                    if (isWhiteSpace(val.codePointAt(0))) {
-                        yield pushToken(value, parseInfo, EnumToken.ColonTokenType);
-                        buffer = '';
-                        break;
-                    }
-                    buffer += value;
+                if (nextCharCode == 61 /* TokenMap.EQUALS */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.ContainMatchTokenType));
                     break;
                 }
-                yield pushToken(value, parseInfo);
-                buffer = '';
-                if (['+', '*', '/'].includes(value) && isWhiteSpace(peek(parseInfo).charCodeAt(0))) {
-                    yield pushToken(next(parseInfo), parseInfo);
+                result.push(yieldResult(value, parseInfo, EnumToken.Star));
+                buffer = "";
+                break;
+            case 38 /* TokenMap.AMPERSAND */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                result.push(yieldResult(value, parseInfo, EnumToken.NestingSelectorTokenType));
+                buffer = "";
+                break;
+            case 124 /* TokenMap.PIPE */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                while (isWhiteSpace(peek(parseInfo).charCodeAt(0))) {
-                    next(parseInfo);
+                // '||'
+                if (nextCharCode == 124 /* TokenMap.PIPE */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.ColumnCombinatorTokenType));
+                    break;
                 }
+                else if (nextCharCode == 61 /* TokenMap.EQUALS */) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.DashMatchTokenType));
+                    break;
+                }
+                result.push(yieldResult(value, parseInfo, EnumToken.Pipe));
+                buffer = "";
                 break;
-            case 41 /* TokenMap.CLOSE_PAREN */:
+            case 33 /* TokenMap.EXCLAMATION */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                if (match(parseInfo, "important")) {
+                    result.push(yieldResult(value + next(parseInfo, 9), parseInfo, EnumToken.ImportantTokenType));
+                    buffer = "";
                 }
-                yield pushToken(value, parseInfo, EnumToken.EndParensTokenType);
                 break;
-            case 40 /* TokenMap.OPEN_PAREN */:
-                if (buffer.length == 0) {
-                    yield pushToken(value, parseInfo);
+            case 47 /* TokenMap.SLASH */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                if (!match(parseInfo, "*")) {
+                    result.push(yieldResult(value, parseInfo, SymbolsMapTokens[value]));
                     break;
                 }
-                buffer += value;
-                // @ts-ignore
-                if (buffer == 'url(') {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                    consumeWhiteSpace(parseInfo);
-                    value = peek(parseInfo);
-                    // let cp: number;
-                    let whitespace = '';
-                    if (value == '"' || value == "'") {
-                        const quote = value;
-                        let inquote = true;
-                        let hasNewLine = false;
-                        buffer = next(parseInfo);
-                        while (value = next(parseInfo)) {
-                            charCode = value.charCodeAt(0);
-                            // consume an invalid string
-                            if (inquote) {
-                                buffer += value;
-                                if (isNewLine(charCode)) {
-                                    hasNewLine = true;
-                                    while (value = next(parseInfo)) {
-                                        buffer += value;
-                                        if (value == ';') {
-                                            inquote = false;
-                                            break;
-                                        }
-                                    }
-                                    charCode = value.charCodeAt(0);
-                                }
-                                // '\\'
-                                if (charCode == 0x5c) {
-                                    buffer += next(parseInfo);
-                                }
-                                else if (value == quote) {
-                                    inquote = false;
-                                }
-                                continue;
-                            }
-                            if (!inquote) {
-                                if (isWhiteSpace(charCode)) {
-                                    whitespace += value;
-                                    while (value = peek(parseInfo)) {
-                                        if (isWhiteSpace(value?.charCodeAt(0))) {
-                                            whitespace += next(parseInfo);
-                                            continue;
-                                        }
-                                        break;
-                                    }
-                                    if (!(value = next(parseInfo))) {
-                                        yield pushToken(buffer, parseInfo, hasNewLine ? EnumToken.BadUrlTokenType : EnumToken.UrlTokenTokenType);
-                                        buffer = '';
-                                        break;
-                                    }
-                                }
-                                charCode = value.charCodeAt(0);
-                                // ')'
-                                if (charCode == 0x29) {
-                                    yield pushToken(buffer, parseInfo, hasNewLine ? EnumToken.BadStringTokenType : EnumToken.StringTokenType);
-                                    yield pushToken(value, parseInfo, EnumToken.EndParensTokenType);
-                                    buffer = '';
-                                    break;
-                                }
-                                while (value = next(parseInfo)) {
-                                    charCode = value.charCodeAt(0);
-                                    buffer += value;
-                                }
-                                if (hasNewLine) {
-                                    yield pushToken(buffer, parseInfo, EnumToken.BadStringTokenType);
-                                    buffer = '';
-                                }
-                                break;
-                            }
-                            buffer += value;
+                buffer += value + next(parseInfo);
+                while ((value = next(parseInfo))) {
+                    if (value == "*") {
+                        buffer += value;
+                        if (match(parseInfo, "/")) {
+                            result.push(yieldResult(buffer + next(parseInfo), parseInfo, EnumToken.CommentTokenType));
+                            buffer = "";
+                            break;
                         }
-                        break;
                     }
                     else {
-                        buffer = '';
-                        while (value = next(parseInfo)) {
-                            charCode = value.charCodeAt(0);
-                            if (charCode == 0x29) { // ')'
-                                yield pushToken(buffer, parseInfo, EnumToken.UrlTokenTokenType);
-                                yield pushToken(value, parseInfo, EnumToken.EndParensTokenType);
-                                buffer = '';
-                                break;
-                            }
-                            buffer += value;
-                        }
-                    }
-                    if (buffer !== '') {
-                        yield pushToken(buffer, parseInfo, EnumToken.UrlTokenTokenType);
-                        buffer = '';
-                        break;
+                        buffer += value;
                     }
-                    break;
                 }
-                yield pushToken(buffer, parseInfo);
-                buffer = '';
-                break;
-            case 91 /* TokenMap.OPEN_BRACKET */:
-            case 93 /* TokenMap.CLOSE_BRACKET */:
-            case 123 /* TokenMap.OPEN_CURLY_BRACE */:
-            case 125 /* TokenMap.CLOSE_CURLY_BRACE */:
-            case 59 /* TokenMap.SEMICOLON */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo, EnumToken.BadCommentTokenType));
+                    buffer = "";
                 }
-                yield pushToken(value, parseInfo);
                 break;
-            case 33 /* TokenMap.EXCLAMATION */:
+            case 62 /* TokenMap.GREATERTHAN */:
                 if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
-                if (match(parseInfo, 'important')) {
-                    yield pushToken(value + next(parseInfo, 9), parseInfo, EnumToken.ImportantTokenType);
-                    buffer = '';
+                if (match(parseInfo, "=")) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.GteTokenType));
                     break;
                 }
-                buffer = '!';
+                result.push(yieldResult(value, parseInfo, EnumToken.GtTokenType));
+                buffer = "";
                 break;
-            case 64 /* TokenMap.AT */:
-                buffer = value;
-                {
-                    let val = peek(parseInfo);
-                    if (val == '-' || isIdentStart(val.charCodeAt(0))) {
-                        buffer = next(parseInfo);
-                        val = peek(parseInfo);
-                        while (isIdent(val) || val == '-') {
-                            buffer += next(parseInfo);
-                            val = peek(parseInfo);
+            case 60 /* TokenMap.LOWERTHAN */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
+                }
+                if (match(parseInfo, "=")) {
+                    result.push(yieldResult(value + next(parseInfo), parseInfo, EnumToken.LteTokenType));
+                    break;
+                }
+                buffer += value;
+                if (match(parseInfo, "!--")) {
+                    buffer += next(parseInfo, 3);
+                    while ((value = next(parseInfo))) {
+                        buffer += value;
+                        if (value == "-" && match(parseInfo, "->")) {
+                            break;
                         }
-                        yield pushToken(buffer, parseInfo, EnumToken.AtRuleTokenType);
-                        buffer = '';
                     }
+                    if (value === "") {
+                        result.push(yieldResult(buffer, parseInfo, EnumToken.BadCdoTokenType));
+                    }
+                    else {
+                        result.push(yieldResult(buffer + next(parseInfo, 2), parseInfo, EnumToken.CDOCOMMTokenType));
+                    }
+                    buffer = "";
+                }
+                break;
+            case 35 /* TokenMap.HASH */:
+                if (buffer.length > 0) {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = "";
                 }
+                buffer += value;
+                break;
+            case 92 /* TokenMap.REVERSE_SOLIDUS */:
+                // EOF
+                if (!(value = next(parseInfo))) {
+                    // end of stream ignore \\
+                    if (buffer.length > 0) {
+                        result.push(yieldResult(buffer, parseInfo));
+                        buffer = "";
+                    }
+                    break;
+                }
+                buffer +=
+                    (parseInfo.offset == parseInfo.currentPosition.ind
+                        ? parseInfo.buffer.slice(-1)
+                        : parseInfo.stream.charAt(parseInfo.currentPosition.ind - parseInfo.offset - 1)) + value;
+                break;
+            case 39 /* TokenMap.SINGLE_QUOTE */:
+            case 34 /* TokenMap.DOUBLE_QUOTE */:
+                result.push(...consumeString(value, buffer, parseInfo));
+                buffer = "";
+                break;
+            case 46 /* TokenMap.DOT */:
+                const codepoint = parseInfo.stream
+                    .charAt(parseInfo.currentPosition.ind - parseInfo.offset + 1)
+                    .charCodeAt(0);
+                if (!isDigit(codepoint) && buffer !== "") {
+                    result.push(yieldResult(buffer, parseInfo));
+                    buffer = value;
+                    break;
+                }
+                buffer += value;
                 break;
             default:
                 buffer += value;
@@ -558,27 +690,31 @@ function* tokenize(parseInfo, yieldEOFToken = true) {
     }
     if (yieldEOFToken) {
         if (buffer.length > 0) {
-            yield pushToken(buffer, parseInfo);
+            result.push(yieldResult(buffer, parseInfo));
         }
-        yield pushToken('', parseInfo, EnumToken.EOFTokenType);
+        result.push(yieldResult("", parseInfo, EnumToken.EOFTokenType));
     }
     else {
         parseInfo.buffer = buffer;
     }
+    parseInfo.time += performance.now() - startTime;
+    return result;
 }
 /**
  * tokenize readable stream
  * @param input
  */
-async function* tokenizeStream(input) {
-    const parseInfo = {
-        stream: '',
-        buffer: '',
+async function* tokenizeStream(input, parseInfo) {
+    parseInfo ??= {
+        stream: "",
+        buffer: "",
+        src: "",
         offset: 0,
-        position: { ind: 0, lin: 1, col: 1 },
-        currentPosition: { ind: -1, lin: 1, col: 0 }
+        time: 0,
+        position: { ind: 0, lin: 1, col: 0 },
+        currentPosition: { ind: -1, lin: 1, col: 0 },
     };
-    const decoder = new TextDecoder('utf-8');
+    const decoder = new TextDecoder("utf-8");
     const reader = input.getReader();
     while (true) {
         const { done, value } = await reader.read();
@@ -599,5 +735,22 @@ async function* tokenizeStream(input) {
         }
     }
 }
+/**
+ * Update position
+ * @param position
+ * @param str
+ */
+function move(position, str) {
+    let i = 0;
+    for (; i < str.length; i++) {
+        if (isNewLine(str[i].charCodeAt(0))) {
+            position.lin++;
+            position.col = 0;
+        }
+        else {
+            position.col++;
+        }
+    }
+}
 
-export { tokenize, tokenizeStream };
+export { SymbolsMapTokens, TokenMap, consumeString, getTokenType, hintsEnum, match, move, next, peek, tokenize, tokenizeStream, yieldResult };