@tbela99/css-parser 1.2.0 → 1.3.1

package/dist/lib/parser/declaration/map.js

@@ -7,6 +7,7 @@ import '../../ast/walk.js';
 import { parseString } from '../parse.js';
 import '../tokenize.js';
 import { renderToken } from '../../renderer/render.js';
+import '../../renderer/sourcemap/lib/encode.js';
 import '../../syntax/color/utils/constants.js';
 import { PropertySet } from './set.js';
 

package/dist/lib/parser/declaration/set.js

@@ -15,7 +15,7 @@ function dedup(values) {
         while (i-- > 1) {
             const t = value[i];
             const k = value[i == 1 ? 0 : i % 2];
-            if (t.val == k.val && t.val == '0') {
+            if (t.val == k.val && t.val == 0) {
                 if ((t.typ == EnumToken.NumberTokenType && isLength(k)) ||
                     (k.typ == EnumToken.NumberTokenType && isLength(t)) ||
                     (isLength(k) || isLength(t))) {
@@ -52,7 +52,7 @@ class PropertySet {
                 // @ts-ignore
                 for (let token of this.declarations.get(this.config.shorthand).val) {
                     // @ts-ignore
-                    if (this.config.types.some(t => token.typ == EnumToken[t]) || (token.typ == EnumToken.NumberTokenType && token.val == '0' &&
+                    if (this.config.types.some(t => token.typ == EnumToken[t]) || (token.typ == EnumToken.NumberTokenType && token.val == 0 &&
                         (this.config.types.includes('Length') ||
                             this.config.types.includes('Angle') ||
                             this.config.types.includes('Dimension')))) {

package/dist/lib/parser/parse.js

@@ -6,9 +6,10 @@ import { expand } from '../ast/expand.js';
 import './utils/config.js';
 import { parseDeclarationNode } from './utils/declaration.js';
 import { renderToken } from '../renderer/render.js';
+import '../renderer/sourcemap/lib/encode.js';
 import { funcLike, timingFunc, timelineFunc, COLORS_NAMES, systemColors, deprecatedSystemColors, colorsFunc } from '../syntax/color/utils/constants.js';
 import { buildExpression } from '../ast/math/expression.js';
-import { tokenize } from './tokenize.js';
+import { tokenize, tokenizeStream } from './tokenize.js';
 import '../validation/config.js';
 import '../validation/parser/types.js';
 import '../validation/parser/parse.js';
@@ -40,10 +41,12 @@ function reject(reason) {
 }
 /**
  * parse css string
- * @param iterator
+ * @param iter
  * @param options
+ *
+ * @private
  */
-async function doParse(iterator, options = {}) {
+async function doParse(iter, options = {}) {
     if (options.signal != null) {
         options.signal.addEventListener('abort', reject);
     }
@@ -113,15 +116,28 @@ async function doParse(iterator, options = {}) {
             src: ''
         };
     }
-    const iter = tokenize(iterator);
     let item;
     let node;
     const rawTokens = [];
     const imports = [];
-    while (item = iter.next().value) {
+    // @ts-ignore ignore error
+    const isAsync = typeof iter[Symbol.asyncIterator] === 'function';
+    while (item = isAsync ? (await iter.next()).value : iter.next().value) {
         stats.bytesIn = item.bytesIn;
         rawTokens.push(item);
         if (item.hint != null && BadTokensTypes.includes(item.hint)) {
+            const node = getTokenType(item.token, item.hint);
+            errors.push({
+                action: 'drop',
+                message: 'Bad token',
+                syntax: null,
+                node,
+                location: {
+                    src,
+                    sta: item.sta,
+                    end: item.end
+                }
+            });
             // bad token
             continue;
         }
@@ -150,7 +166,7 @@ async function doParse(iterator, options = {}) {
                 let inBlock = 1;
                 tokens = [item];
                 do {
-                    item = iter.next().value;
+                    item = isAsync ? (await iter.next()).value : iter.next().value;
                     if (item == null) {
                         break;
                     }
@@ -166,7 +182,12 @@ async function doParse(iterator, options = {}) {
                     errors.push({
                         action: 'drop',
                         message: 'invalid block',
-                        rawTokens: tokens.slice()
+                        rawTokens: tokens.slice(),
+                        location: {
+                            src,
+                            sta: tokens[0].sta,
+                            end: tokens[tokens.length - 1].end
+                        }
                     });
                 }
             }
@@ -207,8 +228,8 @@ async function doParse(iterator, options = {}) {
             }
         }
         if (context != null && context.typ == EnumToken.InvalidRuleTokenType) {
-            // @ts-ignore
-            const index = context.chi.findIndex((node) => node == context);
+            // @ts-ignore ignore error
+            const index = context.chi.findIndex((node) => node === context);
             if (index > -1) {
                 context.chi.splice(index, 1);
             }
@@ -219,13 +240,21 @@ async function doParse(iterator, options = {}) {
             const token = node.tokens[0];
             const url = token.typ == EnumToken.StringTokenType ? token.val.slice(1, -1) : token.val;
             try {
-                const root = await options.load(url, options.src).then((src) => {
-                    return doParse(src, Object.assign({}, options, {
+                const root = await options.getStream(url, options.src).then(async (stream) => {
+                    return doParse(tokenizeStream(stream), Object.assign({}, options, {
                         minify: false,
                         setParent: false,
                         src: options.resolve(url, options.src).absolute
-                    }));
+                    })); // )
                 });
+                // const root: ParseResult = await options.load!(url, <string>options.src).then((src: string) => {
+                //
+                //     return doParse(src, Object.assign({}, options, {
+                //         minify: false,
+                //         setParent: false,
+                //         src: options.resolve!(url, options.src as string).absolute
+                //     }))
+                // });
                 stats.importedBytesIn += root.stats.bytesIn;
                 stats.imports.push(root.stats);
                 node.parent.chi.splice(node.parent.chi.indexOf(node), 1, ...root.ast.chi);
@@ -234,7 +263,7 @@ async function doParse(iterator, options = {}) {
                 }
             }
             catch (error) {
-                // @ts-ignore
+                // @ts-ignore ignore error
                 errors.push({ action: 'ignore', message: 'doParse: ' + error.message, error });
             }
         }));
@@ -258,28 +287,33 @@ async function doParse(iterator, options = {}) {
             if (result.node.typ == EnumToken.DeclarationNodeType &&
                 (typeof options.visitor.Declaration == 'function' || options.visitor.Declaration?.[result.node.nam] != null)) {
                 const callable = typeof options.visitor.Declaration == 'function' ? options.visitor.Declaration : options.visitor.Declaration[result.node.nam];
-                const results = await callable(result.node);
+                const isAsync = Object.getPrototypeOf(callable).constructor.name == 'AsyncFunction';
+                const results = isAsync ? await callable(result.node) : callable(result.node);
                 if (results == null || (Array.isArray(results) && results.length == 0)) {
                     continue;
                 }
+                // @ts-ignore
                 result.parent.chi.splice(result.parent.chi.indexOf(result.node), 1, ...(Array.isArray(results) ? results : [results]));
             }
             else if (options.visitor.Rule != null && result.node.typ == EnumToken.RuleNodeType) {
-                const results = await options.visitor.Rule(result.node);
+                const isAsync = Object.getPrototypeOf(options.visitor.Rule).constructor.name == 'AsyncFunction';
+                const results = isAsync ? await options.visitor.Rule(result.node) : options.visitor.Rule(result.node);
                 if (results == null || (Array.isArray(results) && results.length == 0)) {
                     continue;
                 }
+                // @ts-ignore
                 result.parent.chi.splice(result.parent.chi.indexOf(result.node), 1, ...(Array.isArray(results) ? results : [results]));
             }
             else if (options.visitor.AtRule != null &&
                 result.node.typ == EnumToken.AtRuleNodeType &&
-                // @ts-ignore
                 (typeof options.visitor.AtRule == 'function' || options.visitor.AtRule?.[result.node.nam] != null)) {
                 const callable = typeof options.visitor.AtRule == 'function' ? options.visitor.AtRule : options.visitor.AtRule[result.node.nam];
-                const results = await callable(result.node);
+                const isAsync = Object.getPrototypeOf(callable).constructor.name == 'AsyncFunction';
+                const results = isAsync ? await callable(result.node) : callable(result.node);
                 if (results == null || (Array.isArray(results) && results.length == 0)) {
                     continue;
                 }
+                // @ts-ignore
                 result.parent.chi.splice(result.parent.chi.indexOf(result.node), 1, ...(Array.isArray(results) ? results : [results]));
             }
         }
@@ -334,6 +368,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                 errors.push({
                     action: 'drop',
                     message: `CDOCOMM not allowed here ${JSON.stringify(tokens[i], null, 1)}`,
+                    node: tokens[i],
                     location
                 });
                 continue;
@@ -388,7 +423,14 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                         if (!(type == EnumToken.InvalidAtRuleTokenType &&
                             // @ts-ignore
                             ['charset', 'layer', 'import'].includes(context.chi[i].nam))) {
-                            errors.push({ action: 'drop', message: 'invalid @import', location });
+                            // @ts-ignore
+                            errors.push({
+                                action: 'drop',
+                                message: 'invalid @import',
+                                location,
+                                // @ts-ignore
+                                rawTokens: [atRule, ...tokens]
+                            });
                             return null;
                         }
                     }
@@ -464,7 +506,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                         action: 'drop',
                         message: '@charset must have only one space',
                         // @ts-ignore
-                        location
+                        location, rawTokens: [atRule, ...tokens]
                     });
                     return null;
                 }
@@ -533,6 +575,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
             errors.push({
                 action: 'drop',
                 message: valid.error + ' - "' + tokens.reduce((acc, curr) => acc + renderToken(curr, { minify: false }), '') + '"',
+                node,
                 // @ts-ignore
                 location: { src, ...(map.get(valid.node) ?? location) }
             });
@@ -581,7 +624,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                             if (curr.typ == EnumToken.IdenTokenType && curr.val == 'from') {
                                 Object.assign(curr, { typ: EnumToken.PercentageTokenType, val: '0' });
                             }
-                            else if (curr.typ == EnumToken.PercentageTokenType && curr.val == '100') {
+                            else if (curr.typ == EnumToken.PercentageTokenType && curr.val == 100) {
                                 Object.assign(curr, { typ: EnumToken.IdenTokenType, val: 'to' });
                             }
                         }
@@ -634,6 +677,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                 errors.push({
                     action: 'drop',
                     message: valid.error + ' - "' + tokens.reduce((acc, curr) => acc + renderToken(curr, { minify: false }), '') + '"',
+                    node,
                     // @ts-ignore
                     location
                 });
@@ -658,7 +702,7 @@ function parseNode(results, context, options, errors, src, map, rawTokens) {
                             val: tokens[i].val.charAt(0)
                         }, {
                             typ: EnumToken.NumberTokenType,
-                            val: tokens[i].val.slice(1)
+                            val: +tokens[i].val.slice(1)
                         });
                     }
                     else if (start == '/' && isFunction(val)) {
@@ -859,9 +903,10 @@ function parseAtRulePrelude(tokens, atRule) {
                 }
             }
         }
+        const val = value.typ == EnumToken.IdenTokenType ? value.val.toLowerCase() : null;
         if (value.typ == EnumToken.IdenTokenType) {
             if (parent == null && mediaTypes.some((t) => {
-                if (value.val.localeCompare(t, 'en', { sensitivity: 'base' }) == 0) {
+                if (val === t) {
                     // @ts-ignore
                     value.typ = EnumToken.MediaFeatureTokenType;
                     return true;
@@ -870,18 +915,18 @@ function parseAtRulePrelude(tokens, atRule) {
             })) {
                 continue;
             }
-            if (value.typ == EnumToken.IdenTokenType && 'and'.localeCompare(value.val, 'en', { sensitivity: 'base' }) == 0) {
+            if (value.typ == EnumToken.IdenTokenType && 'and' === val) {
                 // @ts-ignore
                 value.typ = EnumToken.MediaFeatureAndTokenType;
                 continue;
             }
-            if (value.typ == EnumToken.IdenTokenType && 'or'.localeCompare(value.val, 'en', { sensitivity: 'base' }) == 0) {
+            if (value.typ == EnumToken.IdenTokenType && 'or' === val) {
                 // @ts-ignore
                 value.typ = EnumToken.MediaFeatureOrTokenType;
                 continue;
             }
             if (value.typ == EnumToken.IdenTokenType &&
-                ['not', 'only'].some((t) => t.localeCompare(value.val, 'en', { sensitivity: 'base' }) == 0)) {
+                ['not', 'only'].some((t) => val === t)) {
                 // @ts-ignore
                 const array = parent?.chi ?? tokens;
                 const startIndex = array.indexOf(value);
@@ -982,6 +1027,27 @@ function parseAtRulePrelude(tokens, atRule) {
     }
     return tokens;
 }
+/**
+ * parse a string as an array of declaration nodes
+ * @param declaration
+ *
+ * Example:
+ * ````ts
+ *
+ * const declarations = await parseDeclarations('color: red; background: blue');
+ * console.log(declarations);
+ * ```
+ */
+async function parseDeclarations(declaration) {
+    return doParse(tokenize({
+        stream: `.x{${declaration}}`,
+        buffer: '',
+        position: { ind: 0, lin: 1, col: 1 },
+        currentPosition: { ind: -1, lin: 1, col: 0 }
+    }), { setParent: false, minify: false, validation: false }).then(result => {
+        return result.ast.chi[0].chi.filter(t => t.typ == EnumToken.DeclarationNodeType);
+    });
+}
 /**
  * parse selector
  * @param tokens
@@ -1083,12 +1149,33 @@ function parseSelector(tokens) {
     return tokens;
 }
 /**
- * parse css string
+ * parse css string and return an array of tokens
  * @param src
  * @param options
+ *
+ * @private
+ *
+ * Example:
+ *
+ * ```ts
+ *
+ * import {parseString} from '@tbela99/css-parser';
+ *
+ * let tokens = parseString('body { color: red; }');
+ * console.log(tokens);
+ *
+ *  tokens = parseString('#c322c980');
+ * console.log(tokens);
+ * ```
  */
 function parseString(src, options = { location: false }) {
-    return parseTokens([...tokenize(src)].reduce((acc, t) => {
+    const parseInfo = {
+        stream: src,
+        buffer: '',
+        position: { ind: 0, lin: 1, col: 1 },
+        currentPosition: { ind: -1, lin: 1, col: 0 }
+    };
+    return parseTokens([...tokenize(parseInfo)].reduce((acc, t) => {
         if (t.hint == EnumToken.EOFTokenType) {
             return acc;
         }
@@ -1100,6 +1187,11 @@ function parseString(src, options = { location: false }) {
         return acc;
     }, []));
 }
+/**
+ * get token type from a string
+ * @param val
+ * @param hint
+ */
 function getTokenType(val, hint) {
     if (hint != null) {
         return enumTokenHints.has(hint) ? { typ: hint } : { typ: hint, val };
@@ -1194,19 +1286,19 @@ function getTokenType(val, hint) {
     if (isNumber(val)) {
         return {
             typ: EnumToken.NumberTokenType,
-            val
+            val: +val
         };
     }
     if (isPercentage(val)) {
         return {
             typ: EnumToken.PercentageTokenType,
-            val: val.slice(0, -1)
+            val: +val.slice(0, -1)
         };
     }
     if (isFlex(val)) {
         return {
             typ: EnumToken.FlexTokenType,
-            val: val.slice(0, -2)
+            val: +val.slice(0, -2)
         };
     }
     if (isDimension(val)) {
@@ -1265,9 +1357,24 @@ function getTokenType(val, hint) {
     };
 }
 /**
- * parse token array into a tree structure
+ * parse function tokens in a token array
  * @param tokens
  * @param options
+ *
+ * Example:
+ *
+ * ```ts
+ *
+ * import {parseString, parseTokens} from '@tbela99/css-parser';
+ *
+ * let tokens = parseString('body { color: red; }');
+ * console.log(parseTokens(tokens));
+ *
+ *  tokens = parseString('#c322c980');
+ * console.log(parseTokens(tokens));
+ * ```
+ *
+ * @private
  */
 function parseTokens(tokens, options = {}) {
     for (let i = 0; i < tokens.length; i++) {
@@ -1555,4 +1662,4 @@ function parseTokens(tokens, options = {}) {
     return tokens;
 }
 
-export { doParse, getTokenType, parseAtRulePrelude, parseSelector, parseString, parseTokens, urlTokenMatcher };
+export { doParse, getTokenType, parseAtRulePrelude, parseDeclarations, parseSelector, parseString, parseTokens, urlTokenMatcher };

package/dist/lib/parser/tokenize.js

@@ -3,7 +3,7 @@ import '../ast/minify.js';
 import '../ast/walk.js';
 import './parse.js';
 import './utils/config.js';
-import { isWhiteSpace, isIdentStart, isIdent, isNewLine, isDigit, isNonPrintable } from '../syntax/syntax.js';
+import { isWhiteSpace, isIdentStart, isIdent, isNewLine, isDigit } from '../syntax/syntax.js';
 import '../syntax/color/utils/constants.js';
 import '../renderer/sourcemap/lib/encode.js';
 
@@ -40,7 +40,7 @@ var TokenMap;
 })(TokenMap || (TokenMap = {}));
 function consumeWhiteSpace(parseInfo) {
     let count = 0;
-    while (isWhiteSpace(parseInfo.stream.charAt(count + parseInfo.currentPosition.ind + 1).charCodeAt(0))) {
+    while (isWhiteSpace(parseInfo.stream.charCodeAt(count + parseInfo.currentPosition.ind + 1))) {
         count++;
     }
     next(parseInfo, count);
@@ -146,11 +146,8 @@ function peek(parseInfo, count = 1) {
     }
     return parseInfo.stream.slice(parseInfo.currentPosition.ind + 1, parseInfo.currentPosition.ind + count + 1);
 }
-function prev(parseInfo, count = 1) {
-    if (count == 1) {
-        return parseInfo.currentPosition.ind == 0 ? '' : parseInfo.stream.charAt(parseInfo.currentPosition.ind - 1);
-    }
-    return parseInfo.stream.slice(parseInfo.currentPosition.ind - 1 - count, parseInfo.currentPosition.ind - 1);
+function prev(parseInfo) {
+    return parseInfo.stream.charAt(parseInfo.currentPosition.ind - 1);
 }
 function next(parseInfo, count = 1) {
     let char = '';
@@ -170,17 +167,14 @@ function next(parseInfo, count = 1) {
 }
 /**
  * tokenize css string
- * @param stream
+ * @param parseInfo
+ * @param yieldEOFToken
  */
-function* tokenize(stream) {
-    const parseInfo = {
-        stream,
-        position: { ind: 0, lin: 1, col: 1 },
-        currentPosition: { ind: -1, lin: 1, col: 0 }
-    };
+function* tokenize(parseInfo, yieldEOFToken = true) {
     let value;
-    let buffer = '';
+    let buffer = parseInfo.buffer;
     let charCode;
+    parseInfo.buffer = '';
     while (value = next(parseInfo)) {
         charCode = value.charCodeAt(0);
         if (isWhiteSpace(charCode)) {
@@ -424,8 +418,6 @@ function* tokenize(stream) {
                     value = peek(parseInfo);
                     // let cp: number;
                     let whitespace = '';
-                    let hasWhiteSpace = false;
-                    let errorState = false;
                     if (value == '"' || value == "'") {
                         const quote = value;
                         let inquote = true;
@@ -445,11 +437,6 @@ function* tokenize(stream) {
                                             break;
                                         }
                                     }
-                                    if (value === '') {
-                                        yield pushToken(buffer, parseInfo, EnumToken.BadUrlTokenType);
-                                        buffer = '';
-                                        break;
-                                    }
                                     charCode = value.charCodeAt(0);
                                 }
                                 // '\\'
@@ -465,7 +452,6 @@ function* tokenize(stream) {
                                 if (isWhiteSpace(charCode)) {
                                     whitespace += value;
                                     while (value = peek(parseInfo)) {
-                                        hasWhiteSpace = true;
                                         if (isWhiteSpace(value?.charCodeAt(0))) {
                                             whitespace += next(parseInfo);
                                             continue;
@@ -488,16 +474,6 @@ function* tokenize(stream) {
                                 }
                                 while (value = next(parseInfo)) {
                                     charCode = value.charCodeAt(0);
-                                    if (charCode == 0x5c) {
-                                        buffer += value + next(parseInfo);
-                                        continue;
-                                    }
-                                    if (charCode == 0x29) {
-                                        yield pushToken(buffer, parseInfo, EnumToken.BadStringTokenType);
-                                        yield pushToken('', parseInfo, EnumToken.EndParensTokenType);
-                                        buffer = '';
-                                        break;
-                                    }
                                     buffer += value;
                                 }
                                 if (hasNewLine) {
@@ -514,49 +490,12 @@ function* tokenize(stream) {
                         buffer = '';
                         while (value = next(parseInfo)) {
                             charCode = value.charCodeAt(0);
-                            // ')'
-                            if (charCode == 0x29) {
+                            if (charCode == 0x29) { // ')'
                                 yield pushToken(buffer, parseInfo, EnumToken.UrlTokenTokenType);
                                 yield pushToken('', parseInfo, EnumToken.EndParensTokenType);
                                 buffer = '';
                                 break;
                             }
-                            if (isWhiteSpace(charCode)) {
-                                hasWhiteSpace = true;
-                                whitespace = value;
-                                while (isWhiteSpace(peek(parseInfo)?.charCodeAt(0))) {
-                                    whitespace += next(parseInfo);
-                                }
-                                continue;
-                            }
-                            if (isNonPrintable(charCode) ||
-                                // '"'
-                                charCode == 0x22 ||
-                                // "'"
-                                charCode == 0x27 ||
-                                // \('
-                                charCode == 0x28 ||
-                                hasWhiteSpace) {
-                                errorState = true;
-                            }
-                            if (errorState) {
-                                buffer += whitespace + value;
-                                while (value = peek(parseInfo)) {
-                                    charCode = value.charCodeAt(0);
-                                    if (charCode == 0x5c) {
-                                        buffer += next(parseInfo, 2);
-                                        continue;
-                                    }
-                                    // ')'
-                                    if (charCode == 0x29) {
-                                        break;
-                                    }
-                                    buffer += next(parseInfo);
-                                }
-                                yield pushToken(buffer, parseInfo, EnumToken.BadUrlTokenType);
-                                buffer = '';
-                                break;
-                            }
                             buffer += value;
                         }
                     }
@@ -595,10 +534,6 @@ function* tokenize(stream) {
                 buffer = '!';
                 break;
             case 64 /* TokenMap.AT */:
-                if (buffer.length > 0) {
-                    yield pushToken(buffer, parseInfo);
-                    buffer = '';
-                }
                 buffer = value;
                 {
                     let val = peek(parseInfo);
@@ -616,27 +551,40 @@ function* tokenize(stream) {
                 break;
             default:
                 buffer += value;
-                if (buffer.length == 1) {
-                    if (buffer == 'h') {
-                        if (match(parseInfo, 'ttp://') || match(parseInfo, 'ttps://')) {
-                            let val = peek(parseInfo);
-                            while (val != ')' && val != ';' && !isWhiteSpace(val.charCodeAt(0))) {
-                                buffer += next(parseInfo);
-                                val = peek(parseInfo);
-                            }
-                            yield pushToken(buffer, parseInfo);
-                            buffer = '';
-                            break;
-                        }
-                    }
-                }
                 break;
         }
     }
-    if (buffer.length > 0) {
-        yield pushToken(buffer, parseInfo);
+    if (yieldEOFToken) {
+        if (buffer.length > 0) {
+            yield pushToken(buffer, parseInfo);
+        }
+        yield pushToken('', parseInfo, EnumToken.EOFTokenType);
+    }
+    else {
+        parseInfo.buffer = buffer;
+    }
+}
+/**
+ * tokenize readable stream
+ * @param input
+ */
+async function* tokenizeStream(input) {
+    const parseInfo = {
+        stream: '',
+        buffer: '',
+        position: { ind: 0, lin: 1, col: 1 },
+        currentPosition: { ind: -1, lin: 1, col: 0 }
+    };
+    const decoder = new TextDecoder('utf-8');
+    const reader = input.getReader();
+    while (true) {
+        const { done, value } = await reader.read();
+        parseInfo.stream += decoder.decode(value, { stream: true });
+        yield* tokenize(parseInfo, done);
+        if (done) {
+            break;
+        }
     }
-    yield pushToken('', parseInfo, EnumToken.EOFTokenType);
 }
 
-export { tokenize };
+export { tokenize, tokenizeStream };

package/dist/lib/parser/utils/type.js

@@ -14,7 +14,7 @@ function matchType(val, properties) {
         (properties.types.some((t) => EnumToken[t] == val.typ))) {
         return true;
     }
-    if (val.typ == EnumToken.NumberTokenType && val.val == '0') {
+    if (val.typ == EnumToken.NumberTokenType && val.val == 0) {
         // @ts-ignore
         return properties.types.some((type) => {
             // @ts-ignore