@eslint/css-tree 3.2.0

package/lib/tokenizer/TokenStream.js

@@ -0,0 +1,366 @@
+import { adoptBuffer } from './adopt-buffer.js';
+import { cmpStr } from './utils.js';
+import tokenNames from './names.js';
+import {
+    WhiteSpace,
+    Comment,
+    Delim,
+    EOF,
+    Function as FunctionToken,
+    LeftParenthesis,
+    RightParenthesis,
+    LeftSquareBracket,
+    RightSquareBracket,
+    LeftCurlyBracket,
+    RightCurlyBracket
+} from './types.js';
+
+const OFFSET_MASK = 0x00FFFFFF;
+const TYPE_SHIFT = 24;
+const BLOCK_OPEN_TOKEN = 1;
+const BLOCK_CLOSE_TOKEN = 2;
+const balancePair = new Uint8Array(32); // 32b of memory ought to be enough for anyone (any number of tokens)
+balancePair[FunctionToken] = RightParenthesis;
+balancePair[LeftParenthesis] = RightParenthesis;
+balancePair[LeftSquareBracket] = RightSquareBracket;
+balancePair[LeftCurlyBracket] = RightCurlyBracket;
+
+const blockTokens = new Uint8Array(32);
+blockTokens[FunctionToken] = BLOCK_OPEN_TOKEN;
+blockTokens[LeftParenthesis] = BLOCK_OPEN_TOKEN;
+blockTokens[LeftSquareBracket] = BLOCK_OPEN_TOKEN;
+blockTokens[LeftCurlyBracket] = BLOCK_OPEN_TOKEN;
+blockTokens[RightParenthesis] = BLOCK_CLOSE_TOKEN;
+blockTokens[RightSquareBracket] = BLOCK_CLOSE_TOKEN;
+blockTokens[RightCurlyBracket] = BLOCK_CLOSE_TOKEN;
+
+function boundIndex(index, min, max) {
+    return index < min ? min : index > max ? max : index;
+}
+
+export class TokenStream {
+    constructor(source, tokenize) {
+        this.setSource(source, tokenize);
+    }
+    reset() {
+        this.eof = false;
+        this.tokenIndex = -1;
+        this.tokenType = 0;
+        this.tokenStart = this.firstCharOffset;
+        this.tokenEnd = this.firstCharOffset;
+    }
+    setSource(source = '', tokenize = () => {}) {
+        source = String(source || '');
+
+        const sourceLength = source.length;
+        const offsetAndType = adoptBuffer(this.offsetAndType, source.length + 1); // +1 because of eof-token
+        const balance = adoptBuffer(this.balance, source.length + 1);
+        let tokenCount = 0;
+        let firstCharOffset = -1;
+        let balanceCloseType = 0;
+        let balanceStart = source.length;
+
+        // capture buffers
+        this.offsetAndType = null;
+        this.balance = null;
+        balance.fill(0);
+
+        tokenize(source, (type, start, end) => {
+            const index = tokenCount++;
+
+            // type & offset
+            offsetAndType[index] = (type << TYPE_SHIFT) | end;
+
+            if (firstCharOffset === -1) {
+                firstCharOffset = start;
+            }
+
+            // balance
+            balance[index] = balanceStart;
+
+            if (type === balanceCloseType) {
+                const prevBalanceStart = balance[balanceStart];
+
+                // set reference to balance end for a block opener
+                balance[balanceStart] = index;
+
+                // pop state
+                balanceStart = prevBalanceStart;
+                balanceCloseType = balancePair[offsetAndType[prevBalanceStart] >> TYPE_SHIFT];
+            } else if (this.isBlockOpenerTokenType(type)) { // check for FunctionToken, <(-token>, <[-token> and <{-token>
+                // push state
+                balanceStart = index;
+                balanceCloseType = balancePair[type];
+            }
+        });
+
+        // finalize buffers
+        offsetAndType[tokenCount] = (EOF << TYPE_SHIFT) | sourceLength; // <EOF-token>
+        balance[tokenCount] = tokenCount; // prevents false positive balance match with any token
+
+        // reverse references from balance start to end
+        // tokens
+        //   token:   a ( [ b c ] d e ) {
+        //   index:   0 1 2 3 4 5 6 7 8 9
+        // before
+        //   balance: 0 8 5 2 2 2 1 1 1 0
+        //            - > > < < < < < < -
+        // after
+        //   balance: 9 8 5 5 5 2 8 8 1 9
+        //            > > > > > < > > < >
+        for (let i = 0; i < tokenCount; i++) {
+            const balanceStart = balance[i];
+
+            if (balanceStart <= i) {
+                const balanceEnd = balance[balanceStart];
+
+                if (balanceEnd !== i) {
+                    balance[i] = balanceEnd;
+                }
+            } else if (balanceStart > tokenCount) {
+                balance[i] = tokenCount;
+            }
+        }
+
+        // balance[0] = tokenCount;
+
+        this.source = source;
+        this.firstCharOffset = firstCharOffset === -1 ? 0 : firstCharOffset;
+        this.tokenCount = tokenCount;
+        this.offsetAndType = offsetAndType;
+        this.balance = balance;
+
+        this.reset();
+        this.next();
+    }
+
+    lookupType(offset) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return this.offsetAndType[offset] >> TYPE_SHIFT;
+        }
+
+        return EOF;
+    }
+    lookupTypeNonSC(idx) {
+        for (let offset = this.tokenIndex; offset < this.tokenCount; offset++) {
+            const tokenType = this.offsetAndType[offset] >> TYPE_SHIFT;
+
+            if (tokenType !== WhiteSpace && tokenType !== Comment) {
+                if (idx-- === 0) {
+                    return tokenType;
+                }
+            }
+        }
+
+        return EOF;
+    }
+    lookupOffset(offset) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return this.offsetAndType[offset - 1] & OFFSET_MASK;
+        }
+
+        return this.source.length;
+    }
+    lookupOffsetNonSC(idx) {
+        for (let offset = this.tokenIndex; offset < this.tokenCount; offset++) {
+            const tokenType = this.offsetAndType[offset] >> TYPE_SHIFT;
+
+            if (tokenType !== WhiteSpace && tokenType !== Comment) {
+                if (idx-- === 0) {
+                    return offset - this.tokenIndex;
+                }
+            }
+        }
+
+        return EOF;
+    }
+    lookupValue(offset, referenceStr) {
+        offset += this.tokenIndex;
+
+        if (offset < this.tokenCount) {
+            return cmpStr(
+                this.source,
+                this.offsetAndType[offset - 1] & OFFSET_MASK,
+                this.offsetAndType[offset] & OFFSET_MASK,
+                referenceStr
+            );
+        }
+
+        return false;
+    }
+    getTokenStart(tokenIndex) {
+        if (tokenIndex === this.tokenIndex) {
+            return this.tokenStart;
+        }
+
+        if (tokenIndex > 0) {
+            return tokenIndex < this.tokenCount
+                ? this.offsetAndType[tokenIndex - 1] & OFFSET_MASK
+                : this.offsetAndType[this.tokenCount] & OFFSET_MASK;
+        }
+
+        return this.firstCharOffset;
+    }
+    getTokenEnd(tokenIndex) {
+        if (tokenIndex === this.tokenIndex) {
+            return this.tokenEnd;
+        }
+
+        return this.offsetAndType[boundIndex(tokenIndex, 0, this.tokenCount)] & OFFSET_MASK;
+    }
+    getTokenType(tokenIndex) {
+        if (tokenIndex === this.tokenIndex) {
+            return this.tokenType;
+        }
+
+        return this.offsetAndType[boundIndex(tokenIndex, 0, this.tokenCount)] >> TYPE_SHIFT;
+    }
+    substrToCursor(start) {
+        return this.source.substring(start, this.tokenStart);
+    }
+
+    isBlockOpenerTokenType(tokenType) {
+        return blockTokens[tokenType] === BLOCK_OPEN_TOKEN;
+    }
+    isBlockCloserTokenType(tokenType) {
+        return blockTokens[tokenType] === BLOCK_CLOSE_TOKEN;
+    }
+    getBlockTokenPairIndex(tokenIndex) {
+        const type = this.getTokenType(tokenIndex);
+
+        if (blockTokens[type] === 1) {
+            // block open token
+            const pairIndex = this.balance[tokenIndex];
+            const closeType = this.getTokenType(pairIndex);
+
+            return balancePair[type] === closeType ? pairIndex : -1;
+        } else if (blockTokens[type] === 2) {
+            // block close token
+            const pairIndex = this.balance[tokenIndex];
+            const openType = this.getTokenType(pairIndex);
+
+            return balancePair[openType] === type ? pairIndex : -1;
+        }
+
+        return -1;
+    }
+    isBalanceEdge(tokenIndex) {
+        return this.balance[this.tokenIndex] < tokenIndex;
+    }
+
+    isDelim(code, offset) {
+        if (offset) {
+            return (
+                this.lookupType(offset) === Delim &&
+                this.source.charCodeAt(this.lookupOffset(offset)) === code
+            );
+        }
+
+        return (
+            this.tokenType === Delim &&
+            this.source.charCodeAt(this.tokenStart) === code
+        );
+    }
+
+    skip(tokenCount) {
+        let next = this.tokenIndex + tokenCount;
+
+        if (next < this.tokenCount) {
+            this.tokenIndex = next;
+            this.tokenStart = this.offsetAndType[next - 1] & OFFSET_MASK;
+            next = this.offsetAndType[next];
+            this.tokenType = next >> TYPE_SHIFT;
+            this.tokenEnd = next & OFFSET_MASK;
+        } else {
+            this.tokenIndex = this.tokenCount;
+            this.next();
+        }
+    }
+    next() {
+        let next = this.tokenIndex + 1;
+
+        if (next < this.tokenCount) {
+            this.tokenIndex = next;
+            this.tokenStart = this.tokenEnd;
+            next = this.offsetAndType[next];
+            this.tokenType = next >> TYPE_SHIFT;
+            this.tokenEnd = next & OFFSET_MASK;
+        } else {
+            this.eof = true;
+            this.tokenIndex = this.tokenCount;
+            this.tokenType = EOF;
+            this.tokenStart = this.tokenEnd = this.source.length;
+        }
+    }
+    skipSC() {
+        while (this.tokenType === WhiteSpace || this.tokenType === Comment) {
+            this.next();
+        }
+    }
+    skipUntilBalanced(startToken, stopConsume) {
+        let cursor = startToken;
+        let balanceEnd = 0;
+        let offset = 0;
+
+        loop:
+        for (; cursor < this.tokenCount; cursor++) {
+            balanceEnd = this.balance[cursor];
+
+            // stop scanning on balance edge that points to offset before start token
+            if (balanceEnd < startToken) {
+                break loop;
+            }
+
+            offset = cursor > 0 ? this.offsetAndType[cursor - 1] & OFFSET_MASK : this.firstCharOffset;
+
+            // check stop condition
+            switch (stopConsume(this.source.charCodeAt(offset))) {
+                case 1: // just stop
+                    break loop;
+
+                case 2: // stop & included
+                    cursor++;
+                    break loop;
+
+                default:
+                    // fast forward to the end of balanced block for an open block tokens
+                    if (this.isBlockOpenerTokenType(this.offsetAndType[cursor] >> TYPE_SHIFT)) {
+                        cursor = balanceEnd;
+                    }
+            }
+        }
+
+        this.skip(cursor - this.tokenIndex);
+    }
+
+    forEachToken(fn) {
+        for (let i = 0, offset = this.firstCharOffset; i < this.tokenCount; i++) {
+            const start = offset;
+            const item = this.offsetAndType[i];
+            const end = item & OFFSET_MASK;
+            const type = item >> TYPE_SHIFT;
+
+            offset = end;
+
+            fn(type, start, end, i);
+        }
+    }
+    dump() {
+        const tokens = new Array(this.tokenCount);
+
+        this.forEachToken((type, start, end, index) => {
+            tokens[index] = {
+                idx: index,
+                type: tokenNames[type],
+                chunk: this.source.substring(start, end),
+                balance: this.balance[index]
+            };
+        });
+
+        return tokens;
+    }
+};

package/lib/tokenizer/adopt-buffer.js

@@ -0,0 +1,9 @@
+const MIN_SIZE = 16 * 1024;
+
+export function adoptBuffer(buffer = null, size) {
+    if (buffer === null || buffer.length < size) {
+        return new Uint32Array(Math.max(size + 1024, MIN_SIZE));
+    }
+
+    return buffer;
+};

package/lib/tokenizer/char-code-definitions.js

@@ -0,0 +1,212 @@
+const EOF = 0;
+
+// https://drafts.csswg.org/css-syntax-3/
+// § 4.2. Definitions
+
+// digit
+// A code point between U+0030 DIGIT ZERO (0) and U+0039 DIGIT NINE (9).
+export function isDigit(code) {
+    return code >= 0x0030 && code <= 0x0039;
+}
+
+// hex digit
+// A digit, or a code point between U+0041 LATIN CAPITAL LETTER A (A) and U+0046 LATIN CAPITAL LETTER F (F),
+// or a code point between U+0061 LATIN SMALL LETTER A (a) and U+0066 LATIN SMALL LETTER F (f).
+export function isHexDigit(code) {
+    return (
+        isDigit(code) || // 0 .. 9
+        (code >= 0x0041 && code <= 0x0046) || // A .. F
+        (code >= 0x0061 && code <= 0x0066)    // a .. f
+    );
+}
+
+// uppercase letter
+// A code point between U+0041 LATIN CAPITAL LETTER A (A) and U+005A LATIN CAPITAL LETTER Z (Z).
+export function isUppercaseLetter(code) {
+    return code >= 0x0041 && code <= 0x005A;
+}
+
+// lowercase letter
+// A code point between U+0061 LATIN SMALL LETTER A (a) and U+007A LATIN SMALL LETTER Z (z).
+export function isLowercaseLetter(code) {
+    return code >= 0x0061 && code <= 0x007A;
+}
+
+// letter
+// An uppercase letter or a lowercase letter.
+export function isLetter(code) {
+    return isUppercaseLetter(code) || isLowercaseLetter(code);
+}
+
+// non-ASCII code point
+// A code point with a value equal to or greater than U+0080 <control>.
+//
+// 2024-09-02: The latest spec narrows the range for non-ASCII characters (see https://github.com/csstree/csstree/issues/188).
+// However, all modern browsers support a wider range, and strictly following the latest spec could result
+// in some CSS being parsed incorrectly, even though it works in the browser. Therefore, this function adheres
+// to the previous, broader definition of non-ASCII characters.
+export function isNonAscii(code) {
+    return code >= 0x0080;
+}
+
+// name-start code point
+// A letter, a non-ASCII code point, or U+005F LOW LINE (_).
+export function isNameStart(code) {
+    return isLetter(code) || isNonAscii(code) || code === 0x005F;
+}
+
+// name code point
+// A name-start code point, a digit, or U+002D HYPHEN-MINUS (-).
+export function isName(code) {
+    return isNameStart(code) || isDigit(code) || code === 0x002D;
+}
+
+// non-printable code point
+// A code point between U+0000 NULL and U+0008 BACKSPACE, or U+000B LINE TABULATION,
+// or a code point between U+000E SHIFT OUT and U+001F INFORMATION SEPARATOR ONE, or U+007F DELETE.
+export function isNonPrintable(code) {
+    return (
+        (code >= 0x0000 && code <= 0x0008) ||
+        (code === 0x000B) ||
+        (code >= 0x000E && code <= 0x001F) ||
+        (code === 0x007F)
+    );
+}
+
+// newline
+// U+000A LINE FEED. Note that U+000D CARRIAGE RETURN and U+000C FORM FEED are not included in this definition,
+// as they are converted to U+000A LINE FEED during preprocessing.
+// TODO: we doesn't do a preprocessing, so check a code point for U+000D CARRIAGE RETURN and U+000C FORM FEED
+export function isNewline(code) {
+    return code === 0x000A || code === 0x000D || code === 0x000C;
+}
+
+// whitespace
+// A newline, U+0009 CHARACTER TABULATION, or U+0020 SPACE.
+export function isWhiteSpace(code) {
+    return isNewline(code) || code === 0x0020 || code === 0x0009;
+}
+
+// § 4.3.8. Check if two code points are a valid escape
+export function isValidEscape(first, second) {
+    // If the first code point is not U+005C REVERSE SOLIDUS (\), return false.
+    if (first !== 0x005C) {
+        return false;
+    }
+
+    // Otherwise, if the second code point is a newline or EOF, return false.
+    if (isNewline(second) || second === EOF) {
+        return false;
+    }
+
+    // Otherwise, return true.
+    return true;
+}
+
+// § 4.3.9. Check if three code points would start an identifier
+export function isIdentifierStart(first, second, third) {
+    // Look at the first code point:
+
+    // U+002D HYPHEN-MINUS
+    if (first === 0x002D) {
+        // If the second code point is a name-start code point or a U+002D HYPHEN-MINUS,
+        // or the second and third code points are a valid escape, return true. Otherwise, return false.
+        return (
+            isNameStart(second) ||
+            second === 0x002D ||
+            isValidEscape(second, third)
+        );
+    }
+
+    // name-start code point
+    if (isNameStart(first)) {
+        // Return true.
+        return true;
+    }
+
+    // U+005C REVERSE SOLIDUS (\)
+    if (first === 0x005C) {
+        // If the first and second code points are a valid escape, return true. Otherwise, return false.
+        return isValidEscape(first, second);
+    }
+
+    // anything else
+    // Return false.
+    return false;
+}
+
+// § 4.3.10. Check if three code points would start a number
+export function isNumberStart(first, second, third) {
+    // Look at the first code point:
+
+    // U+002B PLUS SIGN (+)
+    // U+002D HYPHEN-MINUS (-)
+    if (first === 0x002B || first === 0x002D) {
+        // If the second code point is a digit, return true.
+        if (isDigit(second)) {
+            return 2;
+        }
+
+        // Otherwise, if the second code point is a U+002E FULL STOP (.)
+        // and the third code point is a digit, return true.
+        // Otherwise, return false.
+        return second === 0x002E && isDigit(third) ? 3 : 0;
+    }
+
+    // U+002E FULL STOP (.)
+    if (first === 0x002E) {
+        // If the second code point is a digit, return true. Otherwise, return false.
+        return isDigit(second) ? 2 : 0;
+    }
+
+    // digit
+    if (isDigit(first)) {
+        // Return true.
+        return 1;
+    }
+
+    // anything else
+    // Return false.
+    return 0;
+}
+
+//
+// Misc
+//
+
+// detect BOM (https://en.wikipedia.org/wiki/Byte_order_mark)
+export function isBOM(code) {
+    // UTF-16BE
+    if (code === 0xFEFF) {
+        return 1;
+    }
+
+    // UTF-16LE
+    if (code === 0xFFFE) {
+        return 1;
+    }
+
+    return 0;
+}
+
+// Fast code category
+// Only ASCII code points has a special meaning, that's why we define a maps for 0..127 codes only
+const CATEGORY = new Array(0x80);
+export const EofCategory = 0x80;
+export const WhiteSpaceCategory = 0x82;
+export const DigitCategory = 0x83;
+export const NameStartCategory = 0x84;
+export const NonPrintableCategory = 0x85;
+
+for (let i = 0; i < CATEGORY.length; i++) {
+    CATEGORY[i] =
+        isWhiteSpace(i) && WhiteSpaceCategory ||
+        isDigit(i) && DigitCategory ||
+        isNameStart(i) && NameStartCategory ||
+        isNonPrintable(i) && NonPrintableCategory ||
+        i || EofCategory;
+}
+
+export function charCodeCategory(code) {
+    return code < 0x80 ? CATEGORY[code] : NameStartCategory;
+}