@eslint/css-tree 3.2.0

package/cjs/tokenizer/utils.cjs

@@ -0,0 +1,261 @@
+'use strict';
+
+const charCodeDefinitions = require('./char-code-definitions.cjs');
+
+function getCharCode(source, offset) {
+    return offset < source.length ? source.charCodeAt(offset) : 0;
+}
+
+function getNewlineLength(source, offset, code) {
+    if (code === 13 /* \r */ && getCharCode(source, offset + 1) === 10 /* \n */) {
+        return 2;
+    }
+
+    return 1;
+}
+
+function cmpChar(testStr, offset, referenceCode) {
+    let code = testStr.charCodeAt(offset);
+
+    // code.toLowerCase() for A..Z
+    if (charCodeDefinitions.isUppercaseLetter(code)) {
+        code = code | 32;
+    }
+
+    return code === referenceCode;
+}
+
+function cmpStr(testStr, start, end, referenceStr) {
+    if (end - start !== referenceStr.length) {
+        return false;
+    }
+
+    if (start < 0 || end > testStr.length) {
+        return false;
+    }
+
+    for (let i = start; i < end; i++) {
+        const referenceCode = referenceStr.charCodeAt(i - start);
+        let testCode = testStr.charCodeAt(i);
+
+        // testCode.toLowerCase() for A..Z
+        if (charCodeDefinitions.isUppercaseLetter(testCode)) {
+            testCode = testCode | 32;
+        }
+
+        if (testCode !== referenceCode) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function findWhiteSpaceStart(source, offset) {
+    for (; offset >= 0; offset--) {
+        if (!charCodeDefinitions.isWhiteSpace(source.charCodeAt(offset))) {
+            break;
+        }
+    }
+
+    return offset + 1;
+}
+
+function findWhiteSpaceEnd(source, offset) {
+    for (; offset < source.length; offset++) {
+        if (!charCodeDefinitions.isWhiteSpace(source.charCodeAt(offset))) {
+            break;
+        }
+    }
+
+    return offset;
+}
+
+function findDecimalNumberEnd(source, offset) {
+    for (; offset < source.length; offset++) {
+        if (!charCodeDefinitions.isDigit(source.charCodeAt(offset))) {
+            break;
+        }
+    }
+
+    return offset;
+}
+
+// § 4.3.7. Consume an escaped code point
+function consumeEscaped(source, offset) {
+    // It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and
+    // that the next input code point has already been verified to be part of a valid escape.
+    offset += 2;
+
+    // hex digit
+    if (charCodeDefinitions.isHexDigit(getCharCode(source, offset - 1))) {
+        // Consume as many hex digits as possible, but no more than 5.
+        // Note that this means 1-6 hex digits have been consumed in total.
+        for (const maxOffset = Math.min(source.length, offset + 5); offset < maxOffset; offset++) {
+            if (!charCodeDefinitions.isHexDigit(getCharCode(source, offset))) {
+                break;
+            }
+        }
+
+        // If the next input code point is whitespace, consume it as well.
+        const code = getCharCode(source, offset);
+        if (charCodeDefinitions.isWhiteSpace(code)) {
+            offset += getNewlineLength(source, offset, code);
+        }
+    }
+
+    return offset;
+}
+
+// §4.3.11. Consume a name
+// Note: This algorithm does not do the verification of the first few code points that are necessary
+// to ensure the returned code points would constitute an <ident-token>. If that is the intended use,
+// ensure that the stream starts with an identifier before calling this algorithm.
+function consumeName(source, offset) {
+    // Let result initially be an empty string.
+    // Repeatedly consume the next input code point from the stream:
+    for (; offset < source.length; offset++) {
+        const code = source.charCodeAt(offset);
+
+        // name code point
+        if (charCodeDefinitions.isName(code)) {
+            // Append the code point to result.
+            continue;
+        }
+
+        // the stream starts with a valid escape
+        if (charCodeDefinitions.isValidEscape(code, getCharCode(source, offset + 1))) {
+            // Consume an escaped code point. Append the returned code point to result.
+            offset = consumeEscaped(source, offset) - 1;
+            continue;
+        }
+
+        // anything else
+        // Reconsume the current input code point. Return result.
+        break;
+    }
+
+    return offset;
+}
+
+// §4.3.12. Consume a number
+function consumeNumber(source, offset) {
+    let code = source.charCodeAt(offset);
+
+    // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-),
+    // consume it and append it to repr.
+    if (code === 0x002B || code === 0x002D) {
+        code = source.charCodeAt(offset += 1);
+    }
+
+    // 3. While the next input code point is a digit, consume it and append it to repr.
+    if (charCodeDefinitions.isDigit(code)) {
+        offset = findDecimalNumberEnd(source, offset + 1);
+        code = source.charCodeAt(offset);
+    }
+
+    // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then:
+    if (code === 0x002E && charCodeDefinitions.isDigit(source.charCodeAt(offset + 1))) {
+        // 4.1 Consume them.
+        // 4.2 Append them to repr.
+        offset += 2;
+
+        // 4.3 Set type to "number".
+        // TODO
+
+        // 4.4 While the next input code point is a digit, consume it and append it to repr.
+
+        offset = findDecimalNumberEnd(source, offset);
+    }
+
+    // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E)
+    // or U+0065 LATIN SMALL LETTER E (e), ... , followed by a digit, then:
+    if (cmpChar(source, offset, 101 /* e */)) {
+        let sign = 0;
+        code = source.charCodeAt(offset + 1);
+
+        // ... optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+) ...
+        if (code === 0x002D || code === 0x002B) {
+            sign = 1;
+            code = source.charCodeAt(offset + 2);
+        }
+
+        // ... followed by a digit
+        if (charCodeDefinitions.isDigit(code)) {
+            // 5.1 Consume them.
+            // 5.2 Append them to repr.
+
+            // 5.3 Set type to "number".
+            // TODO
+
+            // 5.4 While the next input code point is a digit, consume it and append it to repr.
+            offset = findDecimalNumberEnd(source, offset + 1 + sign + 1);
+        }
+    }
+
+    return offset;
+}
+
+// § 4.3.14. Consume the remnants of a bad url
+// ... its sole use is to consume enough of the input stream to reach a recovery point
+// where normal tokenizing can resume.
+function consumeBadUrlRemnants(source, offset) {
+    // Repeatedly consume the next input code point from the stream:
+    for (; offset < source.length; offset++) {
+        const code = source.charCodeAt(offset);
+
+        // U+0029 RIGHT PARENTHESIS ())
+        // EOF
+        if (code === 0x0029) {
+            // Return.
+            offset++;
+            break;
+        }
+
+        if (charCodeDefinitions.isValidEscape(code, getCharCode(source, offset + 1))) {
+            // Consume an escaped code point.
+            // Note: This allows an escaped right parenthesis ("\)") to be encountered
+            // without ending the <bad-url-token>. This is otherwise identical to
+            // the "anything else" clause.
+            offset = consumeEscaped(source, offset);
+        }
+    }
+
+    return offset;
+}
+
+// § 4.3.7. Consume an escaped code point
+// Note: This algorithm assumes that escaped is valid without leading U+005C REVERSE SOLIDUS (\)
+function decodeEscaped(escaped) {
+    // Single char escaped that's not a hex digit
+    if (escaped.length === 1 && !charCodeDefinitions.isHexDigit(escaped.charCodeAt(0))) {
+        return escaped[0];
+    }
+
+    // Interpret the hex digits as a hexadecimal number.
+    let code = parseInt(escaped, 16);
+
+    if (
+        (code === 0) ||                       // If this number is zero,
+        (code >= 0xD800 && code <= 0xDFFF) || // or is for a surrogate,
+        (code > 0x10FFFF)                     // or is greater than the maximum allowed code point
+    ) {
+        // ... return U+FFFD REPLACEMENT CHARACTER
+        code = 0xFFFD;
+    }
+
+    // Otherwise, return the code point with that value.
+    return String.fromCodePoint(code);
+}
+
+exports.cmpChar = cmpChar;
+exports.cmpStr = cmpStr;
+exports.consumeBadUrlRemnants = consumeBadUrlRemnants;
+exports.consumeEscaped = consumeEscaped;
+exports.consumeName = consumeName;
+exports.consumeNumber = consumeNumber;
+exports.decodeEscaped = decodeEscaped;
+exports.findDecimalNumberEnd = findDecimalNumberEnd;
+exports.findWhiteSpaceEnd = findWhiteSpaceEnd;
+exports.findWhiteSpaceStart = findWhiteSpaceStart;
+exports.getNewlineLength = getNewlineLength;