@zzzen/pyright-internal 1.1.254

package/dist/parser/stringTokenUtils.js

@@ -0,0 +1,455 @@
+"use strict";
+/*
+ * stringTokenUtils.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ * Author: Eric Traut
+ *
+ * Methods that handle unescaping of escaped string token
+ * literal values.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getUnescapedString = void 0;
+function completeUnescapedString(incomplete) {
+    return {
+        ...incomplete,
+        value: incomplete.valueParts.join(''),
+        formatStringSegments: incomplete.formatStringSegments.map((segment) => ({
+            ...segment,
+            value: segment.valueParts.join(''),
+        })),
+    };
+}
+function getUnescapedString(stringToken) {
+    const escapedString = stringToken.escapedValue;
+    const isRaw = (stringToken.flags & 8 /* Raw */) !== 0;
+    const isFormat = (stringToken.flags & 64 /* Format */) !== 0;
+    if (isRaw && !isFormat) {
+        return {
+            value: escapedString,
+            unescapeErrors: [],
+            nonAsciiInBytes: false,
+            formatStringSegments: [],
+        };
+    }
+    const charCodes = [];
+    for (let index = 0; index < escapedString.length; index++) {
+        charCodes.push(escapedString.charCodeAt(index));
+    }
+    const isBytes = (stringToken.flags & 32 /* Bytes */) !== 0;
+    // Handle the common case in an expedited manner.
+    if (!isFormat) {
+        if (!charCodes.some((curChar) => curChar === 13 /* CarriageReturn */ || curChar === 10 /* LineFeed */ || curChar === 92 /* Backslash */)) {
+            return {
+                value: escapedString,
+                unescapeErrors: [],
+                nonAsciiInBytes: isBytes && charCodes.some((curChar) => curChar >= 128),
+                formatStringSegments: [],
+            };
+        }
+    }
+    let formatExpressionNestCount = 0;
+    let formatSegment = {
+        offset: 0,
+        length: 0,
+        valueParts: [],
+        isExpression: false,
+        hasFormatSpecifier: false,
+    };
+    let strOffset = 0;
+    const output = {
+        valueParts: [],
+        unescapeErrors: [],
+        nonAsciiInBytes: false,
+        formatStringSegments: [],
+    };
+    const addInvalidEscapeOffset = () => {
+        // Invalid escapes are not reported for raw strings.
+        if (!isRaw) {
+            output.unescapeErrors.push({
+                offset: strOffset - 1,
+                length: 2,
+                errorType: 0 /* InvalidEscapeSequence */,
+            });
+        }
+    };
+    const getEscapedCharacter = (offset = 0) => {
+        if (strOffset + offset >= charCodes.length) {
+            return 3 /* EndOfText */;
+        }
+        return charCodes[strOffset + offset];
+    };
+    const scanHexEscape = (digitCount) => {
+        let foundIllegalHexDigit = false;
+        let hexValue = 0;
+        let localValue = '';
+        for (let i = 0; i < digitCount; i++) {
+            const charCode = getEscapedCharacter(1 + i);
+            if (!_isHexCharCode(charCode)) {
+                foundIllegalHexDigit = true;
+                break;
+            }
+            hexValue = 16 * hexValue + _getHexDigitValue(charCode);
+        }
+        if (foundIllegalHexDigit) {
+            addInvalidEscapeOffset();
+            localValue = '\\' + String.fromCharCode(getEscapedCharacter());
+            strOffset++;
+        }
+        else {
+            localValue = String.fromCharCode(hexValue);
+            strOffset += 1 + digitCount;
+        }
+        return localValue;
+    };
+    const appendOutputChar = (charCode) => {
+        const char = String.fromCharCode(charCode);
+        output.valueParts.push(char);
+        formatSegment.valueParts.push(char);
+    };
+    while (true) {
+        let curChar = getEscapedCharacter();
+        if (curChar === 3 /* EndOfText */) {
+            if (isFormat) {
+                if (formatSegment.isExpression) {
+                    // The last format segment was an unterminated expression.
+                    output.unescapeErrors.push({
+                        offset: formatSegment.offset,
+                        length: strOffset - formatSegment.offset,
+                        errorType: 3 /* UnterminatedFormatExpression */,
+                    });
+                }
+                // Push the last segment.
+                if (strOffset !== formatSegment.offset) {
+                    formatSegment.length = strOffset - formatSegment.offset;
+                    output.formatStringSegments.push(formatSegment);
+                }
+            }
+            return completeUnescapedString(output);
+        }
+        if (curChar === 92 /* Backslash */) {
+            if (isFormat && formatSegment.isExpression && !formatSegment.hasFormatSpecifier) {
+                // Backslashes aren't allowed within format string expressions.
+                output.unescapeErrors.push({
+                    offset: strOffset,
+                    length: 1,
+                    errorType: 1 /* EscapeWithinFormatExpression */,
+                });
+            }
+            // Move past the escape (backslash) character.
+            strOffset++;
+            if (isRaw) {
+                appendOutputChar(curChar);
+                continue;
+            }
+            curChar = getEscapedCharacter();
+            let localValue = '';
+            if (curChar === 13 /* CarriageReturn */ || curChar === 10 /* LineFeed */) {
+                if (curChar === 13 /* CarriageReturn */ && getEscapedCharacter(1) === 10 /* LineFeed */) {
+                    if (isRaw) {
+                        localValue += String.fromCharCode(curChar);
+                    }
+                    strOffset++;
+                    curChar = getEscapedCharacter();
+                }
+                if (isRaw) {
+                    localValue = '\\' + localValue + String.fromCharCode(curChar);
+                }
+                strOffset++;
+            }
+            else {
+                if (isRaw) {
+                    localValue = '\\' + String.fromCharCode(curChar);
+                    strOffset++;
+                }
+                else {
+                    switch (curChar) {
+                        case 92 /* Backslash */:
+                        case 39 /* SingleQuote */:
+                        case 34 /* DoubleQuote */:
+                            localValue = String.fromCharCode(curChar);
+                            strOffset++;
+                            break;
+                        case 97 /* a */:
+                            localValue = '\u0007';
+                            strOffset++;
+                            break;
+                        case 98 /* b */:
+                            localValue = '\b';
+                            strOffset++;
+                            break;
+                        case 102 /* f */:
+                            localValue = '\f';
+                            strOffset++;
+                            break;
+                        case 110 /* n */:
+                            localValue = '\n';
+                            strOffset++;
+                            break;
+                        case 114 /* r */:
+                            localValue = '\r';
+                            strOffset++;
+                            break;
+                        case 116 /* t */:
+                            localValue = '\t';
+                            strOffset++;
+                            break;
+                        case 118 /* v */:
+                            localValue = '\v';
+                            strOffset++;
+                            break;
+                        case 120 /* x */:
+                            localValue = scanHexEscape(2);
+                            break;
+                        case 78 /* N */: {
+                            let foundIllegalChar = false;
+                            let charCount = 1;
+                            if (getEscapedCharacter(charCount) !== 123 /* OpenBrace */) {
+                                foundIllegalChar = true;
+                            }
+                            else {
+                                charCount++;
+                                while (true) {
+                                    const lookaheadChar = getEscapedCharacter(charCount);
+                                    if (lookaheadChar === 125 /* CloseBrace */) {
+                                        break;
+                                    }
+                                    else if (!_isAlphaNumericChar(lookaheadChar) &&
+                                        lookaheadChar !== 45 /* Hyphen */ &&
+                                        !_isWhitespaceChar(lookaheadChar)) {
+                                        foundIllegalChar = true;
+                                        break;
+                                    }
+                                    else {
+                                        charCount++;
+                                    }
+                                }
+                            }
+                            if (foundIllegalChar) {
+                                addInvalidEscapeOffset();
+                                localValue = '\\' + String.fromCharCode(curChar);
+                                strOffset++;
+                            }
+                            else {
+                                // We don't have the Unicode name database handy, so
+                                // assume that the name is valid and use a '-' as a
+                                // replacement character.
+                                localValue = '-';
+                                strOffset += 1 + charCount;
+                            }
+                            break;
+                        }
+                        case 117 /* u */:
+                            localValue = scanHexEscape(4);
+                            break;
+                        case 85 /* U */:
+                            localValue = scanHexEscape(8);
+                            break;
+                        default:
+                            if (_isOctalCharCode(curChar)) {
+                                let octalCode = curChar - 48 /* _0 */;
+                                strOffset++;
+                                curChar = getEscapedCharacter();
+                                if (_isOctalCharCode(curChar)) {
+                                    octalCode = octalCode * 8 + curChar - 48 /* _0 */;
+                                    strOffset++;
+                                    curChar = getEscapedCharacter();
+                                    if (_isOctalCharCode(curChar)) {
+                                        octalCode = octalCode * 8 + curChar - 48 /* _0 */;
+                                        strOffset++;
+                                    }
+                                }
+                                localValue = String.fromCharCode(octalCode);
+                            }
+                            else {
+                                localValue = '\\';
+                                addInvalidEscapeOffset();
+                            }
+                            break;
+                    }
+                }
+            }
+            output.valueParts.push(localValue);
+            formatSegment.valueParts.push(localValue);
+        }
+        else if (curChar === 10 /* LineFeed */ || curChar === 13 /* CarriageReturn */) {
+            // Skip over the escaped new line (either one or two characters).
+            if (curChar === 13 /* CarriageReturn */ && getEscapedCharacter(1) === 10 /* LineFeed */) {
+                appendOutputChar(curChar);
+                strOffset++;
+                curChar = getEscapedCharacter();
+            }
+            appendOutputChar(curChar);
+            strOffset++;
+        }
+        else if (isFormat && curChar === 123 /* OpenBrace */) {
+            if (!formatSegment.isExpression && getEscapedCharacter(1) === 123 /* OpenBrace */) {
+                appendOutputChar(curChar);
+                strOffset += 2;
+            }
+            else {
+                if (formatExpressionNestCount === 0) {
+                    // A single open brace within a format literal indicates that
+                    // an expression is starting.
+                    formatSegment.length = strOffset - formatSegment.offset;
+                    if (formatSegment.length > 0) {
+                        output.formatStringSegments.push(formatSegment);
+                    }
+                    strOffset++;
+                    // Start a new segment.
+                    formatSegment = {
+                        offset: strOffset,
+                        length: 0,
+                        valueParts: [],
+                        isExpression: true,
+                        hasFormatSpecifier: false,
+                    };
+                }
+                else {
+                    appendOutputChar(curChar);
+                    strOffset++;
+                }
+                formatExpressionNestCount++;
+            }
+        }
+        else if (isFormat && curChar === 125 /* CloseBrace */) {
+            if (!formatSegment.isExpression && getEscapedCharacter(1) === 125 /* CloseBrace */) {
+                appendOutputChar(curChar);
+                strOffset += 2;
+            }
+            else if (formatExpressionNestCount === 0) {
+                output.unescapeErrors.push({
+                    offset: strOffset,
+                    length: 1,
+                    errorType: 2 /* SingleCloseBraceWithinFormatLiteral */,
+                });
+                strOffset++;
+            }
+            else {
+                formatExpressionNestCount--;
+                if (formatExpressionNestCount === 0) {
+                    // A close brace within a format expression indicates that
+                    // the expression is complete.
+                    formatSegment.length = strOffset - formatSegment.offset;
+                    output.formatStringSegments.push(formatSegment);
+                    strOffset++;
+                    // Start a new segment.
+                    formatSegment = {
+                        offset: strOffset,
+                        length: 0,
+                        valueParts: [],
+                        isExpression: false,
+                        hasFormatSpecifier: false,
+                    };
+                }
+                else {
+                    appendOutputChar(curChar);
+                    strOffset++;
+                }
+            }
+        }
+        else if (formatSegment.isExpression && (curChar === 39 /* SingleQuote */ || curChar === 34 /* DoubleQuote */)) {
+            // We're within an expression, and we've encountered a string literal.
+            // Skip over it.
+            const quoteChar = curChar;
+            appendOutputChar(curChar);
+            const isTriplicate = getEscapedCharacter(1) === quoteChar && getEscapedCharacter(2) === quoteChar;
+            if (isTriplicate) {
+                strOffset += 2;
+                appendOutputChar(curChar);
+                appendOutputChar(curChar);
+                output.valueParts.push(String.fromCharCode(curChar));
+                output.valueParts.push(String.fromCharCode(curChar));
+            }
+            while (true) {
+                strOffset++;
+                let strChar = getEscapedCharacter();
+                if (strChar === 3 /* EndOfText */) {
+                    break;
+                }
+                if (strChar === 92 /* Backslash */) {
+                    appendOutputChar(strChar);
+                    strOffset++;
+                    strChar = getEscapedCharacter();
+                    appendOutputChar(strChar);
+                    continue;
+                }
+                if (strChar === 10 /* LineFeed */ || strChar === 13 /* CarriageReturn */) {
+                    if (!isTriplicate) {
+                        break;
+                    }
+                }
+                if (strChar === quoteChar) {
+                    if (!isTriplicate) {
+                        strOffset++;
+                        appendOutputChar(strChar);
+                        break;
+                    }
+                    if (getEscapedCharacter(1) === quoteChar && getEscapedCharacter(2) === quoteChar) {
+                        strOffset += 3;
+                        appendOutputChar(strChar);
+                        appendOutputChar(strChar);
+                        appendOutputChar(strChar);
+                        break;
+                    }
+                }
+                appendOutputChar(strChar);
+            }
+        }
+        else {
+            if (formatSegment.isExpression && curChar === 58 /* Colon */) {
+                formatSegment.hasFormatSpecifier = true;
+            }
+            // There's nothing to unescape, so output the escaped character directly.
+            if (isBytes && curChar >= 128) {
+                output.nonAsciiInBytes = true;
+            }
+            appendOutputChar(curChar);
+            strOffset++;
+        }
+    }
+}
+exports.getUnescapedString = getUnescapedString;
+function _isWhitespaceChar(charCode) {
+    return charCode === 32 /* Space */ || charCode === 9 /* Tab */;
+}
+function _isAlphaNumericChar(charCode) {
+    if (charCode >= 48 /* _0 */ && charCode <= 57 /* _9 */) {
+        return true;
+    }
+    if (charCode >= 97 /* a */ && charCode <= 122 /* z */) {
+        return true;
+    }
+    if (charCode >= 65 /* A */ && charCode <= 90 /* Z */) {
+        return true;
+    }
+    return false;
+}
+function _isOctalCharCode(charCode) {
+    return charCode >= 48 /* _0 */ && charCode <= 55 /* _7 */;
+}
+function _isHexCharCode(charCode) {
+    if (charCode >= 48 /* _0 */ && charCode <= 57 /* _9 */) {
+        return true;
+    }
+    if (charCode >= 97 /* a */ && charCode <= 102 /* f */) {
+        return true;
+    }
+    if (charCode >= 65 /* A */ && charCode <= 70 /* F */) {
+        return true;
+    }
+    return false;
+}
+function _getHexDigitValue(charCode) {
+    if (charCode >= 48 /* _0 */ && charCode <= 57 /* _9 */) {
+        return charCode - 48 /* _0 */;
+    }
+    if (charCode >= 97 /* a */ && charCode <= 102 /* f */) {
+        return charCode - 97 /* a */ + 10;
+    }
+    if (charCode >= 65 /* A */ && charCode <= 70 /* F */) {
+        return charCode - 65 /* A */ + 10;
+    }
+    return 0;
+}
+//# sourceMappingURL=stringTokenUtils.js.map

package/dist/parser/stringTokenUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stringTokenUtils.js","sourceRoot":"","sources":["../../../src/parser/stringTokenUtils.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;AAgEH,SAAS,uBAAuB,CAAC,UAAqC;IAClE,OAAO;QACH,GAAG,UAAU;QACb,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACpE,GAAG,OAAO;YACV,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;SACrC,CAAC,CAAC;KACN,CAAC;AACN,CAAC;AAED,SAAgB,kBAAkB,CAAC,WAAwB;IACvD,MAAM,aAAa,GAAG,WAAW,CAAC,YAAY,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,WAAW,CAAC,KAAK,cAAuB,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,CAAC,WAAW,CAAC,KAAK,kBAA0B,CAAC,KAAK,CAAC,CAAC;IAErE,IAAI,KAAK,IAAI,CAAC,QAAQ,EAAE;QACpB,OAAO;YACH,KAAK,EAAE,aAAa;YACpB,cAAc,EAAE,EAAE;YAClB,eAAe,EAAE,KAAK;YACtB,oBAAoB,EAAE,EAAE;SAC3B,CAAC;KACL;IAED,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACvD,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;KACnD;IAED,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,KAAK,iBAAyB,CAAC,KAAK,CAAC,CAAC;IAEnE,iDAAiD;IACjD,IAAI,CAAC,QAAQ,EAAE;QACX,IACI,CAAC,SAAS,CAAC,IAAI,CACX,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,4BAAwB,IAAI,OAAO,sBAAkB,IAAI,OAAO,uBAAmB,CAC1G,EACH;YACE,OAAO;gBACH,KAAK,EAAE,aAAa;gBACpB,cAAc,EAAE,EAAE;gBAClB,eAAe,EAAE,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,GAAG,CAAC;gBACvE,oBAAoB,EAAE,EAAE;aAC3B,CAAC;SACL;KACJ;IAED,IAAI,yBAAyB,GAAG,CAAC,CAAC;IAClC,IAAI,aAAa,GAAkC;QAC/C,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,EAAE;QACd,YAAY,EAAE,KAAK;QACnB,kBAAkB,EAAE,KAAK;KAC5B,CAAC;IACF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,MAAM,GAA8B;QACtC,UAAU,EAAE,EAAE;QACd,cAAc,EAAE,EAAE;QAClB,eAAe,EAAE,KAAK;QACtB,oBAAoB,EAAE,EAAE;KAC3B,CAAC;IAEF,MAAM,sBAAsB,GAAG,GAAG,EAAE;QAChC,oDAAoD;QACpD,IAAI,CAAC,KAAK,EAAE;YACR,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC;gBACvB,MAAM,EAAE,SAAS,GAAG,CAAC;gBACrB,MAAM,EAAE,CAAC;gBACT,SAAS,+BAAyC;aACrD,CAAC,CAAC;SACN;IACL,CAAC,CAAC;IAEF,MAAM,mBAAmB,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE;QACvC,IAAI,SAAS,GAAG,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE;YACxC,yBAAsB;SACzB;QAED,OAAO,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACzC,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,EAAE;QACzC,IAAI,oBAAoB,GAAG,KAAK,CAAC;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE;YACjC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;gBAC3B,oBAAoB,GAAG,IAAI,CAAC;gBAC5B,MAAM;aACT;YACD,QAAQ,GAAG,EAAE,GAAG,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;SAC1D;QAED,IAAI,oBAAoB,EAAE;YACtB,sBAAsB,EAAE,CAAC;YACzB,UAAU,GAAG,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC,CAAC;YAC/D,SAAS,EAAE,CAAC;SACf;aAAM;YACH,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAC3C,SAAS,IAAI,CAAC,GAAG,UAAU,CAAC;SAC/B;QAED,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,EAAE;QAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC;IAEF,OAAO,IAAI,EAAE;QACT,IAAI,OAAO,GAAG,mBAAmB,EAAE,CAAC;QACpC,IAAI,OAAO,sBAAmB,EAAE;YAC5B,IAAI,QAAQ,EAAE;gBACV,IAAI,aAAa,CAAC,YAAY,EAAE;oBAC5B,0DAA0D;oBAC1D,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC;wBACvB,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,MAAM,EAAE,SAAS,GAAG,aAAa,CAAC,MAAM;wBACxC,SAAS,sCAAgD;qBAC5D,CAAC,CAAC;iBACN;gBAED,yBAAyB;gBACzB,IAAI,SAAS,KAAK,aAAa,CAAC,MAAM,EAAE;oBACpC,aAAa,CAAC,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;oBACxD,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;iBACnD;aACJ;YACD,OAAO,uBAAuB,CAAC,MAAM,CAAC,CAAC;SAC1C;QAED,IAAI,OAAO,uBAAmB,EAAE;YAC5B,IAAI,QAAQ,IAAI,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,kBAAkB,EAAE;gBAC7E,+DAA+D;gBAC/D,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,CAAC;oBACT,SAAS,sCAAgD;iBAC5D,CAAC,CAAC;aACN;YAED,8CAA8C;YAC9C,SAAS,EAAE,CAAC;YAEZ,IAAI,KAAK,EAAE;gBACP,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,SAAS;aACZ;YAED,OAAO,GAAG,mBAAmB,EAAE,CAAC;YAChC,IAAI,UAAU,GAAG,EAAE,CAAC;YAEpB,IAAI,OAAO,4BAAwB,IAAI,OAAO,sBAAkB,EAAE;gBAC9D,IAAI,OAAO,4BAAwB,IAAI,mBAAmB,CAAC,CAAC,CAAC,sBAAkB,EAAE;oBAC7E,IAAI,KAAK,EAAE;wBACP,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;qBAC9C;oBACD,SAAS,EAAE,CAAC;oBACZ,OAAO,GAAG,mBAAmB,EAAE,CAAC;iBACnC;gBACD,IAAI,KAAK,EAAE;oBACP,UAAU,GAAG,IAAI,GAAG,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;iBACjE;gBACD,SAAS,EAAE,CAAC;aACf;iBAAM;gBACH,IAAI,KAAK,EAAE;oBACP,UAAU,GAAG,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBACjD,SAAS,EAAE,CAAC;iBACf;qBAAM;oBACH,QAAQ,OAAO,EAAE;wBACb,wBAAoB;wBACpB,0BAAsB;wBACtB;4BACI,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;4BAC1C,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,QAAQ,CAAC;4BACtB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,IAAI,CAAC;4BAClB,SAAS,EAAE,CAAC;4BACZ,MAAM;wBAEV;4BACI,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;4BAC9B,MAAM;wBAEV,eAAW,CAAC,CAAC;4BACT,IAAI,gBAAgB,GAAG,KAAK,CAAC;4BAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;4BAClB,IAAI,mBAAmB,CAAC,SAAS,CAAC,wBAAmB,EAAE;gCACnD,gBAAgB,GAAG,IAAI,CAAC;6BAC3B;iCAAM;gCACH,SAAS,EAAE,CAAC;gCACZ,OAAO,IAAI,EAAE;oCACT,MAAM,aAAa,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;oCACrD,IAAI,aAAa,yBAAoB,EAAE;wCACnC,MAAM;qCACT;yCAAM,IACH,CAAC,mBAAmB,CAAC,aAAa,CAAC;wCACnC,aAAa,oBAAgB;wCAC7B,CAAC,iBAAiB,CAAC,aAAa,CAAC,EACnC;wCACE,gBAAgB,GAAG,IAAI,CAAC;wCACxB,MAAM;qCACT;yCAAM;wCACH,SAAS,EAAE,CAAC;qCACf;iCACJ;6BACJ;4BAED,IAAI,gBAAgB,EAAE;gCAClB,sBAAsB,EAAE,CAAC;gCACzB,UAAU,GAAG,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gCACjD,SAAS,EAAE,CAAC;6BACf;iCAAM;gCACH,oDAAoD;gCACpD,mDAAmD;gCACnD,yBAAyB;gCACzB,UAAU,GAAG,GAAG,CAAC;gCACjB,SAAS,IAAI,CAAC,GAAG,SAAS,CAAC;6BAC9B;4BACD,MAAM;yBACT;wBAED;4BACI,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;4BAC9B,MAAM;wBAEV;4BACI,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;4BAC9B,MAAM;wBAEV;4BACI,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE;gCAC3B,IAAI,SAAS,GAAG,OAAO,cAAU,CAAC;gCAClC,SAAS,EAAE,CAAC;gCACZ,OAAO,GAAG,mBAAmB,EAAE,CAAC;gCAChC,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE;oCAC3B,SAAS,GAAG,SAAS,GAAG,CAAC,GAAG,OAAO,cAAU,CAAC;oCAC9C,SAAS,EAAE,CAAC;oCACZ,OAAO,GAAG,mBAAmB,EAAE,CAAC;oCAEhC,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE;wCAC3B,SAAS,GAAG,SAAS,GAAG,CAAC,GAAG,OAAO,cAAU,CAAC;wCAC9C,SAAS,EAAE,CAAC;qCACf;iCACJ;gCAED,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;6BAC/C;iCAAM;gCACH,UAAU,GAAG,IAAI,CAAC;gCAClB,sBAAsB,EAAE,CAAC;6BAC5B;4BACD,MAAM;qBACb;iBACJ;aACJ;YAED,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACnC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;SAC7C;aAAM,IAAI,OAAO,sBAAkB,IAAI,OAAO,4BAAwB,EAAE;YACrE,iEAAiE;YACjE,IAAI,OAAO,4BAAwB,IAAI,mBAAmB,CAAC,CAAC,CAAC,sBAAkB,EAAE;gBAC7E,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,SAAS,EAAE,CAAC;gBACZ,OAAO,GAAG,mBAAmB,EAAE,CAAC;aACnC;YAED,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1B,SAAS,EAAE,CAAC;SACf;aAAM,IAAI,QAAQ,IAAI,OAAO,wBAAmB,EAAE;YAC/C,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,mBAAmB,CAAC,CAAC,CAAC,wBAAmB,EAAE;gBAC1E,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,SAAS,IAAI,CAAC,CAAC;aAClB;iBAAM;gBACH,IAAI,yBAAyB,KAAK,CAAC,EAAE;oBACjC,6DAA6D;oBAC7D,6BAA6B;oBAC7B,aAAa,CAAC,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;oBACxD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC1B,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;qBACnD;oBACD,SAAS,EAAE,CAAC;oBAEZ,uBAAuB;oBACvB,aAAa,GAAG;wBACZ,MAAM,EAAE,SAAS;wBACjB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,EAAE;wBACd,YAAY,EAAE,IAAI;wBAClB,kBAAkB,EAAE,KAAK;qBAC5B,CAAC;iBACL;qBAAM;oBACH,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1B,SAAS,EAAE,CAAC;iBACf;gBACD,yBAAyB,EAAE,CAAC;aAC/B;SACJ;aAAM,IAAI,QAAQ,IAAI,OAAO,yBAAoB,EAAE;YAChD,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,mBAAmB,CAAC,CAAC,CAAC,yBAAoB,EAAE;gBAC3E,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,SAAS,IAAI,CAAC,CAAC;aAClB;iBAAM,IAAI,yBAAyB,KAAK,CAAC,EAAE;gBACxC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC;oBACvB,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,CAAC;oBACT,SAAS,6CAAuD;iBACnE,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC;aACf;iBAAM;gBACH,yBAAyB,EAAE,CAAC;gBAE5B,IAAI,yBAAyB,KAAK,CAAC,EAAE;oBACjC,0DAA0D;oBAC1D,8BAA8B;oBAC9B,aAAa,CAAC,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;oBACxD,MAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAChD,SAAS,EAAE,CAAC;oBAEZ,uBAAuB;oBACvB,aAAa,GAAG;wBACZ,MAAM,EAAE,SAAS;wBACjB,MAAM,EAAE,CAAC;wBACT,UAAU,EAAE,EAAE;wBACd,YAAY,EAAE,KAAK;wBACnB,kBAAkB,EAAE,KAAK;qBAC5B,CAAC;iBACL;qBAAM;oBACH,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1B,SAAS,EAAE,CAAC;iBACf;aACJ;SACJ;aAAM,IAAI,aAAa,CAAC,YAAY,IAAI,CAAC,OAAO,yBAAqB,IAAI,OAAO,yBAAqB,CAAC,EAAE;YACrG,sEAAsE;YACtE,gBAAgB;YAChB,MAAM,SAAS,GAAG,OAAO,CAAC;YAC1B,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1B,MAAM,YAAY,GAAG,mBAAmB,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,mBAAmB,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC;YAClG,IAAI,YAAY,EAAE;gBACd,SAAS,IAAI,CAAC,CAAC;gBACf,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC1B,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;aACxD;YAED,OAAO,IAAI,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,IAAI,OAAO,GAAG,mBAAmB,EAAE,CAAC;gBACpC,IAAI,OAAO,sBAAmB,EAAE;oBAC5B,MAAM;iBACT;gBAED,IAAI,OAAO,uBAAmB,EAAE;oBAC5B,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1B,SAAS,EAAE,CAAC;oBACZ,OAAO,GAAG,mBAAmB,EAAE,CAAC;oBAChC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1B,SAAS;iBACZ;gBAED,IAAI,OAAO,sBAAkB,IAAI,OAAO,4BAAwB,EAAE;oBAC9D,IAAI,CAAC,YAAY,EAAE;wBACf,MAAM;qBACT;iBACJ;gBAED,IAAI,OAAO,KAAK,SAAS,EAAE;oBACvB,IAAI,CAAC,YAAY,EAAE;wBACf,SAAS,EAAE,CAAC;wBACZ,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1B,MAAM;qBACT;oBAED,IAAI,mBAAmB,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,mBAAmB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE;wBAC9E,SAAS,IAAI,CAAC,CAAC;wBACf,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1B,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1B,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1B,MAAM;qBACT;iBACJ;gBAED,gBAAgB,CAAC,OAAO,CAAC,CAAC;aAC7B;SACJ;aAAM;YACH,IAAI,aAAa,CAAC,YAAY,IAAI,OAAO,mBAAe,EAAE;gBACtD,aAAa,CAAC,kBAAkB,GAAG,IAAI,CAAC;aAC3C;YAED,yEAAyE;YACzE,IAAI,OAAO,IAAI,OAAO,IAAI,GAAG,EAAE;gBAC3B,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC;aACjC;YAED,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1B,SAAS,EAAE,CAAC;SACf;KACJ;AACL,CAAC;AAxaD,gDAwaC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACvC,OAAO,QAAQ,mBAAe,IAAI,QAAQ,gBAAa,CAAC;AAC5D,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB;IACzC,IAAI,QAAQ,eAAW,IAAI,QAAQ,eAAW,EAAE;QAC5C,OAAO,IAAI,CAAC;KACf;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,eAAU,EAAE;QAC1C,OAAO,IAAI,CAAC;KACf;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,cAAU,EAAE;QAC1C,OAAO,IAAI,CAAC;KACf;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACtC,OAAO,QAAQ,eAAW,IAAI,QAAQ,eAAW,CAAC;AACtD,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACpC,IAAI,QAAQ,eAAW,IAAI,QAAQ,eAAW,EAAE;QAC5C,OAAO,IAAI,CAAC;KACf;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,eAAU,EAAE;QAC1C,OAAO,IAAI,CAAC;KACf;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,cAAU,EAAE;QAC1C,OAAO,IAAI,CAAC;KACf;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IACvC,IAAI,QAAQ,eAAW,IAAI,QAAQ,eAAW,EAAE;QAC5C,OAAO,QAAQ,cAAU,CAAC;KAC7B;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,eAAU,EAAE;QAC1C,OAAO,QAAQ,aAAS,GAAG,EAAE,CAAC;KACjC;IAED,IAAI,QAAQ,cAAU,IAAI,QAAQ,cAAU,EAAE;QAC1C,OAAO,QAAQ,aAAS,GAAG,EAAE,CAAC;KACjC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}

package/dist/parser/tokenizer.d.ts

@@ -0,0 +1,71 @@
+import { TextRange } from '../common/textRange';
+import { TextRangeCollection } from '../common/textRangeCollection';
+import { OperatorFlags, OperatorType, Token } from './tokenizerTypes';
+export declare const defaultTabSize = 8;
+export interface TokenizerOutput {
+    tokens: TextRangeCollection<Token>;
+    lines: TextRangeCollection<TextRange>;
+    typeIgnoreLines: Map<number, IgnoreComment>;
+    pyrightIgnoreLines: Map<number, IgnoreComment>;
+    typeIgnoreAll: IgnoreComment | undefined;
+    predominantEndOfLineSequence: string;
+    predominantTabSequence: string;
+    predominantSingleQuoteCharacter: string;
+}
+export interface IgnoreCommentRule {
+    text: string;
+    range: TextRange;
+}
+export interface IgnoreComment {
+    range: TextRange;
+    rulesList: IgnoreCommentRule[] | undefined;
+}
+export declare class Tokenizer {
+    private _cs;
+    private _tokens;
+    private _prevLineStart;
+    private _parenDepth;
+    private _lineRanges;
+    private _indentAmounts;
+    private _typeIgnoreAll;
+    private _typeIgnoreLines;
+    private _pyrightIgnoreLines;
+    private _comments;
+    private _crCount;
+    private _crLfCount;
+    private _lfCount;
+    private _indentCount;
+    private _indentTabCount;
+    private _indentSpacesTotal;
+    private _singleQuoteCount;
+    private _doubleQuoteCount;
+    private _ipythonMode;
+    tokenize(text: string, start?: number, length?: number, initialParenDepth?: number, ipythonMode?: boolean): TokenizerOutput;
+    static getOperatorInfo(operatorType: OperatorType): OperatorFlags;
+    static isOperatorAssignment(operatorType?: OperatorType): boolean;
+    static isOperatorComparison(operatorType?: OperatorType): boolean;
+    private _addNextToken;
+    private _handleCharacter;
+    private _addLineRange;
+    private _handleNewLine;
+    private _readIndentationAfterNewLine;
+    private _setIndent;
+    private _tryIdentifier;
+    private _isPossibleNumber;
+    private _tryNumber;
+    private _tryOperator;
+    private _handleInvalid;
+    private _getComments;
+    private _isIPythonMagics;
+    private _handleIPythonMagics;
+    private _handleComment;
+    private _getIgnoreCommentRulesList;
+    private _addComments;
+    private _getStringPrefixLength;
+    private _getQuoteTypeFlags;
+    private _handleString;
+    private _skipToEndOfStringLiteral;
+    private _skipFloatingPointCandidate;
+    private _skipFractionalNumber;
+    private _skipDecimalNumber;
+}