@zzzen/pyright-internal 1.2.0-dev.20230430 → 1.2.0-dev.20230514

package/dist/languageService/indentationUtils.js

@@ -1,727 +0,0 @@
-"use strict";
-/*
- * indentationUtils.ts
- * Copyright (c) Microsoft Corporation.
- * Licensed under the MIT license.
- *
- * Provides code to get indentation and re-indent code for the
- * given indentation.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createIndentationString = exports.getIndentationFromText = exports.findNonWhitespaceTokenAtOrBeforeOffset = exports.getModuleStatementIndentation = exports.reindentSpan = exports.getNewlineIndentation = void 0;
-const parseTreeUtils_1 = require("../analyzer/parseTreeUtils");
-const collectionUtils_1 = require("../common/collectionUtils");
-const positionUtils_1 = require("../common/positionUtils");
-const textRange_1 = require("../common/textRange");
-const tokenizer_1 = require("../parser/tokenizer");
-const insertionPointUtils_1 = require("./insertionPointUtils");
-function getNewlineIndentation(parseResults, newlineOffset, preferDedent) {
-    // ex)
-    // a = """
-    //      | <= here
-    const strIndent = _tryHandleStringLiterals(parseResults, newlineOffset);
-    if (strIndent !== undefined) {
-        return strIndent;
-    }
-    // ex)
-    // a = 1 + \
-    //     | <= here
-    // or
-    // a = (1 +
-    //     | <= here
-    const exprIndent = _tryHandleMultilineConstructs(parseResults, newlineOffset);
-    if (exprIndent !== undefined) {
-        return exprIndent;
-    }
-    preferDedent = preferDedent !== null && preferDedent !== void 0 ? preferDedent : _shouldDedentAfterKeyword(parseResults, newlineOffset);
-    return Math.max(_getIndentation(parseResults, newlineOffset, preferDedent).indentation, 0);
-}
-exports.getNewlineIndentation = getNewlineIndentation;
-function reindentSpan(parseResults, span, indentation, indentFirstToken = true) {
-    let indentDelta = 0;
-    const texts = [];
-    // Currently _convertTokenStreams converts text in the span as whitespace and non whitespace
-    // and then this function puts those back to string with reindentation if needed.
-    //
-    // Another approach we can take is converting the text in 2 chunks that require reindentation and not
-    // and process chunks that require reindentation line by line (like how it currently does for
-    // multiline doc comments) and put chunks that don't require reindentation as it is.
-    const tokenInfo = _convertTokenStreams(parseResults, span);
-    let previousInfo = tokenInfo[0];
-    indentDelta =
-        indentation -
-            getIndentationFromText(parseResults, previousInfo.range.start.line, previousInfo.range.start.character)
-                .indentation;
-    if (previousInfo.multilineDocComment) {
-        (0, collectionUtils_1.appendArray)(texts, _reindentLinesFromText(parseResults, previousInfo, indentDelta));
-    }
-    else {
-        if (indentFirstToken) {
-            texts.push(createIndentationString(parseResults, indentation));
-        }
-        texts.push(previousInfo.text);
-    }
-    for (let i = 1; i < tokenInfo.length; i++) {
-        const info = tokenInfo[i];
-        if (info.firstTokenOnLine) {
-            texts.push(parseResults.tokenizerOutput.predominantEndOfLineSequence.repeat(info.range.start.line - previousInfo.range.end.line));
-            if (info.multilineDocComment) {
-                (0, collectionUtils_1.appendArray)(texts, _reindentLinesFromText(parseResults, info, indentDelta));
-            }
-            else {
-                // Put indentation for the first token on the line.
-                texts.push(createIndentationString(parseResults, Math.max(0, getIndentationFromText(parseResults, info.range.start.line, info.range.start.character)
-                    .indentation + indentDelta)));
-                texts.push(info.text);
-            }
-        }
-        else {
-            // Put whitespace between 2 tokens on same line
-            // token1[space]token2
-            texts.push(' '.repeat(info.range.start.character - previousInfo.range.end.character));
-            texts.push(info.text);
-        }
-        previousInfo = info;
-    }
-    return {
-        originalSpan: textRange_1.TextRange.combine(tokenInfo),
-        text: texts.join(''),
-    };
-}
-exports.reindentSpan = reindentSpan;
-function getModuleStatementIndentation(parseResults) {
-    if (parseResults.parseTree.statements.length === 0) {
-        return getNewlineIndentation(parseResults, parseResults.parseTree.length, /* preferDedent */ true);
-    }
-    return getNewlineIndentation(parseResults, parseResults.parseTree.statements[0].start, /* preferDedent */ true);
-}
-exports.getModuleStatementIndentation = getModuleStatementIndentation;
-function _getIndentation(parseResults, offset, preferDedent) {
-    const tokens = parseResults.tokenizerOutput.tokens;
-    const startingToken = findNonWhitespaceTokenAtOrBeforeOffset(tokens, offset);
-    if (!startingToken) {
-        return {
-            indentation: 0,
-        };
-    }
-    const node = (0, parseTreeUtils_1.findNodeByOffset)(parseResults.parseTree, textRange_1.TextRange.getEnd(startingToken));
-    if (!node) {
-        return {
-            indentation: 0,
-        };
-    }
-    // Special-case the match statement since it does not contain a suite. Case statements do,
-    // but match does not.
-    if (node.nodeType === 63 /* Match */) {
-        const tabSize = _getTabSize(parseResults);
-        const outerContainer = (0, insertionPointUtils_1.getContainer)(node, /* includeSelf */ false);
-        const result = _getIndentationForNode(parseResults, offset, outerContainer !== null && outerContainer !== void 0 ? outerContainer : parseResults.parseTree, node);
-        result.indentation += tabSize;
-        return result;
-    }
-    const suite = (0, parseTreeUtils_1.getFirstAncestorOrSelfOfKind)(node, 50 /* Suite */);
-    if (!suite) {
-        return _getIndentationForNode(parseResults, offset, parseResults.parseTree, node);
-    }
-    const suiteSpan = (0, positionUtils_1.convertTextRangeToRange)(suite, parseResults.tokenizerOutput.lines);
-    if (preferDedent || (suiteSpan.start.line === suiteSpan.end.line && suite.statements.length > 0)) {
-        // Go one more level up.
-        const outerContainer = (0, insertionPointUtils_1.getContainer)(suite, /* includeSelf */ false);
-        return _getIndentationForNode(parseResults, offset, outerContainer !== null && outerContainer !== void 0 ? outerContainer : parseResults.parseTree, suite);
-    }
-    return _getIndentationForNode(parseResults, offset, suite, node);
-}
-function _getIndentationForNode(parseResults, offset, container, current) {
-    if (container.nodeType === 36 /* Module */) {
-        // It is at the module level
-        return {
-            token: _getFirstTokenOfStatement(parseResults, container, current),
-            indentation: 0,
-        };
-    }
-    if (container.nodeType === 63 /* Match */ ||
-        _containsNoIndentBeforeFirstStatement(parseResults, offset, container)) {
-        const tabSize = _getTabSize(parseResults);
-        const outerContainer = (0, insertionPointUtils_1.getContainer)(container, /* includeSelf */ false);
-        const result = _getIndentationForNode(parseResults, offset, outerContainer !== null && outerContainer !== void 0 ? outerContainer : parseResults.parseTree, container);
-        return {
-            token: result.token,
-            indentation: result.indentation + tabSize,
-        };
-    }
-    else {
-        const tokens = parseResults.tokenizerOutput.tokens;
-        return {
-            token: _getFirstTokenOfStatement(parseResults, container, current),
-            indentation: _getIndentationFromIndentToken(tokens, tokens.getItemAtPosition(container.start)),
-        };
-    }
-}
-function _containsNoIndentBeforeFirstStatement(parseResults, offset, suite) {
-    const statements = suite.statements.filter((s) => s.length > 0);
-    if (statements.length === 0) {
-        // There is no statement in the suite.
-        // ex)
-        // def foo():
-        // | <= here
-        return true;
-    }
-    if (statements.length === 1) {
-        if (statements[0].nodeType !== 47 /* StatementList */ || statements[0].statements.length === 1) {
-            if (statements[0].start >= offset) {
-                const statementLine = parseResults.tokenizerOutput.lines.getItemAtPosition(statements[0].start);
-                const offsetLine = parseResults.tokenizerOutput.lines.getItemAtPosition(offset);
-                if (statementLine === offsetLine) {
-                    // We are calculating indent for only statement in suite.
-                    // ex)
-                    // def foo():
-                    //     |pass <= offset before first statement
-                    return true;
-                }
-            }
-        }
-    }
-    // If suite contains no indent before first statement, then consider user is in the middle of writing block
-    // and parser is in broken state.
-    // ex)
-    // def foo():
-    //     while True:
-    //     | <= here
-    // def bar():
-    //     pass
-    //
-    // parser will think "def bar" belongs to "while True" with invalid indentation.
-    const tokens = parseResults.tokenizerOutput.tokens;
-    const start = tokens.getItemAtPosition(suite.start);
-    const end = tokens.getItemAtPosition(suite.statements[0].start);
-    for (let i = start; i <= end; i++) {
-        const token = _getTokenAtIndex(tokens, i);
-        if ((token === null || token === void 0 ? void 0 : token.type) === 3 /* Indent */) {
-            return false;
-        }
-    }
-    return true;
-}
-function _getFirstTokenOfStatement(parseResults, container, span) {
-    const tokens = parseResults.tokenizerOutput.tokens;
-    for (const statement of container.statements) {
-        if (!textRange_1.TextRange.containsRange(statement, span)) {
-            continue;
-        }
-        return (0, parseTreeUtils_1.getTokenAt)(tokens, statement.start);
-    }
-    return (0, parseTreeUtils_1.getTokenAt)(tokens, container.start);
-}
-function _getIndentationFromIndentToken(tokens, index) {
-    for (let i = index; i < tokens.count; i++) {
-        const token = _getTokenAtIndex(tokens, i);
-        if ((token === null || token === void 0 ? void 0 : token.type) === 3 /* Indent */) {
-            return token.indentAmount;
-        }
-    }
-    // At the module level.
-    return 0;
-}
-function _tryHandleMultilineConstructs(parseResults, offset) {
-    const tokens = parseResults.tokenizerOutput.tokens;
-    const lines = parseResults.tokenizerOutput.lines;
-    // Make sure we use next token to get line delta.
-    // This is just to handle how tokenizer associates new lines to which token.
-    // ex) a = 1 + \
-    //         | <= here
-    //    [b] = 2
-    const index = _findNextTokenIndex(tokens, offset);
-    if (index < 0) {
-        return undefined;
-    }
-    for (let i = index; i > 0; i--) {
-        const token = _getTokenAtIndex(tokens, i);
-        if (textRange_1.TextRange.getEnd(token) < offset) {
-            return undefined;
-        }
-        const previousToken = _getTokenAtIndex(tokens, i - 1);
-        const tokenSpan = token ? (0, positionUtils_1.convertTextRangeToRange)(token, lines) : undefined;
-        const previousTokenSpan = previousToken ? (0, positionUtils_1.convertTextRangeToRange)(previousToken, lines) : undefined;
-        if (tokenSpan &&
-            previousTokenSpan &&
-            previousTokenSpan.start.line < tokenSpan.start.line &&
-            previousToken.type !== 2 /* NewLine */) {
-            return _getIndentationForNextLine(parseResults, previousToken, token, offset);
-        }
-    }
-    return undefined;
-}
-function _tryHandleStringLiterals(parseResults, offset) {
-    const tokens = parseResults.tokenizerOutput.tokens;
-    const index = tokens.getItemAtPosition(offset);
-    if (index < 0) {
-        return undefined;
-    }
-    const token = _findStringToken(tokens, index);
-    if (!token || token.type !== 5 /* String */) {
-        return undefined;
-    }
-    const stringToken = token;
-    if (!(stringToken.flags & 4 /* Triplicate */)) {
-        // We only care """ string literal
-        return undefined;
-    }
-    if (!(stringToken.flags & 65536 /* Unterminated */) &&
-        !textRange_1.TextRange.contains((0, parseTreeUtils_1.getStringValueRange)(stringToken), offset)) {
-        // ex) We only support these 2 cases.
-        //     """
-        //     | <= here
-        //     or
-        //     """
-        //     | <= here
-        //     """
-        return undefined;
-    }
-    const lines = parseResults.tokenizerOutput.lines;
-    const begin = (0, positionUtils_1.convertOffsetToPosition)(token.start, lines);
-    const current = (0, positionUtils_1.convertOffsetToPosition)(offset, lines);
-    return _getFirstNonBlankLineIndentationFromText(parseResults, current.line, begin.line);
-}
-function _isOpenToken(token) {
-    return (token.type === 13 /* OpenParenthesis */ ||
-        token.type === 15 /* OpenBracket */ ||
-        token.type === 17 /* OpenCurlyBrace */);
-}
-function _isCloseToken(token) {
-    return (token.type === 14 /* CloseParenthesis */ ||
-        token.type === 16 /* CloseBracket */ ||
-        token.type === 18 /* CloseCurlyBrace */);
-}
-function _getIndentationForNextLine(parseResults, prevToken, nextToken, offset) {
-    // Get the last token on the same line as the previous token
-    const lines = parseResults.tokenizerOutput.lines;
-    const lineIndex = (0, positionUtils_1.convertOffsetToPosition)(prevToken.start, lines).line;
-    const line = lines.getItemAt(lineIndex);
-    const tabSize = _getTabSize(parseResults);
-    let token = prevToken;
-    // Go backwards through tokens up until the front of the line
-    let whitespaceOnly = true;
-    let closeCount = 0;
-    while (token && token.start >= line.start) {
-        if (_isCloseToken(token)) {
-            whitespaceOnly = false;
-            closeCount += 1;
-        }
-        else if (_isOpenToken(token) && closeCount === 0) {
-            // Special case for parenthesis
-            if (token.type === 13 /* OpenParenthesis */ && whitespaceOnly) {
-                const baseIndentation = _getIndentation(parseResults, token.start, false).indentation;
-                // In PEP 8, this should be this case here:
-                // # Add 4 spaces (an extra level of indentation) to distinguish arguments from the rest.
-                // def long_function_name(
-                //         var_one, var_two, var_three,
-                //         var_four):
-                //     print(var_one)
-                //
-                const node = (0, parseTreeUtils_1.findNodeByOffset)(parseResults.parseTree, token.start - 1);
-                const funcNode = (0, parseTreeUtils_1.getFirstAncestorOrSelfOfKind)(node, 28 /* Function */);
-                if (funcNode &&
-                    funcNode.nodeType === 28 /* Function */ &&
-                    (0, positionUtils_1.convertOffsetToPosition)(funcNode.start, lines).line === lineIndex) {
-                    return baseIndentation + tabSize * 2;
-                }
-                // Not inside a function, just need one tab. See this in PEP 8
-                // # Hanging indents should add a level.
-                // foo = long_function_name(
-                //     var_one, var_two,
-                //     var_three, var_four)
-                return baseIndentation + tabSize;
-            }
-            else if (whitespaceOnly) {
-                // This is the case where the user put a newline right after a (, [, or {. We want
-                // to be one tab over from the [.
-                const line = (0, positionUtils_1.convertOffsetToPosition)(token.start, lines).line;
-                return getIndentationFromText(parseResults, line).indentation + tabSize;
-            }
-            else {
-                // In PEP 8, this should be this case here:
-                // # Aligned with opening delimiter.
-                // def long_function_name(var_one, var_two,
-                //                        var_three, var_four)
-                // + 1 is to accommodate for the parenthesis.
-                //
-                // This same indentation applies for function calls or just dictionary or list assignments.
-                return token.start - line.start + 1;
-            }
-        }
-        else if (_isOpenToken(token) && closeCount > 0) {
-            closeCount--;
-            whitespaceOnly = false;
-        }
-        else if (!_isWhitespaceToken(token.type)) {
-            // Found a non whitespace token before we returned.
-            whitespaceOnly = false;
-        }
-        token = findNonWhitespaceTokenAtOrBeforeOffset(parseResults.tokenizerOutput.tokens, token.start - 1);
-    }
-    // No parenthesis found
-    return _getFirstNonBlankLineIndentationFromText(parseResults, (0, positionUtils_1.convertOffsetToPosition)(offset, parseResults.tokenizerOutput.lines).line, lineIndex);
-}
-function _getFirstNonBlankLineIndentationFromText(parseResults, currentLine, endingLine) {
-    endingLine = Math.max(endingLine, 0);
-    for (let i = currentLine; i >= endingLine; i--) {
-        const result = getIndentationFromText(parseResults, i);
-        if (!_isBlankLine(parseResults, i, result.charOffset)) {
-            // Not blank line.
-            // ex) [indentation]i = 1
-            return result.indentation;
-        }
-    }
-    return getIndentationFromText(parseResults, endingLine).indentation;
-}
-function _findStringToken(tokens, index) {
-    const token = _findNonWhitespaceTokenAtOrBeforeIndex(tokens, index);
-    if (!token) {
-        return undefined;
-    }
-    return token.type === 5 /* String */ ? token : undefined;
-}
-function findNonWhitespaceTokenAtOrBeforeOffset(tokens, offset) {
-    const index = tokens.getItemAtPosition(offset);
-    if (index < 0) {
-        return undefined;
-    }
-    return _findNonWhitespaceTokenAtOrBeforeIndex(tokens, index);
-}
-exports.findNonWhitespaceTokenAtOrBeforeOffset = findNonWhitespaceTokenAtOrBeforeOffset;
-function _findNonWhitespaceTokenAtOrBeforeIndex(tokens, index) {
-    for (let i = index; i >= 0; i--) {
-        const token = _getTokenAtIndex(tokens, i);
-        if (!token) {
-            break;
-        }
-        if (_isWhitespaceToken(token.type)) {
-            continue;
-        }
-        return token;
-    }
-    return undefined;
-}
-function _findNextTokenIndex(tokens, offset) {
-    const index = tokens.getItemAtPosition(offset);
-    if (index < 0) {
-        return index;
-    }
-    for (let i = index + 1; i < tokens.count; i++) {
-        const token = _getTokenAtIndex(tokens, i);
-        if ((token === null || token === void 0 ? void 0 : token.type) === 4 /* Dedent */ || (token === null || token === void 0 ? void 0 : token.type) === 2 /* NewLine */) {
-            continue;
-        }
-        return i;
-    }
-    return tokens.count - 1;
-}
-function _getTokenAtIndex(tokens, index) {
-    if (index < 0) {
-        return undefined;
-    }
-    return tokens.getItemAt(index);
-}
-function _shouldDedentAfterKeyword(parseResults, offset) {
-    // For now, we won't include all small statements that can put at single line.
-    // See parser.ts to see all small statements or see python grammar.
-    // ex) def foo(): pass
-    const tokens = parseResults.tokenizerOutput.tokens;
-    const index = tokens.getItemAtPosition(offset);
-    if (index < 0) {
-        return false;
-    }
-    for (let i = index; i >= 0; i--) {
-        const token = _getTokenAtIndex(tokens, i);
-        if (!token) {
-            return false;
-        }
-        switch (token.type) {
-            case 4 /* Dedent */:
-            case 2 /* NewLine */:
-            case 1 /* EndOfStream */:
-                continue;
-            case 8 /* Keyword */: {
-                const previousToken = _getTokenAtIndex(tokens, i - 1);
-                if ((previousToken === null || previousToken === void 0 ? void 0 : previousToken.type) === 10 /* Colon */) {
-                    // Not for single line construct.
-                    // ex) def foo(): pass
-                    return false;
-                }
-                const keyword = token;
-                // Dedent if we found one of these keywords
-                if (keyword.keywordType === 30 /* Pass */ ||
-                    keyword.keywordType === 32 /* Return */ ||
-                    keyword.keywordType === 5 /* Break */ ||
-                    keyword.keywordType === 8 /* Continue */ ||
-                    keyword.keywordType === 31 /* Raise */) {
-                    return true;
-                }
-                // Otherwise, unless the keyword can be used as a return/raise value, don't dedent.
-                if (keyword.keywordType !== 33 /* True */ &&
-                    keyword.keywordType !== 15 /* False */ &&
-                    keyword.keywordType !== 26 /* None */ &&
-                    keyword.keywordType !== 9 /* Debug */) {
-                    return false;
-                }
-            }
-        }
-        // Dedent if we've found a return or raise statement
-        const node = (0, parseTreeUtils_1.findNodeByOffset)(parseResults.parseTree, token.start);
-        const returnOrRaise = (0, parseTreeUtils_1.getFirstAncestorOrSelf)(node, (x) => x.nodeType === 44 /* Return */ || x.nodeType === 43 /* Raise */);
-        return !!returnOrRaise;
-    }
-    return false;
-}
-function _isBlankLine(parseResults, line, charOffset) {
-    const endingLength = _getLineEndingLength(parseResults, line);
-    const lineSpan = parseResults.tokenizerOutput.lines.getItemAt(line);
-    return charOffset === lineSpan.length - endingLength;
-}
-function _getLineEndingLength(parseResults, line) {
-    let length = 0;
-    const range = parseResults.tokenizerOutput.lines.getItemAt(line);
-    for (let i = range.length - 1; i >= 0; i--) {
-        const charCode = parseResults.text.charCodeAt(range.start + i);
-        switch (charCode) {
-            case 12 /* FormFeed */:
-            case 35 /* Hash */:
-            case 10 /* LineFeed */:
-            case 13 /* CarriageReturn */:
-                length++;
-                break;
-            default:
-                return length;
-        }
-    }
-    return length;
-}
-function getIndentationFromText(parseResults, line, uptoLineOffset) {
-    let indentation = 0;
-    let charOffset = 0;
-    const tabSize = _getTabSize(parseResults);
-    const range = parseResults.tokenizerOutput.lines.getItemAt(line);
-    for (let i = 0; i < range.length; i++) {
-        const charCode = parseResults.text.charCodeAt(range.start + i);
-        switch (charCode) {
-            case 32 /* Space */:
-                charOffset++;
-                indentation++;
-                break;
-            case 9 /* Tab */:
-                charOffset++;
-                indentation += tabSize;
-                break;
-            default:
-                if (!uptoLineOffset || uptoLineOffset === i) {
-                    return {
-                        charOffset,
-                        indentation,
-                    };
-                }
-                // calculate indentation upto line offset given.
-                charOffset++;
-                indentation++;
-        }
-    }
-    return {
-        charOffset,
-        indentation,
-    };
-}
-exports.getIndentationFromText = getIndentationFromText;
-function _convertTokenStreams(parseResults, span) {
-    // Existing token stream contains text and whitespace mixed, making it difficult
-    // to process for re-indentation. This will convert those to strictly text and whitespace.
-    const tokens = parseResults.tokenizerOutput.tokens;
-    let startIndex = Math.max(tokens.getItemAtPosition(span.start), 0);
-    const startToken = _getTokenAtIndex(tokens, startIndex);
-    if (textRange_1.TextRange.getEnd(startToken) < span.start) {
-        // ex) firstToken | <= span start.
-        startIndex++;
-    }
-    let endIndex = Math.min(tokens.getItemAtPosition(textRange_1.TextRange.getEnd(span)), tokens.length - 1);
-    const endToken = _getTokenAtIndex(tokens, endIndex);
-    if (textRange_1.TextRange.getEnd(span) <= endToken.start) {
-        // ex) |< = span end [endToken]
-        endIndex--;
-    }
-    // If the next token is a newline, we want to include it in the range
-    // so that we can include comments if there is any.
-    if (endIndex < tokens.length - 1 && _isWhitespaceToken(tokens.getItemAt(endIndex + 1).type)) {
-        endIndex++;
-    }
-    const tokenInfoArray = [];
-    const lines = parseResults.tokenizerOutput.lines;
-    for (let i = startIndex; i <= endIndex; i++) {
-        const token = _getTokenAtIndex(tokens, i);
-        if (token.comments) {
-            for (const comment of token.comments) {
-                tokenInfoArray.push({
-                    start: comment.start,
-                    length: comment.length,
-                    range: (0, positionUtils_1.convertTextRangeToRange)(comment, lines),
-                    text: comment.value,
-                    kind: 'comment',
-                    firstTokenOnLine: false,
-                    multilineDocComment: false,
-                });
-            }
-        }
-        if (_isWhitespaceToken(token.type) || token.length === 0) {
-            continue;
-        }
-        tokenInfoArray.push({
-            start: token.start,
-            length: token.length,
-            range: (0, positionUtils_1.convertTextRangeToRange)(token, lines),
-            text: parseResults.text.substr(token.start, token.length),
-            kind: token.type === 5 /* String */ ? 'string' : 'token',
-            firstTokenOnLine: false,
-            multilineDocComment: false,
-        });
-    }
-    if (tokenInfoArray.length === 0) {
-        return tokenInfoArray;
-    }
-    tokenInfoArray.sort((a, b) => a.start - b.start);
-    // Handle text in whitespace that is not part of token stream.
-    let previousInfo = tokenInfoArray[0];
-    const additionalTokens = [];
-    if (previousInfo.kind === 'comment') {
-        // ex) token [#] comment
-        const start = startIndex === 0 ? 0 : textRange_1.TextRange.getEnd(_getTokenAtIndex(tokens, startIndex - 1));
-        _addTokenInfoIfMatch(parseResults, start, previousInfo.start, 35 /* Hash */, additionalTokens);
-    }
-    for (let i = 1; i < tokenInfoArray.length; i++) {
-        const info = tokenInfoArray[i];
-        // Another approach is just blindly go through the range looking for
-        // non whitespace char rather than looking for specific cases like below.
-        if (previousInfo.kind !== 'comment') {
-            for (let whitespaceLine = previousInfo.range.end.line; whitespaceLine < info.range.start.line; whitespaceLine++) {
-                const lineTextRange = lines.getItemAt(whitespaceLine);
-                const lastCharOffset = lineTextRange.length - _getLineEndingLength(parseResults, whitespaceLine) - 1;
-                if (lastCharOffset >= 0) {
-                    // ex) i = 1 \ <= explicit multiline construct
-                    //           +
-                    //             2
-                    const start = lineTextRange.start + lastCharOffset;
-                    _addTokenInfoIfMatch(parseResults, start, start + 1, 92 /* Backslash */, additionalTokens);
-                }
-            }
-        }
-        if (info.kind === 'comment') {
-            const start = previousInfo.range.end.line === info.range.start.line
-                ? textRange_1.TextRange.getEnd(previousInfo)
-                : lines.getItemAt(info.range.start.line).start;
-            // ex) token [#] comment
-            _addTokenInfoIfMatch(parseResults, start, info.start, 35 /* Hash */, additionalTokens);
-        }
-        previousInfo = info;
-    }
-    (0, collectionUtils_1.appendArray)(tokenInfoArray, additionalTokens);
-    tokenInfoArray.sort((a, b) => a.start - b.start);
-    // Update firstTokenOnLine and multilineDocComment
-    previousInfo = tokenInfoArray[0];
-    if (startIndex === 0) {
-        // It is the first token in the file.
-        previousInfo.firstTokenOnLine = true;
-    }
-    else {
-        const previousNonWhitespaceToken = _findNonWhitespaceTokenAtOrBeforeIndex(tokens, startIndex - 1);
-        if (previousNonWhitespaceToken) {
-            const previousEnd = (0, positionUtils_1.convertOffsetToPosition)(textRange_1.TextRange.getEnd(previousNonWhitespaceToken), lines);
-            previousInfo.firstTokenOnLine = previousEnd.line !== previousInfo.range.start.line;
-        }
-        else {
-            previousInfo.firstTokenOnLine = true;
-        }
-    }
-    previousInfo.multilineDocComment = _isMultilineDocComment(parseResults, previousInfo);
-    for (let i = 1; i < tokenInfoArray.length; i++) {
-        const info = tokenInfoArray[i];
-        info.firstTokenOnLine = previousInfo.range.end.line !== info.range.start.line;
-        info.multilineDocComment = _isMultilineDocComment(parseResults, info);
-        previousInfo = info;
-    }
-    return tokenInfoArray;
-}
-function _addTokenInfoIfMatch(parseResults, start, end, charCode, tokens) {
-    for (let i = start; i < end; i++) {
-        if (parseResults.text.charCodeAt(i) === charCode) {
-            tokens.push({
-                start: i,
-                length: 1,
-                range: (0, positionUtils_1.convertTextRangeToRange)(textRange_1.TextRange.create(i, 1), parseResults.tokenizerOutput.lines),
-                text: String.fromCharCode(charCode),
-                kind: 'token',
-                firstTokenOnLine: false,
-                multilineDocComment: false,
-            });
-        }
-    }
-}
-function _isWhitespaceToken(type) {
-    switch (type) {
-        case 4 /* Dedent */:
-        case 2 /* NewLine */:
-        case 3 /* Indent */:
-        case 1 /* EndOfStream */:
-            return true;
-        default:
-            return false;
-    }
-}
-function _isMultilineDocComment(parseResults, info) {
-    var _a, _b;
-    if (info.kind !== 'string' || !info.firstTokenOnLine || info.range.start.line === info.range.end.line) {
-        return false;
-    }
-    const node = (0, parseTreeUtils_1.findNodeByOffset)(parseResults.parseTree, info.start);
-    if ((node === null || node === void 0 ? void 0 : node.nodeType) !== 49 /* String */ ||
-        ((_a = node.parent) === null || _a === void 0 ? void 0 : _a.nodeType) !== 48 /* StringList */ ||
-        ((_b = node.parent.parent) === null || _b === void 0 ? void 0 : _b.nodeType) !== 47 /* StatementList */) {
-        return false;
-    }
-    return (0, parseTreeUtils_1.isDocString)(node.parent.parent);
-}
-function _reindentLinesFromText(parseResults, info, indentDelta) {
-    const texts = [];
-    for (let i = info.range.start.line; i <= info.range.end.line; i++) {
-        texts.push(_reindentLineFromText(parseResults, i, indentDelta, i === info.range.end.line ? info : undefined));
-    }
-    return texts;
-}
-function _reindentLineFromText(parseResults, line, indentDelta, range) {
-    const result = getIndentationFromText(parseResults, line);
-    if (_isBlankLine(parseResults, line, result.charOffset)) {
-        return '';
-    }
-    let lineRange = parseResults.tokenizerOutput.lines.getItemAt(line);
-    if (range) {
-        lineRange = textRange_1.TextRange.fromBounds(lineRange.start, Math.min(textRange_1.TextRange.getEnd(range), textRange_1.TextRange.getEnd(lineRange)));
-    }
-    const text = parseResults.text.substr(lineRange.start + result.charOffset, lineRange.length - result.charOffset);
-    return createIndentationString(parseResults, Math.max(result.indentation + indentDelta, 0)) + text;
-}
-function _getTabSize(parseResults) {
-    const tab = parseResults.tokenizerOutput.predominantTabSequence;
-    const tabLength = tab.length;
-    if (tabLength === 1 && tab.charCodeAt(0) === 9 /* Tab */) {
-        // Tokenizer will use 8 for Char.Tab and put that info in indentToken's indent size.
-        return tokenizer_1.defaultTabSize;
-    }
-    return tabLength;
-}
-function createIndentationString(parseResults, indentation) {
-    const tab = parseResults.tokenizerOutput.predominantTabSequence;
-    const tabLength = tab.length;
-    if (tabLength === 1 && tab.charCodeAt(0) === 9 /* Tab */) {
-        const spaceCount = indentation % tokenizer_1.defaultTabSize;
-        const tabCount = (indentation - spaceCount) / tokenizer_1.defaultTabSize;
-        return '\t'.repeat(tabCount) + ' '.repeat(spaceCount);
-    }
-    return ' '.repeat(indentation);
-}
-exports.createIndentationString = createIndentationString;
-//# sourceMappingURL=indentationUtils.js.map