@typespec/compiler 0.41.0

package/dist/core/scanner.js

@@ -0,0 +1,1012 @@
+import { isAsciiIdentifierContinue, isAsciiIdentifierStart, isBinaryDigit, isDigit, isHexDigit, isIdentifierContinue, isIdentifierStart, isLineBreak, isLowercaseAsciiLetter, isNonAsciiIdentifierCharacter, isNonAsciiWhiteSpaceSingleLine, isWhiteSpace, isWhiteSpaceSingleLine, utf16CodeUnits, } from "./charcode.js";
+import { compilerAssert, createSourceFile } from "./diagnostics.js";
+import { createDiagnostic } from "./messages.js";
+// All conflict markers consist of the same character repeated seven times.  If it is
+// a <<<<<<< or >>>>>>> marker then it is also followed by a space.
+const mergeConflictMarkerLength = 7;
+export var Token;
+(function (Token) {
+    Token[Token["None"] = 0] = "None";
+    Token[Token["Invalid"] = 1] = "Invalid";
+    Token[Token["EndOfFile"] = 2] = "EndOfFile";
+    Token[Token["Identifier"] = 3] = "Identifier";
+    Token[Token["NumericLiteral"] = 4] = "NumericLiteral";
+    Token[Token["StringLiteral"] = 5] = "StringLiteral";
+    // Add new tokens above if they don't fit any of the categories below
+    ///////////////////////////////////////////////////////////////
+    // Trivia
+    /**@internal */ Token[Token["__StartTrivia"] = 6] = "__StartTrivia";
+    Token[Token["SingleLineComment"] = 6] = "SingleLineComment";
+    Token[Token["MultiLineComment"] = 7] = "MultiLineComment";
+    Token[Token["NewLine"] = 8] = "NewLine";
+    Token[Token["Whitespace"] = 9] = "Whitespace";
+    Token[Token["ConflictMarker"] = 10] = "ConflictMarker";
+    // Add new trivia above
+    /** @internal */ Token[Token["__EndTrivia"] = 11] = "__EndTrivia";
+    ///////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////
+    // Doc comment content
+    /** @internal */ Token[Token["__StartDocComment"] = 11] = "__StartDocComment";
+    Token[Token["DocText"] = 11] = "DocText";
+    Token[Token["DocCodeSpan"] = 12] = "DocCodeSpan";
+    Token[Token["DocCodeFenceDelimiter"] = 13] = "DocCodeFenceDelimiter";
+    /** @internal */ Token[Token["__EndDocComment"] = 14] = "__EndDocComment";
+    ///////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////
+    // Punctuation
+    /** @internal */ Token[Token["__StartPunctuation"] = 14] = "__StartPunctuation";
+    Token[Token["OpenBrace"] = 14] = "OpenBrace";
+    Token[Token["CloseBrace"] = 15] = "CloseBrace";
+    Token[Token["OpenParen"] = 16] = "OpenParen";
+    Token[Token["CloseParen"] = 17] = "CloseParen";
+    Token[Token["OpenBracket"] = 18] = "OpenBracket";
+    Token[Token["CloseBracket"] = 19] = "CloseBracket";
+    Token[Token["Dot"] = 20] = "Dot";
+    Token[Token["Ellipsis"] = 21] = "Ellipsis";
+    Token[Token["Semicolon"] = 22] = "Semicolon";
+    Token[Token["Comma"] = 23] = "Comma";
+    Token[Token["LessThan"] = 24] = "LessThan";
+    Token[Token["GreaterThan"] = 25] = "GreaterThan";
+    Token[Token["Equals"] = 26] = "Equals";
+    Token[Token["Ampersand"] = 27] = "Ampersand";
+    Token[Token["Bar"] = 28] = "Bar";
+    Token[Token["Question"] = 29] = "Question";
+    Token[Token["Colon"] = 30] = "Colon";
+    Token[Token["ColonColon"] = 31] = "ColonColon";
+    Token[Token["At"] = 32] = "At";
+    Token[Token["AtAt"] = 33] = "AtAt";
+    Token[Token["Hash"] = 34] = "Hash";
+    Token[Token["Star"] = 35] = "Star";
+    Token[Token["ForwardSlash"] = 36] = "ForwardSlash";
+    Token[Token["Plus"] = 37] = "Plus";
+    Token[Token["Hyphen"] = 38] = "Hyphen";
+    Token[Token["Exclamation"] = 39] = "Exclamation";
+    Token[Token["LessThanEquals"] = 40] = "LessThanEquals";
+    Token[Token["GreaterThanEquals"] = 41] = "GreaterThanEquals";
+    Token[Token["AmpsersandAmpersand"] = 42] = "AmpsersandAmpersand";
+    Token[Token["BarBar"] = 43] = "BarBar";
+    Token[Token["EqualsEquals"] = 44] = "EqualsEquals";
+    Token[Token["ExclamationEquals"] = 45] = "ExclamationEquals";
+    Token[Token["EqualsGreaterThan"] = 46] = "EqualsGreaterThan";
+    // Add new punctuation above
+    /** @internal */ Token[Token["__EndPunctuation"] = 47] = "__EndPunctuation";
+    ///////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////
+    // Statement keywords
+    /** @internal */ Token[Token["__StartKeyword"] = 47] = "__StartKeyword";
+    /** @internal */ Token[Token["__StartStatementKeyword"] = 47] = "__StartStatementKeyword";
+    Token[Token["ImportKeyword"] = 47] = "ImportKeyword";
+    Token[Token["ModelKeyword"] = 48] = "ModelKeyword";
+    Token[Token["ScalarKeyword"] = 49] = "ScalarKeyword";
+    Token[Token["NamespaceKeyword"] = 50] = "NamespaceKeyword";
+    Token[Token["UsingKeyword"] = 51] = "UsingKeyword";
+    Token[Token["OpKeyword"] = 52] = "OpKeyword";
+    Token[Token["EnumKeyword"] = 53] = "EnumKeyword";
+    Token[Token["AliasKeyword"] = 54] = "AliasKeyword";
+    Token[Token["IsKeyword"] = 55] = "IsKeyword";
+    Token[Token["InterfaceKeyword"] = 56] = "InterfaceKeyword";
+    Token[Token["UnionKeyword"] = 57] = "UnionKeyword";
+    Token[Token["ProjectionKeyword"] = 58] = "ProjectionKeyword";
+    Token[Token["ElseKeyword"] = 59] = "ElseKeyword";
+    Token[Token["IfKeyword"] = 60] = "IfKeyword";
+    Token[Token["DecKeyword"] = 61] = "DecKeyword";
+    Token[Token["FnKeyword"] = 62] = "FnKeyword";
+    // Add new statement keyword above
+    /** @internal */ Token[Token["__EndStatementKeyword"] = 63] = "__EndStatementKeyword";
+    ///////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////
+    /** @internal */ Token[Token["__StartModifierKeyword"] = 63] = "__StartModifierKeyword";
+    Token[Token["ExternKeyword"] = 63] = "ExternKeyword";
+    /** @internal */ Token[Token["__EndModifierKeyword"] = 64] = "__EndModifierKeyword";
+    ///////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////
+    // Other keywords
+    Token[Token["ExtendsKeyword"] = 64] = "ExtendsKeyword";
+    Token[Token["TrueKeyword"] = 65] = "TrueKeyword";
+    Token[Token["FalseKeyword"] = 66] = "FalseKeyword";
+    Token[Token["ReturnKeyword"] = 67] = "ReturnKeyword";
+    Token[Token["VoidKeyword"] = 68] = "VoidKeyword";
+    Token[Token["NeverKeyword"] = 69] = "NeverKeyword";
+    Token[Token["UnknownKeyword"] = 70] = "UnknownKeyword";
+    // Add new non-statement keyword above
+    /** @internal */ Token[Token["__EndKeyword"] = 71] = "__EndKeyword";
+    ///////////////////////////////////////////////////////////////
+    /** @internal */ Token[Token["__Count"] = 71] = "__Count";
+})(Token = Token || (Token = {}));
+/** @internal */
+export const TokenDisplay = getTokenDisplayTable([
+    [Token.None, "none"],
+    [Token.Invalid, "invalid"],
+    [Token.EndOfFile, "end of file"],
+    [Token.SingleLineComment, "single-line comment"],
+    [Token.MultiLineComment, "multi-line comment"],
+    [Token.ConflictMarker, "conflict marker"],
+    [Token.NumericLiteral, "numeric literal"],
+    [Token.StringLiteral, "string literal"],
+    [Token.NewLine, "newline"],
+    [Token.Whitespace, "whitespace"],
+    [Token.DocCodeFenceDelimiter, "doc code fence delimiter"],
+    [Token.DocCodeSpan, "doc code span"],
+    [Token.DocText, "doc text"],
+    [Token.OpenBrace, "'{'"],
+    [Token.CloseBrace, "'}'"],
+    [Token.OpenParen, "'('"],
+    [Token.CloseParen, "')'"],
+    [Token.OpenBracket, "'['"],
+    [Token.CloseBracket, "']'"],
+    [Token.Dot, "'.'"],
+    [Token.Ellipsis, "'...'"],
+    [Token.Semicolon, "';'"],
+    [Token.Comma, "','"],
+    [Token.LessThan, "'<'"],
+    [Token.GreaterThan, "'>'"],
+    [Token.Equals, "'='"],
+    [Token.Ampersand, "'&'"],
+    [Token.Bar, "'|'"],
+    [Token.Question, "'?'"],
+    [Token.Colon, "':'"],
+    [Token.ColonColon, "'::'"],
+    [Token.At, "'@'"],
+    [Token.AtAt, "'@@'"],
+    [Token.Hash, "'#'"],
+    [Token.Star, "'*'"],
+    [Token.ForwardSlash, "'/'"],
+    [Token.Plus, "'+'"],
+    [Token.Hyphen, "'-'"],
+    [Token.Exclamation, "'!'"],
+    [Token.LessThanEquals, "'<='"],
+    [Token.GreaterThanEquals, "'>='"],
+    [Token.AmpsersandAmpersand, "'&&'"],
+    [Token.BarBar, "'||'"],
+    [Token.EqualsEquals, "'=='"],
+    [Token.ExclamationEquals, "'!='"],
+    [Token.EqualsGreaterThan, "'=>'"],
+    [Token.Identifier, "identifier"],
+    [Token.ImportKeyword, "'import'"],
+    [Token.ModelKeyword, "'model'"],
+    [Token.ScalarKeyword, "'scalar'"],
+    [Token.NamespaceKeyword, "'namespace'"],
+    [Token.UsingKeyword, "'using'"],
+    [Token.OpKeyword, "'op'"],
+    [Token.EnumKeyword, "'enum'"],
+    [Token.AliasKeyword, "'alias'"],
+    [Token.IsKeyword, "'is'"],
+    [Token.InterfaceKeyword, "'interface'"],
+    [Token.UnionKeyword, "'union'"],
+    [Token.ProjectionKeyword, "'projection'"],
+    [Token.ElseKeyword, "'else'"],
+    [Token.IfKeyword, "'if'"],
+    [Token.DecKeyword, "'dec'"],
+    [Token.FnKeyword, "'fn'"],
+    [Token.ExtendsKeyword, "'extends'"],
+    [Token.TrueKeyword, "'true'"],
+    [Token.FalseKeyword, "'false'"],
+    [Token.ReturnKeyword, "'return'"],
+    [Token.VoidKeyword, "'void'"],
+    [Token.NeverKeyword, "'never'"],
+    [Token.UnknownKeyword, "'unknown'"],
+    [Token.ExternKeyword, "'extern'"],
+]);
+/** @internal */
+export const Keywords = new Map([
+    ["import", Token.ImportKeyword],
+    ["model", Token.ModelKeyword],
+    ["scalar", Token.ScalarKeyword],
+    ["namespace", Token.NamespaceKeyword],
+    ["interface", Token.InterfaceKeyword],
+    ["union", Token.UnionKeyword],
+    ["if", Token.IfKeyword],
+    ["else", Token.ElseKeyword],
+    ["projection", Token.ProjectionKeyword],
+    ["using", Token.UsingKeyword],
+    ["op", Token.OpKeyword],
+    ["extends", Token.ExtendsKeyword],
+    ["is", Token.IsKeyword],
+    ["enum", Token.EnumKeyword],
+    ["alias", Token.AliasKeyword],
+    ["dec", Token.DecKeyword],
+    ["fn", Token.FnKeyword],
+    ["true", Token.TrueKeyword],
+    ["false", Token.FalseKeyword],
+    ["return", Token.ReturnKeyword],
+    ["void", Token.VoidKeyword],
+    ["never", Token.NeverKeyword],
+    ["unknown", Token.UnknownKeyword],
+    ["extern", Token.ExternKeyword],
+]);
+export var TokenFlags;
+(function (TokenFlags) {
+    TokenFlags[TokenFlags["None"] = 0] = "None";
+    TokenFlags[TokenFlags["Escaped"] = 1] = "Escaped";
+    TokenFlags[TokenFlags["TripleQuoted"] = 2] = "TripleQuoted";
+    TokenFlags[TokenFlags["Unterminated"] = 4] = "Unterminated";
+    TokenFlags[TokenFlags["NonAscii"] = 8] = "NonAscii";
+    TokenFlags[TokenFlags["DocComment"] = 16] = "DocComment";
+    TokenFlags[TokenFlags["Backticked"] = 32] = "Backticked";
+})(TokenFlags = TokenFlags || (TokenFlags = {}));
+export function isTrivia(token) {
+    return token >= Token.__StartTrivia && token < Token.__EndTrivia;
+}
+export function isComment(token) {
+    return token === Token.SingleLineComment || token === Token.MultiLineComment;
+}
+export function isKeyword(token) {
+    return token >= Token.__StartKeyword && token < Token.__EndKeyword;
+}
+export function isPunctuation(token) {
+    return token >= Token.__StartPunctuation && token < Token.__EndPunctuation;
+}
+export function isModifier(token) {
+    return token >= Token.__StartModifierKeyword && token < Token.__EndModifierKeyword;
+}
+export function isStatementKeyword(token) {
+    return token >= Token.__StartStatementKeyword && token < Token.__EndStatementKeyword;
+}
+export function createScanner(source, diagnosticHandler) {
+    const file = typeof source === "string" ? createSourceFile(source, "<anonymous file>") : source;
+    const input = file.text;
+    let position = 0;
+    let endPosition = input.length;
+    let token = Token.None;
+    let tokenPosition = -1;
+    let tokenFlags = TokenFlags.None;
+    // Skip BOM
+    if (position < endPosition && input.charCodeAt(position) === 65279 /* CharCode.ByteOrderMark */) {
+        position++;
+    }
+    return {
+        get position() {
+            return position;
+        },
+        get token() {
+            return token;
+        },
+        get tokenPosition() {
+            return tokenPosition;
+        },
+        get tokenFlags() {
+            return tokenFlags;
+        },
+        file,
+        scan,
+        scanRange,
+        scanDoc,
+        eof,
+        getTokenText,
+        getTokenValue,
+    };
+    function eof() {
+        return position >= endPosition;
+    }
+    function getTokenText() {
+        return input.substring(tokenPosition, position);
+    }
+    function getTokenValue() {
+        switch (token) {
+            case Token.StringLiteral:
+                return getStringTokenValue();
+            case Token.Identifier:
+                return getIdentifierTokenValue();
+            default:
+                return getTokenText();
+        }
+    }
+    function lookAhead(offset) {
+        const p = position + offset;
+        if (p >= endPosition) {
+            return Number.NaN;
+        }
+        return input.charCodeAt(p);
+    }
+    function scan() {
+        tokenPosition = position;
+        tokenFlags = TokenFlags.None;
+        if (!eof()) {
+            const ch = input.charCodeAt(position);
+            switch (ch) {
+                case 13 /* CharCode.CarriageReturn */:
+                    if (lookAhead(1) === 10 /* CharCode.LineFeed */) {
+                        position++;
+                    }
+                // fallthrough
+                case 10 /* CharCode.LineFeed */:
+                    return next(Token.NewLine);
+                case 32 /* CharCode.Space */:
+                case 9 /* CharCode.Tab */:
+                case 11 /* CharCode.VerticalTab */:
+                case 12 /* CharCode.FormFeed */:
+                    return scanWhitespace();
+                case 40 /* CharCode.OpenParen */:
+                    return next(Token.OpenParen);
+                case 41 /* CharCode.CloseParen */:
+                    return next(Token.CloseParen);
+                case 44 /* CharCode.Comma */:
+                    return next(Token.Comma);
+                case 58 /* CharCode.Colon */:
+                    return lookAhead(1) === 58 /* CharCode.Colon */ ? next(Token.ColonColon, 2) : next(Token.Colon);
+                case 59 /* CharCode.Semicolon */:
+                    return next(Token.Semicolon);
+                case 91 /* CharCode.OpenBracket */:
+                    return next(Token.OpenBracket);
+                case 93 /* CharCode.CloseBracket */:
+                    return next(Token.CloseBracket);
+                case 123 /* CharCode.OpenBrace */:
+                    return next(Token.OpenBrace);
+                case 125 /* CharCode.CloseBrace */:
+                    return next(Token.CloseBrace);
+                case 64 /* CharCode.At */:
+                    return lookAhead(1) === 64 /* CharCode.At */ ? next(Token.AtAt, 2) : next(Token.At);
+                case 35 /* CharCode.Hash */:
+                    return next(Token.Hash);
+                case 43 /* CharCode.Plus */:
+                    return isDigit(lookAhead(1)) ? scanSignedNumber() : next(Token.Plus);
+                case 45 /* CharCode.Minus */:
+                    return isDigit(lookAhead(1)) ? scanSignedNumber() : next(Token.Hyphen);
+                case 42 /* CharCode.Asterisk */:
+                    return next(Token.Star);
+                case 63 /* CharCode.Question */:
+                    return next(Token.Question);
+                case 38 /* CharCode.Ampersand */:
+                    return lookAhead(1) === 38 /* CharCode.Ampersand */
+                        ? next(Token.AmpsersandAmpersand, 2)
+                        : next(Token.Ampersand);
+                case 46 /* CharCode.Dot */:
+                    return lookAhead(1) === 46 /* CharCode.Dot */ && lookAhead(2) === 46 /* CharCode.Dot */
+                        ? next(Token.Ellipsis, 3)
+                        : next(Token.Dot);
+                case 47 /* CharCode.Slash */:
+                    switch (lookAhead(1)) {
+                        case 47 /* CharCode.Slash */:
+                            return scanSingleLineComment();
+                        case 42 /* CharCode.Asterisk */:
+                            return scanMultiLineComment();
+                    }
+                    return next(Token.ForwardSlash);
+                case 48 /* CharCode._0 */:
+                    switch (lookAhead(1)) {
+                        case 120 /* CharCode.x */:
+                            return scanHexNumber();
+                        case 98 /* CharCode.b */:
+                            return scanBinaryNumber();
+                    }
+                // fallthrough
+                case 49 /* CharCode._1 */:
+                case 50 /* CharCode._2 */:
+                case 51 /* CharCode._3 */:
+                case 52 /* CharCode._4 */:
+                case 53 /* CharCode._5 */:
+                case 54 /* CharCode._6 */:
+                case 55 /* CharCode._7 */:
+                case 56 /* CharCode._8 */:
+                case 57 /* CharCode._9 */:
+                    return scanNumber();
+                case 60 /* CharCode.LessThan */:
+                    if (atConflictMarker())
+                        return scanConflictMarker();
+                    return lookAhead(1) === 61 /* CharCode.Equals */
+                        ? next(Token.LessThanEquals, 2)
+                        : next(Token.LessThan);
+                case 62 /* CharCode.GreaterThan */:
+                    if (atConflictMarker())
+                        return scanConflictMarker();
+                    return lookAhead(1) === 61 /* CharCode.Equals */
+                        ? next(Token.GreaterThanEquals, 2)
+                        : next(Token.GreaterThan);
+                case 61 /* CharCode.Equals */:
+                    if (atConflictMarker())
+                        return scanConflictMarker();
+                    switch (lookAhead(1)) {
+                        case 61 /* CharCode.Equals */:
+                            return next(Token.EqualsEquals, 2);
+                        case 62 /* CharCode.GreaterThan */:
+                            return next(Token.EqualsGreaterThan, 2);
+                    }
+                    return next(Token.Equals);
+                case 124 /* CharCode.Bar */:
+                    if (atConflictMarker())
+                        return scanConflictMarker();
+                    return lookAhead(1) === 124 /* CharCode.Bar */ ? next(Token.BarBar, 2) : next(Token.Bar);
+                case 34 /* CharCode.DoubleQuote */:
+                    return lookAhead(1) === 34 /* CharCode.DoubleQuote */ && lookAhead(2) === 34 /* CharCode.DoubleQuote */
+                        ? scanTripleQuotedString()
+                        : scanString();
+                case 33 /* CharCode.Exclamation */:
+                    return lookAhead(1) === 61 /* CharCode.Equals */
+                        ? next(Token.ExclamationEquals, 2)
+                        : next(Token.Exclamation);
+                case 96 /* CharCode.Backtick */:
+                    return scanBacktickedIdentifier();
+                default:
+                    if (isLowercaseAsciiLetter(ch)) {
+                        return scanIdentifierOrKeyword();
+                    }
+                    if (isAsciiIdentifierStart(ch)) {
+                        return scanIdentifier();
+                    }
+                    if (ch <= 127 /* CharCode.MaxAscii */) {
+                        return scanInvalidCharacter();
+                    }
+                    return scanNonAsciiToken();
+            }
+        }
+        return (token = Token.EndOfFile);
+    }
+    function scanDoc() {
+        tokenPosition = position;
+        tokenFlags = TokenFlags.None;
+        if (!eof()) {
+            const ch = input.charCodeAt(position);
+            switch (ch) {
+                case 13 /* CharCode.CarriageReturn */:
+                    if (lookAhead(1) === 10 /* CharCode.LineFeed */) {
+                        position++;
+                    }
+                // fallthrough
+                case 10 /* CharCode.LineFeed */:
+                    return next(Token.NewLine);
+                case 32 /* CharCode.Space */:
+                case 9 /* CharCode.Tab */:
+                case 11 /* CharCode.VerticalTab */:
+                case 12 /* CharCode.FormFeed */:
+                    return scanWhitespace();
+                case 125 /* CharCode.CloseBrace */:
+                    return next(Token.CloseBrace);
+                case 64 /* CharCode.At */:
+                    return next(Token.At);
+                case 42 /* CharCode.Asterisk */:
+                    return next(Token.Star);
+                case 96 /* CharCode.Backtick */:
+                    return lookAhead(1) === 96 /* CharCode.Backtick */ && lookAhead(2) === 96 /* CharCode.Backtick */
+                        ? next(Token.DocCodeFenceDelimiter, 3)
+                        : scanDocCodeSpan();
+                case 60 /* CharCode.LessThan */:
+                case 62 /* CharCode.GreaterThan */:
+                case 61 /* CharCode.Equals */:
+                case 124 /* CharCode.Bar */:
+                    if (atConflictMarker())
+                        return scanConflictMarker();
+                    return next(Token.DocText);
+            }
+            if (isAsciiIdentifierStart(ch)) {
+                return scanIdentifier();
+            }
+            if (ch <= 127 /* CharCode.MaxAscii */) {
+                return next(Token.DocText);
+            }
+            const cp = input.codePointAt(position);
+            if (isIdentifierStart(cp)) {
+                return scanNonAsciiIdentifier(cp);
+            }
+            return scanUnknown(Token.DocText);
+        }
+        return (token = Token.EndOfFile);
+    }
+    function scanRange(range, callback) {
+        const savedPosition = position;
+        const savedEndPosition = endPosition;
+        const savedToken = token;
+        const savedTokenPosition = tokenPosition;
+        const savedTokenFlags = tokenFlags;
+        position = range.pos;
+        endPosition = range.end;
+        token = Token.None;
+        tokenPosition = -1;
+        tokenFlags = TokenFlags.None;
+        const result = callback();
+        position = savedPosition;
+        endPosition = savedEndPosition;
+        token = savedToken;
+        tokenPosition = savedTokenPosition;
+        tokenFlags = savedTokenFlags;
+        return result;
+    }
+    function next(t, count = 1) {
+        position += count;
+        return (token = t);
+    }
+    function unterminated(t) {
+        tokenFlags |= TokenFlags.Unterminated;
+        error({ code: "unterminated", format: { token: TokenDisplay[t] } });
+        return (token = t);
+    }
+    function scanNonAsciiToken() {
+        tokenFlags |= TokenFlags.NonAscii;
+        const ch = input.charCodeAt(position);
+        if (isNonAsciiWhiteSpaceSingleLine(ch)) {
+            return scanWhitespace();
+        }
+        const cp = input.codePointAt(position);
+        if (isNonAsciiIdentifierCharacter(cp)) {
+            return scanNonAsciiIdentifier(cp);
+        }
+        return scanInvalidCharacter();
+    }
+    function scanInvalidCharacter() {
+        token = scanUnknown(Token.Invalid);
+        error({ code: "invalid-character" });
+        return token;
+    }
+    function scanUnknown(t) {
+        const codePoint = input.codePointAt(position);
+        return (token = next(t, utf16CodeUnits(codePoint)));
+    }
+    function error(report) {
+        const diagnostic = createDiagnostic({
+            ...report,
+            target: { file, pos: tokenPosition, end: position },
+        });
+        diagnosticHandler(diagnostic);
+    }
+    function scanWhitespace() {
+        do {
+            position++;
+        } while (!eof() && isWhiteSpaceSingleLine(input.charCodeAt(position)));
+        return (token = Token.Whitespace);
+    }
+    function scanSignedNumber() {
+        position++; // consume '+/-'
+        return scanNumber();
+    }
+    function scanNumber() {
+        scanKnownDigits();
+        if (!eof() && input.charCodeAt(position) === 46 /* CharCode.Dot */) {
+            position++;
+            scanRequiredDigits();
+        }
+        if (!eof() && input.charCodeAt(position) === 101 /* CharCode.e */) {
+            position++;
+            const ch = input.charCodeAt(position);
+            if (ch === 43 /* CharCode.Plus */ || ch === 45 /* CharCode.Minus */) {
+                position++;
+            }
+            scanRequiredDigits();
+        }
+        return (token = Token.NumericLiteral);
+    }
+    function scanKnownDigits() {
+        do {
+            position++;
+        } while (!eof() && isDigit(input.charCodeAt(position)));
+    }
+    function scanRequiredDigits() {
+        if (eof() || !isDigit(input.charCodeAt(position))) {
+            error({ code: "digit-expected" });
+            return;
+        }
+        scanKnownDigits();
+    }
+    function scanHexNumber() {
+        position += 2; // consume '0x'
+        if (eof() || !isHexDigit(input.charCodeAt(position))) {
+            error({ code: "hex-digit-expected" });
+            return (token = Token.NumericLiteral);
+        }
+        do {
+            position++;
+        } while (!eof() && isHexDigit(input.charCodeAt(position)));
+        return (token = Token.NumericLiteral);
+    }
+    function scanBinaryNumber() {
+        position += 2; // consume '0b'
+        if (eof() || !isBinaryDigit(input.charCodeAt(position))) {
+            error({ code: "binary-digit-expected" });
+            return (token = Token.NumericLiteral);
+        }
+        do {
+            position++;
+        } while (!eof() && isBinaryDigit(input.charCodeAt(position)));
+        return (token = Token.NumericLiteral);
+    }
+    function scanSingleLineComment() {
+        position = skipSingleLineComment(input, position, endPosition);
+        return (token = Token.SingleLineComment);
+    }
+    function scanMultiLineComment() {
+        token = Token.MultiLineComment;
+        if (lookAhead(2) === 42 /* CharCode.Asterisk */) {
+            tokenFlags |= TokenFlags.DocComment;
+        }
+        const [newPosition, terminated] = skipMultiLineComment(input, position);
+        position = newPosition;
+        return terminated ? token : unterminated(token);
+    }
+    function scanDocCodeSpan() {
+        position++; // consume '`'
+        loop: for (; !eof(); position++) {
+            const ch = input.charCodeAt(position);
+            switch (ch) {
+                case 96 /* CharCode.Backtick */:
+                    position++;
+                    return (token = Token.DocCodeSpan);
+                case 13 /* CharCode.CarriageReturn */:
+                case 10 /* CharCode.LineFeed */:
+                    break loop;
+            }
+        }
+        return unterminated(Token.DocCodeSpan);
+    }
+    function scanString() {
+        position++; // consume '"'
+        loop: for (; !eof(); position++) {
+            const ch = input.charCodeAt(position);
+            switch (ch) {
+                case 92 /* CharCode.Backslash */:
+                    tokenFlags |= TokenFlags.Escaped;
+                    position++;
+                    if (eof()) {
+                        break loop;
+                    }
+                    continue;
+                case 34 /* CharCode.DoubleQuote */:
+                    position++;
+                    return (token = Token.StringLiteral);
+                case 13 /* CharCode.CarriageReturn */:
+                case 10 /* CharCode.LineFeed */:
+                    break loop;
+            }
+        }
+        return unterminated(Token.StringLiteral);
+    }
+    function scanTripleQuotedString() {
+        tokenFlags |= TokenFlags.TripleQuoted;
+        position += 3; // consume '"""'
+        for (; !eof(); position++) {
+            if (input.charCodeAt(position) === 34 /* CharCode.DoubleQuote */ &&
+                lookAhead(1) === 34 /* CharCode.DoubleQuote */ &&
+                lookAhead(2) === 34 /* CharCode.DoubleQuote */) {
+                position += 3;
+                return (token = Token.StringLiteral);
+            }
+        }
+        return unterminated(Token.StringLiteral);
+    }
+    function getStringTokenValue() {
+        const quoteLength = tokenFlags & TokenFlags.TripleQuoted ? 3 : 1;
+        const start = tokenPosition + quoteLength;
+        const end = tokenFlags & TokenFlags.Unterminated ? position : position - quoteLength;
+        if (tokenFlags & TokenFlags.TripleQuoted) {
+            return unindentAndUnescapeTripleQuotedString(start, end);
+        }
+        if (tokenFlags & TokenFlags.Escaped) {
+            return unescapeString(start, end);
+        }
+        return input.substring(start, end);
+    }
+    function getIdentifierTokenValue() {
+        const start = tokenFlags & TokenFlags.Backticked ? tokenPosition + 1 : tokenPosition;
+        const end = tokenFlags & TokenFlags.Backticked && !(tokenFlags & TokenFlags.Unterminated)
+            ? position - 1
+            : position;
+        const text = tokenFlags & TokenFlags.Escaped ? unescapeString(start, end) : input.substring(start, end);
+        if (tokenFlags & TokenFlags.NonAscii) {
+            return text.normalize("NFC");
+        }
+        return text;
+    }
+    function unindentAndUnescapeTripleQuotedString(start, end) {
+        // ignore leading whitespace before required initial line break
+        while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) {
+            start++;
+        }
+        // remove required initial line break
+        if (isLineBreak(input.charCodeAt(start))) {
+            if (isCrlf(start, start, end)) {
+                start++;
+            }
+            start++;
+        }
+        else {
+            error({ code: "no-new-line-start-triple-quote" });
+        }
+        // remove whitespace before closing delimiter and record it as required
+        // indentation for all lines
+        const indentationEnd = end;
+        while (end > start && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) {
+            end--;
+        }
+        const indentationStart = end;
+        // remove required final line break
+        if (isLineBreak(input.charCodeAt(end - 1))) {
+            if (isCrlf(end - 2, start, end)) {
+                end--;
+            }
+            end--;
+        }
+        else {
+            error({ code: "no-new-line-end-triple-quote" });
+        }
+        // remove required matching indentation from each line and unescape in the
+        // process of doing so
+        let result = "";
+        let pos = start;
+        while (pos < end) {
+            // skip indentation at start of line
+            start = skipMatchingIndentation(pos, end, indentationStart, indentationEnd);
+            let ch;
+            while (pos < end && !isLineBreak((ch = input.charCodeAt(pos)))) {
+                if (ch !== 92 /* CharCode.Backslash */) {
+                    pos++;
+                    continue;
+                }
+                result += input.substring(start, pos);
+                if (pos === end - 1) {
+                    error({ code: "invalid-escape-sequence" });
+                    pos++;
+                }
+                else {
+                    result += unescapeOne(pos);
+                    pos += 2;
+                }
+                start = pos;
+            }
+            if (pos < end) {
+                if (isCrlf(pos, start, end)) {
+                    // CRLF in multi-line string is normalized to LF in string value.
+                    // This keeps program behavior unchanged by line-ending conversion.
+                    result += input.substring(start, pos);
+                    result += "\n";
+                    pos += 2;
+                }
+                else {
+                    pos++; // include non-CRLF newline
+                    result += input.substring(start, pos);
+                }
+                start = pos;
+            }
+        }
+        result += input.substring(start, pos);
+        return result;
+    }
+    function isCrlf(pos, start, end) {
+        return (pos >= start &&
+            pos < end - 1 &&
+            input.charCodeAt(pos) === 13 /* CharCode.CarriageReturn */ &&
+            input.charCodeAt(pos + 1) === 10 /* CharCode.LineFeed */);
+    }
+    function skipMatchingIndentation(pos, end, indentationStart, indentationEnd) {
+        let indentationPos = indentationStart;
+        end = Math.min(end, pos + (indentationEnd - indentationStart));
+        while (pos < end) {
+            const ch = input.charCodeAt(pos);
+            if (isLineBreak(ch)) {
+                // allow subset of indentation if line has only whitespace
+                break;
+            }
+            if (ch !== input.charCodeAt(indentationPos)) {
+                error({ code: "triple-quote-indent" });
+                break;
+            }
+            indentationPos++;
+            pos++;
+        }
+        return pos;
+    }
+    function unescapeString(start, end) {
+        let result = "";
+        let pos = start;
+        while (pos < end) {
+            const ch = input.charCodeAt(pos);
+            if (ch !== 92 /* CharCode.Backslash */) {
+                pos++;
+                continue;
+            }
+            if (pos === end - 1) {
+                error({ code: "invalid-escape-sequence" });
+                break;
+            }
+            result += input.substring(start, pos);
+            result += unescapeOne(pos);
+            pos += 2;
+            start = pos;
+        }
+        result += input.substring(start, pos);
+        return result;
+    }
+    function unescapeOne(pos) {
+        const ch = input.charCodeAt(pos + 1);
+        switch (ch) {
+            case 114 /* CharCode.r */:
+                return "\r";
+            case 110 /* CharCode.n */:
+                return "\n";
+            case 116 /* CharCode.t */:
+                return "\t";
+            case 34 /* CharCode.DoubleQuote */:
+                return '"';
+            case 92 /* CharCode.Backslash */:
+                return "\\";
+            case 96 /* CharCode.Backtick */:
+                return "`";
+            default:
+                error({ code: "invalid-escape-sequence" });
+                return String.fromCharCode(ch);
+        }
+    }
+    function scanIdentifierOrKeyword() {
+        let count = 0;
+        let ch = input.charCodeAt(position);
+        while (true) {
+            position++;
+            count++;
+            if (eof()) {
+                break;
+            }
+            ch = input.charCodeAt(position);
+            if (count < 10 /* KeywordLimit.MaxLength */ && isLowercaseAsciiLetter(ch)) {
+                continue;
+            }
+            if (isAsciiIdentifierContinue(ch)) {
+                return scanIdentifier();
+            }
+            if (ch > 127 /* CharCode.MaxAscii */) {
+                const cp = input.codePointAt(position);
+                if (isNonAsciiIdentifierCharacter(cp)) {
+                    return scanNonAsciiIdentifier(cp);
+                }
+            }
+            break;
+        }
+        if (count >= 2 /* KeywordLimit.MinLength */ && count <= 10 /* KeywordLimit.MaxLength */) {
+            const keyword = Keywords.get(getTokenText());
+            if (keyword) {
+                return (token = keyword);
+            }
+        }
+        return (token = Token.Identifier);
+    }
+    function scanIdentifier() {
+        let ch;
+        do {
+            position++;
+            if (eof()) {
+                return (token = Token.Identifier);
+            }
+        } while (isAsciiIdentifierContinue((ch = input.charCodeAt(position))));
+        if (ch > 127 /* CharCode.MaxAscii */) {
+            const cp = input.codePointAt(position);
+            if (isNonAsciiIdentifierCharacter(cp)) {
+                return scanNonAsciiIdentifier(cp);
+            }
+        }
+        return (token = Token.Identifier);
+    }
+    function scanBacktickedIdentifier() {
+        position++; // consume '`'
+        tokenFlags |= TokenFlags.Backticked;
+        loop: for (; !eof(); position++) {
+            const ch = input.charCodeAt(position);
+            switch (ch) {
+                case 92 /* CharCode.Backslash */:
+                    position++;
+                    tokenFlags |= TokenFlags.Escaped;
+                    continue;
+                case 96 /* CharCode.Backtick */:
+                    position++;
+                    return (token = Token.Identifier);
+                case 13 /* CharCode.CarriageReturn */:
+                case 10 /* CharCode.LineFeed */:
+                    break loop;
+                default:
+                    if (ch > 127 /* CharCode.MaxAscii */) {
+                        tokenFlags |= TokenFlags.NonAscii;
+                    }
+            }
+        }
+        return unterminated(Token.Identifier);
+    }
+    function scanNonAsciiIdentifier(startCodePoint) {
+        tokenFlags |= TokenFlags.NonAscii;
+        let cp = startCodePoint;
+        do {
+            position += utf16CodeUnits(cp);
+        } while (!eof() && isIdentifierContinue((cp = input.codePointAt(position))));
+        return (token = Token.Identifier);
+    }
+    function atConflictMarker() {
+        return isConflictMarker(input, position, endPosition);
+    }
+    function scanConflictMarker() {
+        const marker = input.charCodeAt(position);
+        position += mergeConflictMarkerLength;
+        error({ code: "conflict-marker" });
+        if (marker === 60 /* CharCode.LessThan */ || marker === 62 /* CharCode.GreaterThan */) {
+            // Consume everything from >>>>>>> or <<<<<<< to the end of the line.
+            while (position < endPosition && !isLineBreak(input.charCodeAt(position))) {
+                position++;
+            }
+        }
+        else {
+            // Consume everything from the start of a ||||||| or =======
+            // marker to the start of the next ======= or >>>>>>> marker.
+            while (position < endPosition) {
+                const ch = input.charCodeAt(position);
+                if ((ch === 61 /* CharCode.Equals */ || ch === 62 /* CharCode.GreaterThan */) &&
+                    ch !== marker &&
+                    isConflictMarker(input, position, endPosition)) {
+                    break;
+                }
+                position++;
+            }
+        }
+        return (token = Token.ConflictMarker);
+    }
+}
+export function skipTrivia(input, position, endPosition = input.length) {
+    while (position < endPosition) {
+        const ch = input.charCodeAt(position);
+        if (isWhiteSpace(ch)) {
+            position++;
+            continue;
+        }
+        if (ch === 47 /* CharCode.Slash */) {
+            switch (input.charCodeAt(position + 1)) {
+                case 47 /* CharCode.Slash */:
+                    position = skipSingleLineComment(input, position, endPosition);
+                    continue;
+                case 42 /* CharCode.Asterisk */:
+                    position = skipMultiLineComment(input, position, endPosition)[0];
+                    continue;
+            }
+        }
+        break;
+    }
+    return position;
+}
+export function skipWhiteSpace(input, position, endPosition = input.length) {
+    while (position < endPosition) {
+        const ch = input.charCodeAt(position);
+        if (!isWhiteSpace(ch)) {
+            break;
+        }
+        position++;
+    }
+    return position;
+}
+function skipSingleLineComment(input, position, endPosition = input.length) {
+    position += 2; // consume '//'
+    for (; position < endPosition; position++) {
+        if (isLineBreak(input.charCodeAt(position))) {
+            break;
+        }
+    }
+    return position;
+}
+function skipMultiLineComment(input, position, endPosition = input.length) {
+    position += 2; // consume '/*'
+    for (; position < endPosition; position++) {
+        if (input.charCodeAt(position) === 42 /* CharCode.Asterisk */ &&
+            input.charCodeAt(position + 1) === 47 /* CharCode.Slash */) {
+            return [position + 2, true];
+        }
+    }
+    return [position, false];
+}
+function isConflictMarker(input, position, endPosition = input.length) {
+    // Conflict markers must be at the start of a line.
+    const ch = input.charCodeAt(position);
+    if (position === 0 || isLineBreak(input.charCodeAt(position - 1))) {
+        if (position + mergeConflictMarkerLength < endPosition) {
+            for (let i = 0; i < mergeConflictMarkerLength; i++) {
+                if (input.charCodeAt(position + i) !== ch) {
+                    return false;
+                }
+            }
+            return (ch === 61 /* CharCode.Equals */ ||
+                input.charCodeAt(position + mergeConflictMarkerLength) === 32 /* CharCode.Space */);
+        }
+    }
+    return false;
+}
+function getTokenDisplayTable(entries) {
+    const table = new Array(entries.length);
+    for (const [token, display] of entries) {
+        compilerAssert(token >= 0 && token < Token.__Count, `Invalid entry in token display table, ${token}, ${Token[token]}, ${display}`);
+        compilerAssert(!table[token], `Duplicate entry in token display table for: ${token}, ${Token[token]}, ${display}`);
+        table[token] = display;
+    }
+    for (let token = 0; token < Token.__Count; token++) {
+        compilerAssert(table[token], `Missing entry in token display table: ${token}, ${Token[token]}`);
+    }
+    return table;
+}
+//# sourceMappingURL=scanner.js.map