@typespec/compiler 0.57.0-dev.27 → 0.57.0-dev.28

package/dist/src/core/parser.js

@@ -1,8 +1,8 @@
 import { isArray, mutate } from "../utils/misc.js";
-import { trim } from "./charcode.js";
+import { codePointBefore, isIdentifierContinue, trim } from "./charcode.js";
 import { compilerAssert } from "./diagnostics.js";
 import { createDiagnostic } from "./messages.js";
-import { Token, TokenDisplay, TokenFlags, createScanner, isComment, isKeyword, isPunctuation, isStatementKeyword, isTrivia, } from "./scanner.js";
+import { Token, TokenDisplay, TokenFlags, createScanner, isComment, isKeyword, isPunctuation, isStatementKeyword, isTrivia, skipContinuousIdentifier, skipTrivia, skipTriviaBackward, } from "./scanner.js";
 import { IdentifierKind, SyntaxKind, } from "./types.js";
 /**
  * The fixed set of options for each of the kinds of delimited lists in TypeSpec.
@@ -450,8 +450,8 @@ function createParser(code, options = {}) {
     function parseInterfaceStatement(pos, decorators) {
         parseExpected(Token.InterfaceKeyword);
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
-        let extendList = [];
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        let extendList = createEmptyList();
         if (token() === Token.ExtendsKeyword) {
             nextToken();
             extendList = parseList(ListKind.Heritage, parseReferenceExpression);
@@ -460,21 +460,23 @@ function createParser(code, options = {}) {
             error({ code: "token-expected", format: { token: "'extends' or '{'" } });
             nextToken();
         }
-        const operations = parseList(ListKind.InterfaceMembers, (pos, decorators) => parseOperationStatement(pos, decorators, true));
+        const { items: operations, range: bodyRange } = parseList(ListKind.InterfaceMembers, (pos, decorators) => parseOperationStatement(pos, decorators, true));
         return {
             kind: SyntaxKind.InterfaceStatement,
             id,
             templateParameters,
+            templateParametersRange,
             operations,
-            extends: extendList,
+            bodyRange,
+            extends: extendList.items,
             decorators,
             ...finishNode(pos),
         };
     }
     function parseTemplateParameterList() {
-        const list = parseOptionalList(ListKind.TemplateParameters, parseTemplateParameter);
+        const detail = parseOptionalList(ListKind.TemplateParameters, parseTemplateParameter);
         let setDefault = false;
-        for (const item of list) {
+        for (const item of detail.items) {
             if (!item.default && setDefault) {
                 error({ code: "default-required", target: item });
                 continue;
@@ -483,17 +485,18 @@ function createParser(code, options = {}) {
                 setDefault = true;
             }
         }
-        return list;
+        return detail;
     }
     function parseUnionStatement(pos, decorators) {
         parseExpected(Token.UnionKeyword);
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
-        const options = parseList(ListKind.UnionVariants, parseUnionVariant);
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        const { items: options } = parseList(ListKind.UnionVariants, parseUnionVariant);
         return {
             kind: SyntaxKind.UnionStatement,
             id,
             templateParameters,
+            templateParametersRange,
             decorators,
             options,
             ...finishNode(pos),
@@ -559,7 +562,7 @@ function createParser(code, options = {}) {
             parseExpected(Token.OpKeyword);
         }
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
         // Make sure the next token is one that is expected
         const token = expectTokenIsOneOf(Token.OpenParen, Token.IsKeyword);
         // Check if we're parsing a declaration or reuse of another operation
@@ -593,6 +596,7 @@ function createParser(code, options = {}) {
             kind: SyntaxKind.OperationStatement,
             id,
             templateParameters,
+            templateParametersRange,
             signature,
             decorators,
             ...finishNode(pos),
@@ -600,10 +604,11 @@ function createParser(code, options = {}) {
     }
     function parseOperationParameters() {
         const pos = tokenPos();
-        const properties = parseList(ListKind.OperationParameters, parseModelPropertyOrSpread);
+        const { items: properties, range: bodyRange } = parseList(ListKind.OperationParameters, parseModelPropertyOrSpread);
         const parameters = {
             kind: SyntaxKind.ModelExpression,
             properties,
+            bodyRange,
             ...finishNode(pos),
         };
         return parameters;
@@ -611,22 +616,22 @@ function createParser(code, options = {}) {
     function parseModelStatement(pos, decorators) {
         parseExpected(Token.ModelKeyword);
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
         expectTokenIsOneOf(Token.OpenBrace, Token.Equals, Token.ExtendsKeyword, Token.IsKeyword);
         const optionalExtends = parseOptionalModelExtends();
         const optionalIs = optionalExtends ? undefined : parseOptionalModelIs();
-        let properties = [];
+        let propDetail = createEmptyList();
         if (optionalIs) {
             const tok = expectTokenIsOneOf(Token.Semicolon, Token.OpenBrace);
             if (tok === Token.Semicolon) {
                 nextToken();
             }
             else {
-                properties = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+                propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
             }
         }
         else {
-            properties = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+            propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
         }
         return {
             kind: SyntaxKind.ModelStatement,
@@ -634,8 +639,10 @@ function createParser(code, options = {}) {
             extends: optionalExtends,
             is: optionalIs,
             templateParameters,
+            templateParametersRange,
             decorators,
-            properties,
+            properties: propDetail.items,
+            bodyRange: propDetail.range,
             ...finishNode(pos),
         };
     }
@@ -767,15 +774,17 @@ function createParser(code, options = {}) {
     function parseScalarStatement(pos, decorators) {
         parseExpected(Token.ScalarKeyword);
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
         const optionalExtends = parseOptionalScalarExtends();
-        const members = parseScalarMembers();
+        const { items: members, range: bodyRange } = parseScalarMembers();
         return {
             kind: SyntaxKind.ScalarStatement,
             id,
             templateParameters,
+            templateParametersRange,
             extends: optionalExtends,
             members,
+            bodyRange,
             decorators,
             ...finishNode(pos),
         };
@@ -789,7 +798,7 @@ function createParser(code, options = {}) {
     function parseScalarMembers() {
         if (token() === Token.Semicolon) {
             nextToken();
-            return [];
+            return createEmptyList();
         }
         else {
             return parseList(ListKind.ScalarMembers, parseScalarMember);
@@ -799,7 +808,7 @@ function createParser(code, options = {}) {
         reportInvalidDecorators(decorators, "scalar member");
         parseExpected(Token.InitKeyword);
         const id = parseIdentifier();
-        const parameters = parseFunctionParameters();
+        const { items: parameters } = parseFunctionParameters();
         return {
             kind: SyntaxKind.ScalarConstructor,
             id,
@@ -810,7 +819,7 @@ function createParser(code, options = {}) {
     function parseEnumStatement(pos, decorators) {
         parseExpected(Token.EnumKeyword);
         const id = parseIdentifier();
-        const members = parseList(ListKind.EnumMembers, parseEnumMemberOrSpread);
+        const { items: members } = parseList(ListKind.EnumMembers, parseEnumMemberOrSpread);
         return {
             kind: SyntaxKind.EnumStatement,
             id,
@@ -864,7 +873,7 @@ function createParser(code, options = {}) {
     function parseAliasStatement(pos) {
         parseExpected(Token.AliasKeyword);
         const id = parseIdentifier();
-        const templateParameters = parseTemplateParameterList();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
         parseExpected(Token.Equals);
         const value = parseExpression();
         parseExpected(Token.Semicolon);
@@ -872,6 +881,7 @@ function createParser(code, options = {}) {
             kind: SyntaxKind.AliasStatement,
             id,
             templateParameters,
+            templateParametersRange,
             value,
             ...finishNode(pos),
         };
@@ -1011,17 +1021,18 @@ function createParser(code, options = {}) {
         const pos = tokenPos();
         const target = parseIdentifierOrMemberExpression(message);
         if (token() === Token.OpenParen) {
+            const { items: args } = parseList(ListKind.FunctionArguments, parseExpression);
             return {
                 kind: SyntaxKind.CallExpression,
                 target,
-                arguments: parseList(ListKind.FunctionArguments, parseExpression),
+                arguments: args,
                 ...finishNode(pos),
             };
         }
         return parseReferenceExpressionInternal(target, pos);
     }
     function parseReferenceExpressionInternal(target, pos) {
-        const args = parseOptionalList(ListKind.TemplateArguments, parseTemplateArgument);
+        const { items: args } = parseOptionalList(ListKind.TemplateArguments, parseTemplateArgument);
         return {
             kind: SyntaxKind.TypeReference,
             target,
@@ -1072,29 +1083,31 @@ function createParser(code, options = {}) {
         // `@<missing identifier>` applied to `model Foo`, and not as `@model`
         // applied to invalid statement `Foo`.
         const target = parseIdentifierOrMemberExpression(undefined, false);
-        const args = parseOptionalList(ListKind.DecoratorArguments, parseExpression);
+        const { items: args } = parseOptionalList(ListKind.DecoratorArguments, parseExpression);
         if (args.length === 0) {
             error({ code: "augment-decorator-target" });
+            const emptyList = createEmptyList();
             return {
                 kind: SyntaxKind.AugmentDecoratorStatement,
                 target,
                 targetType: {
                     kind: SyntaxKind.TypeReference,
                     target: createMissingIdentifier(),
-                    arguments: [],
+                    arguments: emptyList.items,
                     ...finishNode(pos),
                 },
-                arguments: [],
+                arguments: args,
                 ...finishNode(pos),
             };
         }
         let [targetEntity, ...decoratorArgs] = args;
         if (targetEntity.kind !== SyntaxKind.TypeReference) {
             error({ code: "augment-decorator-target", target: targetEntity });
+            const emptyList = createEmptyList();
             targetEntity = {
                 kind: SyntaxKind.TypeReference,
                 target: createMissingIdentifier(),
-                arguments: [],
+                arguments: emptyList.items,
                 ...finishNode(pos),
             };
         }
@@ -1126,7 +1139,7 @@ function createParser(code, options = {}) {
         // `@<missing identifier>` applied to `model Foo`, and not as `@model`
         // applied to invalid statement `Foo`.
         const target = parseIdentifierOrMemberExpression(undefined, false);
-        const args = parseOptionalList(ListKind.DecoratorArguments, parseExpression);
+        const { items: args } = parseOptionalList(ListKind.DecoratorArguments, parseExpression);
         return {
             kind: SyntaxKind.DecoratorExpression,
             arguments: args,
@@ -1310,7 +1323,7 @@ function createParser(code, options = {}) {
     }
     function parseTupleExpression() {
         const pos = tokenPos();
-        const values = parseList(ListKind.Tuple, parseExpression);
+        const { items: values } = parseList(ListKind.Tuple, parseExpression);
         return {
             kind: SyntaxKind.TupleExpression,
             values,
@@ -1319,25 +1332,27 @@ function createParser(code, options = {}) {
     }
     function parseModelExpression() {
         const pos = tokenPos();
-        const properties = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+        const { items: properties, range: bodyRange } = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
         return {
             kind: SyntaxKind.ModelExpression,
             properties,
+            bodyRange,
             ...finishNode(pos),
         };
     }
     function parseObjectLiteral() {
         const pos = tokenPos();
-        const properties = parseList(ListKind.ObjectLiteralProperties, parseObjectLiteralPropertyOrSpread);
+        const { items: properties, range: bodyRange } = parseList(ListKind.ObjectLiteralProperties, parseObjectLiteralPropertyOrSpread);
         return {
             kind: SyntaxKind.ObjectLiteral,
             properties,
+            bodyRange,
             ...finishNode(pos),
         };
     }
     function parseArrayLiteral() {
         const pos = tokenPos();
-        const values = parseList(ListKind.ArrayLiteral, parseExpression);
+        const { items: values } = parseList(ListKind.ArrayLiteral, parseExpression);
         return {
             kind: SyntaxKind.ArrayLiteral,
             values,
@@ -1510,7 +1525,8 @@ function createParser(code, options = {}) {
         const modifierFlags = modifiersToFlags(modifiers);
         parseExpected(Token.DecKeyword);
         const id = parseIdentifier();
-        let [target, ...parameters] = parseFunctionParameters();
+        const allParamListDetail = parseFunctionParameters();
+        let [target, ...parameters] = allParamListDetail.items;
         if (target === undefined) {
             error({ code: "decorator-decl-target", target: { pos, end: previousTokenEnd } });
             target = {
@@ -1540,7 +1556,7 @@ function createParser(code, options = {}) {
         const modifierFlags = modifiersToFlags(modifiers);
         parseExpected(Token.FnKeyword);
         const id = parseIdentifier();
-        const parameters = parseFunctionParameters();
+        const { items: parameters } = parseFunctionParameters();
         let returnType;
         if (parseOptional(Token.Colon)) {
             returnType = parseExpression();
@@ -1559,7 +1575,7 @@ function createParser(code, options = {}) {
     function parseFunctionParameters() {
         const parameters = parseList(ListKind.FunctionParameters, parseFunctionParameter);
         let foundOptional = false;
-        for (const [index, item] of parameters.entries()) {
+        for (const [index, item] of parameters.items.entries()) {
             if (!item.optional && foundOptional) {
                 error({ code: "required-parameter-first", target: item });
                 continue;
@@ -1570,7 +1586,7 @@ function createParser(code, options = {}) {
             if (item.rest && item.optional) {
                 error({ code: "rest-parameter-required", target: item });
             }
-            if (item.rest && index !== parameters.length - 1) {
+            if (item.rest && index !== parameters.items.length - 1) {
                 error({ code: "rest-parameter-last", target: item });
             }
         }
@@ -1662,7 +1678,7 @@ function createParser(code, options = {}) {
         }
         let parameters;
         if (token() === Token.OpenParen) {
-            parameters = parseList(ListKind.ProjectionParameter, parseProjectionParameter);
+            parameters = parseList(ListKind.ProjectionParameter, parseProjectionParameter).items;
         }
         else {
             parameters = [];
@@ -1884,7 +1900,7 @@ function createParser(code, options = {}) {
                     kind: SyntaxKind.ProjectionCallExpression,
                     callKind: "method",
                     target: expr,
-                    arguments: parseList(ListKind.CallArguments, parseProjectionExpression),
+                    arguments: parseList(ListKind.CallArguments, parseProjectionExpression).items,
                     ...finishNode(pos),
                 };
             }
@@ -1967,7 +1983,7 @@ function createParser(code, options = {}) {
     }
     function parseProjectionLambdaOrParenthesizedExpression() {
         const pos = tokenPos();
-        const exprs = parseList(ListKind.ProjectionExpression, parseProjectionExpression);
+        const exprs = parseList(ListKind.ProjectionExpression, parseProjectionExpression).items;
         if (token() === Token.EqualsGreaterThan) {
             // unpack the exprs (which should be just identifiers) into a param list
             const params = [];
@@ -2018,7 +2034,7 @@ function createParser(code, options = {}) {
     }
     function parseProjectionModelExpression() {
         const pos = tokenPos();
-        const properties = parseList(ListKind.ModelProperties, parseProjectionModelPropertyOrSpread);
+        const { items: properties } = parseList(ListKind.ModelProperties, parseProjectionModelPropertyOrSpread);
         return {
             kind: SyntaxKind.ProjectionModelExpression,
             properties,
@@ -2092,7 +2108,7 @@ function createParser(code, options = {}) {
     }
     function parseProjectionTupleExpression() {
         const pos = tokenPos();
-        const values = parseList(ListKind.Tuple, parseProjectionExpression);
+        const { items: values } = parseList(ListKind.Tuple, parseProjectionExpression);
         return {
             kind: SyntaxKind.ProjectionTupleExpression,
             values,
@@ -2476,10 +2492,11 @@ function createParser(code, options = {}) {
     }
     function createMissingTypeReference() {
         const pos = tokenPos();
+        const { items: args } = createEmptyList();
         return {
             kind: SyntaxKind.TypeReference,
             target: createMissingIdentifier(),
-            arguments: [],
+            arguments: args,
             ...finishNode(pos),
         };
     }
@@ -2492,6 +2509,12 @@ function createParser(code, options = {}) {
     function withSymbol(obj) {
         return obj;
     }
+    function createEmptyList(range = { pos: -1, end: -1 }) {
+        return {
+            items: [],
+            range,
+        };
+    }
     /**
      * Parse a delimited list of elements, including the surrounding open and
      * close punctuation
@@ -2508,13 +2531,17 @@ function createParser(code, options = {}) {
      * do not go through here.
      */
     function parseList(kind, parseItem) {
+        const r = createEmptyList();
         if (kind.open !== Token.None) {
-            parseExpected(kind.open);
+            const t = tokenPos();
+            if (parseExpected(kind.open)) {
+                mutate(r.range).pos = t;
+            }
         }
         if (kind.allowEmpty && parseOptional(kind.close)) {
-            return [];
+            mutate(r.range).end = previousTokenEnd;
+            return r;
         }
-        const items = [];
         while (true) {
             const startingPos = tokenPos();
             const { pos, docs, directives, decorators } = parseAnnotations({
@@ -2529,7 +2556,9 @@ function createParser(code, options = {}) {
                 // of file. Note, however, that we must parse a missing element if
                 // there were directives or decorators as we cannot drop those from
                 // the tree.
-                parseExpected(kind.close);
+                if (parseExpected(kind.close)) {
+                    mutate(r.range).end = previousTokenEnd;
+                }
                 break;
             }
             let item;
@@ -2541,12 +2570,13 @@ function createParser(code, options = {}) {
                 mutate(item).docs = docs;
                 mutate(item).directives = directives;
             }
-            items.push(item);
+            r.items.push(item);
             const delimiter = token();
             const delimiterPos = tokenPos();
             if (parseOptionalDelimiter(kind)) {
                 // Delimiter found: check if it's trailing.
                 if (parseOptional(kind.close)) {
+                    mutate(r.range).end = previousTokenEnd;
                     if (!kind.trailingDelimiterIsValid) {
                         error({
                             code: "trailing-token",
@@ -2570,6 +2600,7 @@ function createParser(code, options = {}) {
                 break;
             }
             else if (parseOptional(kind.close)) {
+                mutate(r.range).end = previousTokenEnd;
                 // If a list *is* surrounded by punctuation, then the list ends when we
                 // reach the close token.
                 break;
@@ -2580,7 +2611,9 @@ function createParser(code, options = {}) {
                 // assumption that the closing delimiter is missing. This check is
                 // duplicated from above to preempt the parseExpected(delimeter)
                 // below.
-                parseExpected(kind.close);
+                if (parseExpected(kind.close)) {
+                    mutate(r.range).end = previousTokenEnd;
+                }
                 break;
             }
             else {
@@ -2599,21 +2632,23 @@ function createParser(code, options = {}) {
                 //
                 // Simple repro: `model M { ]` would loop forever without this check.
                 //
-                parseExpected(kind.close);
+                if (parseExpected(kind.close)) {
+                    mutate(r.range).end = previousTokenEnd;
+                }
                 nextToken();
                 // remove the item that was entirely inserted by error recovery.
-                items.pop();
+                r.items.pop();
                 break;
             }
         }
-        return items;
+        return r;
     }
     /**
      * Parse a delimited list with surrounding open and close punctuation if the
      * open token is present. Otherwise, return an empty list.
      */
     function parseOptionalList(kind, parseItem) {
-        return token() === kind.open ? parseList(kind, parseItem) : [];
+        return token() === kind.open ? parseList(kind, parseItem) : createEmptyList();
     }
     function parseOptionalDelimiter(kind) {
         if (parseOptional(kind.delimiter)) {
@@ -3027,19 +3062,58 @@ function visitEach(cb, nodes) {
     }
     return;
 }
-export function getNodeAtPositionDetail(script, position, filter) {
-    const node = getNodeAtPosition(script, position, filter);
-    if (!node)
-        return undefined;
-    const char = script.file.text.charCodeAt(position);
-    const preChar = position >= 0 ? script.file.text.charCodeAt(position - 1) : NaN;
-    const nextChar = position < script.file.text.length ? script.file.text.charCodeAt(position + 1) : NaN;
+/**
+ * check whether a position belongs to a range (excluding the start and end pos)
+ * i.e. <range.pos>{<start to return true>...<end to return true>}<range.end>
+ *
+ * remark: if range.pos is -1 means no start point found, so return false
+ *         if range.end is -1 means no end point found, so return true if position is greater than range.pos
+ */
+export function positionInRange(position, range) {
+    return range.pos >= 0 && position > range.pos && (range.end === -1 || position < range.end);
+}
+export function getNodeAtPositionDetail(script, position, filter = () => true) {
+    const cur = getNodeAtPosition(script, position, (n) => filter(n, "cur"));
+    const input = script.file.text;
+    const char = input.charCodeAt(position);
+    const preChar = position >= 0 ? input.charCodeAt(position - 1) : NaN;
+    const nextChar = position < input.length ? input.charCodeAt(position + 1) : NaN;
+    let inTrivia = false;
+    let triviaStart;
+    let triviaEnd;
+    if (!cur || cur.kind !== SyntaxKind.StringLiteral) {
+        const { char: cp } = codePointBefore(input, position);
+        if (!cp || !isIdentifierContinue(cp)) {
+            triviaEnd = skipTrivia(input, position);
+            triviaStart = skipTriviaBackward(script, position) + 1;
+            inTrivia = triviaEnd !== position;
+        }
+    }
+    if (!inTrivia) {
+        const beforeId = skipContinuousIdentifier(input, position, true /*isBackward*/);
+        triviaStart = skipTriviaBackward(script, beforeId) + 1;
+        const afterId = skipContinuousIdentifier(input, position, false /*isBackward*/);
+        triviaEnd = skipTrivia(input, afterId);
+    }
+    if (triviaStart === undefined || triviaEnd === undefined) {
+        compilerAssert(false, "unexpected, triviaStart and triviaEnd should be defined");
+    }
     return {
-        node,
-        position,
+        node: cur,
+        char,
         preChar,
         nextChar,
-        char,
+        position,
+        inTrivia,
+        triviaStartPosition: triviaStart,
+        triviaEndPosition: triviaEnd,
+        getPositionDetailBeforeTrivia: () => {
+            // getNodeAtPosition will also include the 'node.end' position which is the triviaStart pos
+            return getNodeAtPositionDetail(script, triviaStart, (n) => filter(n, "pre"));
+        },
+        getPositionDetailAfterTrivia: () => {
+            return getNodeAtPositionDetail(script, triviaEnd, (n) => filter(n, "post"));
+        },
     };
 }
 export function getNodeAtPosition(script, position, filter = (node) => true) {
@@ -3109,7 +3183,10 @@ function isBlocklessNamespace(node) {
     }
     return node.statements === undefined;
 }
-export function getFirstAncestor(node, test) {
+export function getFirstAncestor(node, test, includeSelf = false) {
+    if (includeSelf && test(node)) {
+        return node;
+    }
     for (let n = node.parent; n; n = n.parent) {
         if (test(n)) {
             return n;