@typespec/compiler 1.16.0 → 1.17.0-dev.10

package/dist/src/core/parser.js

@@ -3,7 +3,7 @@ import { codePointBefore, isIdentifierContinue, trim } from "./charcode.js";
 import { compilerAssert } from "./diagnostics.js";
 import { createDiagnostic } from "./messages.js";
 import { modifiersToFlags } from "./modifiers.js";
-import { createScanner, isComment, isKeyword, isModifier, isPunctuation, isReservedKeyword, isStatementKeyword, isTrivia, skipContinuousIdentifier, skipTrivia, skipTriviaBackward, Token, TokenDisplay, TokenFlags, } from "./scanner.js";
+import { createScanner, isComment, isDeclarationExpressionStatementKeyword, isKeyword, isModifier, isPunctuation, isReservedKeyword, isStatementKeyword, isTrivia, skipContinuousIdentifier, skipTrivia, skipTriviaBackward, Token, TokenDisplay, TokenFlags, } from "./scanner.js";
 import { IdentifierKind, ModifierFlags, NodeFlags, SyntaxKind } from "./types.js";
 /**
  * The fixed set of options for each of the kinds of delimited lists in TypeSpec.
@@ -26,10 +26,12 @@ var ListKind;
     ListKind.DecoratorArguments = {
         ...ListKind.OperationParameters,
         invalidAnnotationTarget: "expression",
+        allowDeclarationExpression: true,
     };
     ListKind.FunctionArguments = {
         ...ListKind.OperationParameters,
         invalidAnnotationTarget: "expression",
+        allowDeclarationExpression: true,
     };
     ListKind.ModelProperties = {
         ...PropertiesBase,
@@ -91,12 +93,14 @@ var ListKind;
     };
     ListKind.TemplateArguments = {
         ...ListKind.TemplateParameters,
+        allowDeclarationExpression: true,
     };
     ListKind.CallArguments = {
         ...ExpresionsBase,
         allowEmpty: true,
         open: Token.OpenParen,
         close: Token.CloseParen,
+        allowDeclarationExpression: true,
     };
     ListKind.Heritage = {
         ...ExpresionsBase,
@@ -109,6 +113,7 @@ var ListKind;
         allowEmpty: true,
         open: Token.OpenBracket,
         close: Token.CloseBracket,
+        allowDeclarationExpression: true,
     };
     ListKind.ArrayLiteral = {
         ...ExpresionsBase,
@@ -466,6 +471,25 @@ function createParser(code, options = {}) {
         }
         return undefined;
     }
+    function parseUnionDeclarationExpression(pos, decorators = []) {
+        parseExpected(Token.UnionKeyword);
+        const id = parseOptionalDeclarationExpressionIdentifier();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        const optionalExtends = parseOptionalUnionExtends();
+        const { items: options } = parseList(ListKind.UnionVariants, parseUnionVariant);
+        return {
+            kind: SyntaxKind.UnionDeclarationExpression,
+            id,
+            templateParameters,
+            templateParametersRange,
+            extends: optionalExtends,
+            decorators,
+            modifiers: [],
+            modifierFlags: ModifierFlags.None,
+            options,
+            ...finishNode(pos),
+        };
+    }
     function parseIdOrValueForVariant() {
         const nextToken = token();
         let id;
@@ -611,35 +635,74 @@ function createParser(code, options = {}) {
         parseExpected(Token.ModelKeyword);
         const id = parseIdentifier();
         const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        const { extends: optionalExtends, is: optionalIs, properties, bodyRange } = parseModelBody();
+        return {
+            kind: SyntaxKind.ModelStatement,
+            id,
+            extends: optionalExtends,
+            is: optionalIs,
+            templateParameters,
+            templateParametersRange,
+            decorators,
+            properties,
+            bodyRange,
+            modifiers,
+            modifierFlags: modifiersToFlags(modifiers),
+            ...finishNode(pos),
+        };
+    }
+    function parseModelDeclarationExpression(pos, decorators = []) {
+        parseExpected(Token.ModelKeyword);
+        const id = parseOptionalDeclarationExpressionIdentifier();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        const { extends: optionalExtends, is: optionalIs, properties, bodyRange, } = parseModelBody(true);
+        return {
+            kind: SyntaxKind.ModelDeclarationExpression,
+            id,
+            extends: optionalExtends,
+            is: optionalIs,
+            templateParameters,
+            templateParametersRange,
+            decorators,
+            properties,
+            bodyRange,
+            modifiers: [],
+            modifierFlags: ModifierFlags.None,
+            ...finishNode(pos),
+        };
+    }
+    function parseModelBody(inExpressionPosition = false) {
         expectTokenIsOneOf(Token.OpenBrace, Token.Equals, Token.ExtendsKeyword, Token.IsKeyword);
         const optionalExtends = parseOptionalModelExtends();
         const optionalIs = optionalExtends ? undefined : parseOptionalModelIs();
         let propDetail = createEmptyList();
         if (optionalIs) {
-            const tok = expectTokenIsOneOf(Token.Semicolon, Token.OpenBrace);
-            if (tok === Token.Semicolon) {
-                nextToken();
+            if (inExpressionPosition) {
+                // In expression position there is no `;` terminator (it belongs to the enclosing
+                // construct): only parse a `{ ... }` body when present, otherwise the model has no
+                // own properties.
+                if (token() === Token.OpenBrace) {
+                    propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+                }
             }
             else {
-                propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+                const tok = expectTokenIsOneOf(Token.Semicolon, Token.OpenBrace);
+                if (tok === Token.Semicolon) {
+                    nextToken();
+                }
+                else {
+                    propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
+                }
             }
         }
         else {
             propDetail = parseList(ListKind.ModelProperties, parseModelPropertyOrSpread);
         }
         return {
-            kind: SyntaxKind.ModelStatement,
-            id,
             extends: optionalExtends,
             is: optionalIs,
-            templateParameters,
-            templateParametersRange,
-            decorators,
             properties: propDetail.items,
             bodyRange: propDetail.range,
-            modifiers,
-            modifierFlags: modifiersToFlags(modifiers),
-            ...finishNode(pos),
         };
     }
     function parseOptionalModelExtends() {
@@ -789,13 +852,38 @@ function createParser(code, options = {}) {
             ...finishNode(pos),
         };
     }
+    function parseScalarDeclarationExpression(pos, decorators = []) {
+        parseExpected(Token.ScalarKeyword);
+        const id = parseOptionalDeclarationExpressionIdentifier();
+        const { items: templateParameters, range: templateParametersRange } = parseTemplateParameterList();
+        const optionalExtends = parseOptionalScalarExtends();
+        const { items: members, range: bodyRange } = parseScalarMembers(true);
+        return {
+            kind: SyntaxKind.ScalarDeclarationExpression,
+            id,
+            templateParameters,
+            templateParametersRange,
+            extends: optionalExtends,
+            members,
+            bodyRange,
+            decorators,
+            modifiers: [],
+            modifierFlags: ModifierFlags.None,
+            ...finishNode(pos),
+        };
+    }
     function parseOptionalScalarExtends() {
         if (parseOptional(Token.ExtendsKeyword)) {
             return parseReferenceExpression();
         }
         return undefined;
     }
-    function parseScalarMembers() {
+    function parseScalarMembers(inExpressionPosition = false) {
+        // In expression position there is no `;` terminator: only parse a `{ ... }` body
+        // when present, otherwise the scalar has no members.
+        if (inExpressionPosition && token() !== Token.OpenBrace) {
+            return createEmptyList();
+        }
         if (token() === Token.Semicolon) {
             nextToken();
             return createEmptyList();
@@ -830,6 +918,20 @@ function createParser(code, options = {}) {
             ...finishNode(pos),
         };
     }
+    function parseEnumDeclarationExpression(pos, decorators = []) {
+        parseExpected(Token.EnumKeyword);
+        const id = parseOptionalDeclarationExpressionIdentifier();
+        const { items: members } = parseList(ListKind.EnumMembers, parseEnumMemberOrSpread);
+        return {
+            kind: SyntaxKind.EnumDeclarationExpression,
+            id,
+            decorators,
+            modifiers: [],
+            modifierFlags: ModifierFlags.None,
+            members,
+            ...finishNode(pos),
+        };
+    }
     function parseEnumMemberOrSpread(pos, decorators) {
         return token() === Token.Ellipsis
             ? parseEnumSpreadMember(pos, decorators)
@@ -1277,6 +1379,35 @@ function createParser(code, options = {}) {
         }
         return base;
     }
+    /**
+     * Parse a declaration used in expression position (`model`/`enum`/`union`/`scalar`).
+     * {@link pos} should point at the start of the node (the first decorator's `@` when
+     * {@link decorators} are provided, otherwise the keyword). A leading doc comment
+     * ({@link docs}) is attached to the node so it is applied just like an `@doc` decorator.
+     */
+    function parseDeclarationExpression(pos, decorators = [], docs = []) {
+        let node;
+        switch (token()) {
+            case Token.ModelKeyword:
+                node = parseModelDeclarationExpression(pos, decorators);
+                break;
+            case Token.EnumKeyword:
+                node = parseEnumDeclarationExpression(pos, decorators);
+                break;
+            case Token.UnionKeyword:
+                node = parseUnionDeclarationExpression(pos, decorators);
+                break;
+            case Token.ScalarKeyword:
+                node = parseScalarDeclarationExpression(pos, decorators);
+                break;
+            default:
+                compilerAssert(false, "Expected a declaration expression keyword.");
+        }
+        if (docs.length > 0) {
+            mutate(node).docs = docs;
+        }
+        return node;
+    }
     function parsePrimaryExpression() {
         while (true) {
             switch (token()) {
@@ -1297,14 +1428,36 @@ function createParser(code, options = {}) {
                     return parseNumericLiteral();
                 case Token.OpenBrace:
                     return parseModelExpression();
+                case Token.ModelKeyword:
+                case Token.EnumKeyword:
+                case Token.UnionKeyword:
+                case Token.ScalarKeyword: {
+                    // A leading doc comment applies to the declaration expression, just like `@doc`.
+                    // `pos` starts at the doc comment so the resulting node span includes it.
+                    const [pos, docs] = parseDocList();
+                    return parseDeclarationExpression(pos, [], docs);
+                }
                 case Token.OpenBracket:
                     return parseTupleExpression();
                 case Token.OpenParen:
                     return parseParenthesizedExpression();
-                case Token.At:
+                case Token.At: {
+                    // Capture a doc comment that precedes the decorators before they are parsed
+                    // (parsing decorators advances the scanner and clears the pending doc range).
+                    const [docPos, docs] = parseDocList();
+                    // `pos` starts at the doc comment (if any) or the `@` so the resulting node
+                    // span includes the docs and decorators.
+                    const pos = docs.length > 0 ? docPos : tokenPos();
                     const decorators = parseDecoratorList();
+                    // Decorators are only valid in expression position when they decorate a
+                    // declaration expression (the keyword forms below). `pos` starts at the `@`
+                    // so the resulting node span includes the decorators.
+                    if (isDeclarationExpressionStatementKeyword(token())) {
+                        return parseDeclarationExpression(pos, decorators, docs);
+                    }
                     reportInvalidDecorators(decorators, "expression");
                     continue;
+                }
                 case Token.Hash:
                     const directives = parseDirectiveList();
                     reportInvalidDirective(directives, "expression");
@@ -1533,6 +1686,13 @@ function createParser(code, options = {}) {
             ...finishNode(pos),
         };
     }
+    /**
+     * Parse the optional identifier of a declaration used in expression position. The name
+     * is only parsed when actually present; an anonymous declaration expression has no `id`.
+     */
+    function parseOptionalDeclarationExpressionIdentifier() {
+        return token() === Token.Identifier ? parseIdentifier() : undefined;
+    }
     function parseIdentifier(options) {
         if (isKeyword(token())) {
             if (!(isModifier(token()) && options?.allowReservedIdentifier)) {
@@ -1721,15 +1881,26 @@ function createParser(code, options = {}) {
         if (docRanges.length === 0 || options.docs === false) {
             return [tokenPos(), []];
         }
+        return [docRanges[0].pos, parseDocRanges(docRanges)];
+    }
+    /**
+     * Parse the given doc comment ranges into doc nodes. Unlike {@link parseDocList} this does
+     * not read the pending {@link docRanges}, so it can be used to attach doc comments that were
+     * captured earlier (e.g. before a list element's decorators were parsed).
+     */
+    function parseDocRanges(ranges) {
+        if (ranges.length === 0 || options.docs === false) {
+            return [];
+        }
         const docs = [];
-        for (const range of docRanges) {
+        for (const range of ranges) {
             const doc = parseRange(ParseMode.Doc, innerDocRange(range), () => parseDoc(range));
             docs.push(doc);
             if (range.comment) {
                 mutate(range.comment).parsedAsDocs = true;
             }
         }
-        return [docRanges[0].pos, docs];
+        return docs;
     }
     function parseDoc(range) {
         const content = [];
@@ -2065,14 +2236,28 @@ function createParser(code, options = {}) {
         }
         while (true) {
             const startingPos = tokenPos();
+            // Capture any pending doc comment before `parseAnnotations` parses (and clears the
+            // pending range for) decorators, so a doc comment preceding a decorated declaration
+            // expression element can still be attached to it below.
+            const pendingDocRanges = kind.allowDeclarationExpression ? docRanges : undefined;
             const { pos, docs, directives, decorators } = parseAnnotations({
                 skipParsingDocNodes: Boolean(kind.invalidAnnotationTarget),
             });
+            // A declaration expression element (e.g. `enum { a }`) can carry decorators (and a
+            // doc comment) even in an expression list. Attach them to the declaration expression
+            // instead of rejecting them as an invalid annotation target.
+            const declarationExpressionWithDecorators = kind.allowDeclarationExpression &&
+                decorators.length > 0 &&
+                isDeclarationExpressionStatementKeyword(token());
             if (kind.invalidAnnotationTarget) {
-                reportInvalidDecorators(decorators, kind.invalidAnnotationTarget);
                 reportInvalidDirective(directives, kind.invalidAnnotationTarget);
+                if (!declarationExpressionWithDecorators) {
+                    reportInvalidDecorators(decorators, kind.invalidAnnotationTarget);
+                }
             }
-            if (directives.length === 0 && decorators.length === 0 && atEndOfListWithError(kind)) {
+            if (directives.length === 0 &&
+                decorators.length === 0 &&
+                atEndOfListWithError(kind, { atElementStart: true })) {
                 // Error recovery: end surrounded list at statement keyword or end
                 // of file. Note, however, that we must parse a missing element if
                 // there were directives or decorators as we cannot drop those from
@@ -2083,7 +2268,14 @@ function createParser(code, options = {}) {
                 break;
             }
             let item;
-            if (kind.invalidAnnotationTarget) {
+            if (declarationExpressionWithDecorators) {
+                const declDocs = pendingDocRanges ? parseDocRanges(pendingDocRanges) : [];
+                // When a doc comment precedes the decorators, start the node span at the doc
+                // comment so the doc nodes stay within the parent node's range.
+                const declPos = declDocs.length > 0 ? pendingDocRanges[0].pos : pos;
+                item = parseDeclarationExpression(declPos, decorators, declDocs);
+            }
+            else if (kind.invalidAnnotationTarget) {
                 item = parseItem();
             }
             else {
@@ -2114,7 +2306,7 @@ function createParser(code, options = {}) {
                 // reach the close token.
                 break;
             }
-            else if (atEndOfListWithError(kind)) {
+            else if (atEndOfListWithError(kind, { atElementStart: false })) {
                 // Error recovery: If a list *is* surrounded by punctuation, then
                 // the list ends at statement keyword or end-of-file under the
                 // assumption that the closing delimiter is missing. This check is
@@ -2172,10 +2364,28 @@ function createParser(code, options = {}) {
         }
         return false;
     }
-    function atEndOfListWithError(kind) {
+    /**
+     * Whether the parser is at a position where the current list (when surrounded by
+     * punctuation) should be ended under the assumption that its closing token is missing —
+     * i.e. the current token is a statement keyword or end-of-file.
+     *
+     * @param atElementStart Whether we are about to parse a new element (`true`) or have just
+     * parsed one and found neither a delimiter nor the close token (`false`). A
+     * declaration-expression keyword (`model`/`enum`/`union`/`scalar`) is a valid element
+     * start in lists that {@link ListKind.allowDeclarationExpression allow declaration
+     * expressions}, so it must not end the list at the start of an element. After an element,
+     * a declaration-expression keyword cannot start a new element (a delimiter is required
+     * first), so it is treated like any other statement keyword and ends the list — restoring
+     * error recovery for an unclosed delimiter (e.g. `@foo(model X {}` followed by a
+     * statement).
+     */
+    function atEndOfListWithError(kind, { atElementStart }) {
         return (kind.close !== Token.None &&
             (isStatementKeyword(token()) || token() === Token.EndOfFile) &&
-            token() !== kind.allowedStatementKeyword);
+            token() !== kind.allowedStatementKeyword &&
+            !(atElementStart &&
+                kind.allowDeclarationExpression &&
+                isDeclarationExpressionStatementKeyword(token())));
     }
     function parseEmptyStatement(pos) {
         parseExpected(Token.Semicolon);
@@ -2402,6 +2612,13 @@ export function visitChildren(node, cb) {
                 visitNode(cb, node.extends) ||
                 visitNode(cb, node.is) ||
                 visitEach(cb, node.properties));
+        case SyntaxKind.ModelDeclarationExpression:
+            return (visitEach(cb, node.decorators) ||
+                visitNode(cb, node.id) ||
+                visitEach(cb, node.templateParameters) ||
+                visitNode(cb, node.extends) ||
+                visitNode(cb, node.is) ||
+                visitEach(cb, node.properties));
         case SyntaxKind.ScalarStatement:
             return (visitEach(cb, node.modifiers) ||
                 visitEach(cb, node.decorators) ||
@@ -2409,6 +2626,12 @@ export function visitChildren(node, cb) {
                 visitEach(cb, node.templateParameters) ||
                 visitEach(cb, node.members) ||
                 visitNode(cb, node.extends));
+        case SyntaxKind.ScalarDeclarationExpression:
+            return (visitEach(cb, node.decorators) ||
+                visitNode(cb, node.id) ||
+                visitEach(cb, node.templateParameters) ||
+                visitEach(cb, node.members) ||
+                visitNode(cb, node.extends));
         case SyntaxKind.ScalarConstructor:
             return visitNode(cb, node.id) || visitEach(cb, node.parameters);
         case SyntaxKind.UnionStatement:
@@ -2418,6 +2641,12 @@ export function visitChildren(node, cb) {
                 visitEach(cb, node.templateParameters) ||
                 visitNode(cb, node.extends) ||
                 visitEach(cb, node.options));
+        case SyntaxKind.UnionDeclarationExpression:
+            return (visitEach(cb, node.decorators) ||
+                visitNode(cb, node.id) ||
+                visitEach(cb, node.templateParameters) ||
+                visitNode(cb, node.extends) ||
+                visitEach(cb, node.options));
         case SyntaxKind.UnionVariant:
             return visitEach(cb, node.decorators) || visitNode(cb, node.id) || visitNode(cb, node.value);
         case SyntaxKind.EnumStatement:
@@ -2425,6 +2654,8 @@ export function visitChildren(node, cb) {
                 visitEach(cb, node.decorators) ||
                 visitNode(cb, node.id) ||
                 visitEach(cb, node.members));
+        case SyntaxKind.EnumDeclarationExpression:
+            return (visitEach(cb, node.decorators) || visitNode(cb, node.id) || visitEach(cb, node.members));
         case SyntaxKind.EnumMember:
             return visitEach(cb, node.decorators) || visitNode(cb, node.id) || visitNode(cb, node.value);
         case SyntaxKind.EnumSpreadMember:
@@ -2698,6 +2929,9 @@ export function getIdentifierContext(id) {
                 case SyntaxKind.ModelStatement:
                     kind = IdentifierKind.ModelStatementProperty;
                     break;
+                case SyntaxKind.ModelDeclarationExpression:
+                    kind = IdentifierKind.ModelStatementProperty;
+                    break;
                 default:
                     compilerAssert("false", "ModelProperty with unexpected parent kind.");
                     kind =