@mpeggroup/mpeg-sdl-parser 4.0.0

package/src/ast/build/node/build-utf-string-literal.ts

@@ -0,0 +1,110 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import { StringLiteral } from "../../node/string-literal.ts";
+import { StringLiteralKind } from "../../node/enum/string-literal-kind.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import { Token } from "../../node/token.ts";
+import { fetchOneToManyList } from "../util/fetch-node.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+import { getLocationFromTextPosition } from "../../../util/location-utils.ts";
+import type { TreeCursor } from "@lezer/common";
+import { Text } from "@codemirror/state";
+import { isToken } from "../../util/types.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+
+const ESCAPED_BACKSLASH_REGEX = /\\\\/g;
+const ESCAPED_DOUBLE_QUOTE_REGEX = /\\"/g;
+const FOUR_HEXADECIMAL_UNIVERSAL_CHARACTER_NAME_REGEX = /\\u[0-9A-F]{4}/g;
+const EIGHT_HEXADECIMAL_UNIVERSAL_CHARACTER_NAME_REGEX = /\\U[0-9A-F]{8}/g;
+
+function getStringValueFromLiteralText(
+  literalText: string,
+  cursor: TreeCursor,
+  text: Text,
+): string {
+  let value = literalText.replaceAll(ESCAPED_DOUBLE_QUOTE_REGEX, "'")
+    .replaceAll(ESCAPED_BACKSLASH_REGEX, "\\");
+
+  value = value.replaceAll(
+    FOUR_HEXADECIMAL_UNIVERSAL_CHARACTER_NAME_REGEX,
+    (match) => {
+      if (!match.startsWith("\\u")) {
+        throw new InternalScannerError(
+          `Missing prefix "\\u" in universal character name: ${match}`,
+          getLocationFromTextPosition(text, cursor.from),
+        );
+      }
+
+      const codePoint = parseInt(match.substring(2), 16);
+
+      if (isNaN(codePoint)) {
+        throw new InternalScannerError(
+          `Unable to convert universal character name: ${codePoint} to code point number`,
+          getLocationFromTextPosition(text, cursor.from),
+        );
+      }
+
+      return String.fromCodePoint(0x1234);
+    },
+  );
+
+  value = value.replaceAll(
+    EIGHT_HEXADECIMAL_UNIVERSAL_CHARACTER_NAME_REGEX,
+    (match) => {
+      if (!match.startsWith("\\U")) {
+        throw new InternalScannerError(
+          `Missing prefix "\\U" in universal character name: ${match}`,
+          getLocationFromTextPosition(text, cursor.from),
+        );
+      }
+
+      const codePoint = parseInt(match.substring(2), 16);
+
+      if (isNaN(codePoint)) {
+        throw new InternalScannerError(
+          `Unable to convert universal character name: ${match} to code point number`,
+          getLocationFromTextPosition(text, cursor.from),
+        );
+      }
+
+      return String.fromCodePoint(codePoint);
+    },
+  );
+
+  return value;
+}
+
+export function buildUtfStringLiteral(
+  buildContext: BuildContext,
+): StringLiteral {
+  const literals = fetchOneToManyList<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    [
+      TokenKind.UTF_PREFIX,
+      TokenKind.DOUBLE_QUOTE,
+      TokenKind.UTF_STRING_LITERAL_CHARACTERS,
+    ],
+  );
+
+  let value = "";
+
+  for (const literal of literals) {
+    if (
+      isToken(literal) &&
+      (literal.tokenKind === TokenKind.UTF_STRING_LITERAL_CHARACTERS)
+    ) {
+      value += getStringValueFromLiteralText(
+        literal.text,
+        buildContext.cursor,
+        buildContext.text,
+      );
+    }
+  }
+
+  return new StringLiteral(
+    StringLiteralKind.UTF,
+    value,
+    literals,
+    [...literals],
+  );
+}

package/src/ast/build/node/build-while-statement.ts

@@ -0,0 +1,55 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import type { AbstractExpression } from "../../node/abstract-expression.ts";
+import type { NumberLiteral } from "../../node/number-literal.ts";
+import type { Identifier } from "../../node/identifier.ts";
+import { WhileStatement } from "../../node/while-statement.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { Token } from "../../node/token.ts";
+import { fetchRequiredNode } from "../util/fetch-node.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+import type { AbstractStatement } from "../../node/abstract-statement.ts";
+
+export function buildWhileStatement(
+  buildContext: BuildContext,
+): WhileStatement {
+  const whileKeyword = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.WHILE,
+  );
+  const openParenthesisPunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.OPEN_PARENTHESIS,
+  );
+  const condition = fetchRequiredNode<
+    AbstractExpression | Identifier | NumberLiteral
+  >(
+    buildContext,
+    [NodeKind.EXPRESSION, NodeKind.IDENTIFIER, NodeKind.NUMBER_LITERAL],
+  );
+  const closeParenthesisPunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.CLOSE_PARENTHESIS,
+  );
+  const statement = fetchRequiredNode<AbstractStatement>(
+    buildContext,
+    NodeKind.STATEMENT,
+  );
+
+  return new WhileStatement(
+    whileKeyword,
+    openParenthesisPunctuator,
+    condition,
+    closeParenthesisPunctuator,
+    statement,
+    [
+      whileKeyword,
+      openParenthesisPunctuator,
+      condition,
+      closeParenthesisPunctuator,
+      statement,
+    ],
+  );
+}

package/src/ast/build/util/array-dimension-kind-by-token-type-id-map.ts

@@ -0,0 +1,167 @@
+import { ArrayDimensionKind } from "../../node/enum/array-dimension-kind.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../../lezer/parser.terms.ts";
+
+export const arrayDimensionKindByTokenTypeId: Map<number, ArrayDimensionKind> =
+  new Map();
+
+/**
+ * Map parsed token type ID to AST token type enum.
+ * Exhaustive switch statement to ensure all token type IDs are handled.
+ */
+Object.values(TokenTypeId).forEach((tokenTypeId) => {
+  switch (tokenTypeId) {
+    case TokenTypeId.ExplicitArrayDimension:
+      arrayDimensionKindByTokenTypeId.set(
+        tokenTypeId,
+        ArrayDimensionKind.EXPLICIT,
+      );
+      break;
+    case TokenTypeId.ImplicitArrayDimension:
+      arrayDimensionKindByTokenTypeId.set(
+        tokenTypeId,
+        ArrayDimensionKind.IMPLICIT,
+      );
+      break;
+    case TokenTypeId.PartialArrayDimension:
+      arrayDimensionKindByTokenTypeId.set(
+        tokenTypeId,
+        ArrayDimensionKind.PARTIAL,
+      );
+      break;
+    case TokenTypeId.AlignedModifier:
+    case TokenTypeId.AggregateOutputValue:
+    case TokenTypeId.ArrayDefinition:
+    case TokenTypeId.ClassDefinition:
+    case TokenTypeId.ClassDeclaration:
+    case TokenTypeId.CompoundStatement:
+    case TokenTypeId.ComputedArrayDefinition:
+    case TokenTypeId.ComputedElementaryTypeDefinition:
+    case TokenTypeId.DoStatement:
+    case TokenTypeId.ElementaryTypeDefinition:
+    case TokenTypeId.ExpressionStatement:
+    case TokenTypeId.ForStatement:
+    case TokenTypeId.IfStatement:
+    case TokenTypeId.MapDeclaration:
+    case TokenTypeId.StringDefinition:
+    case TokenTypeId.SwitchStatement:
+    case TokenTypeId.WhileStatement:
+    case TokenTypeId.AssignmentExpression:
+    case TokenTypeId.BinaryExpression:
+    case TokenTypeId.LengthofExpression:
+    case TokenTypeId.UnaryExpression:
+    case TokenTypeId.ArrayElementAccess:
+    case TokenTypeId.Base64StringLiteral:
+    case TokenTypeId.UtfStringLiteral:
+    case TokenTypeId.BitModifier:
+    case TokenTypeId.CaseClause:
+    case TokenTypeId.ClassId:
+    case TokenTypeId.ClassIdRange:
+    case TokenTypeId.ExtendedClassIdRange:
+    case TokenTypeId.ClassMemberAccess:
+    case TokenTypeId.DefaultClause:
+    case TokenTypeId.ElementaryType:
+    case TokenTypeId.ElementaryTypeOutputValue:
+    case TokenTypeId.ExpandableModifier:
+    case TokenTypeId.ExtendsModifier:
+    case TokenTypeId.LengthAttribute:
+    case TokenTypeId.MapDefinition:
+    case TokenTypeId.MapEntry:
+    case TokenTypeId.MultipleCharacterLiteral:
+    case TokenTypeId.Parameter:
+    case TokenTypeId.ParameterList:
+    case TokenTypeId.ParameterValueList:
+    case TokenTypeId.Specification:
+    case TokenTypeId._break:
+    case TokenTypeId._case:
+    case TokenTypeId._class:
+    case TokenTypeId._const:
+    case TokenTypeId._default:
+    case TokenTypeId._do:
+    case TokenTypeId._else:
+    case TokenTypeId._extends:
+    case TokenTypeId._for:
+    case TokenTypeId._if:
+    case TokenTypeId._switch:
+    case TokenTypeId._while:
+    case TokenTypeId.abstract:
+    case TokenTypeId.Addition:
+    case TokenTypeId.AlignmentBitCount8:
+    case TokenTypeId.AlignmentBitCount16:
+    case TokenTypeId.AlignmentBitCount32:
+    case TokenTypeId.AlignmentBitCount64:
+    case TokenTypeId.AlignmentBitCount128:
+    case TokenTypeId.aligned:
+    case TokenTypeId.Assignment:
+    case TokenTypeId.base64string:
+    case TokenTypeId.Base64StringLiteralCharacters:
+    case TokenTypeId.BinaryLiteral:
+    case TokenTypeId.bit:
+    case TokenTypeId.BitwiseAnd:
+    case TokenTypeId.BitwiseOr:
+    case TokenTypeId.BitwiseShiftLeft:
+    case TokenTypeId.BitwiseShiftRight:
+    case TokenTypeId.CloseBrace:
+    case TokenTypeId.CloseBracket:
+    case TokenTypeId.CloseParenthesis:
+    case TokenTypeId.Colon:
+    case TokenTypeId.Comma:
+    case TokenTypeId.Comment:
+    case TokenTypeId.computed:
+    case TokenTypeId.DecimalLiteral:
+    case TokenTypeId.Division:
+    case TokenTypeId.DoubleQuote:
+    case TokenTypeId.expandable:
+    case TokenTypeId.float:
+    case TokenTypeId.FloatingPointLiteral:
+    case TokenTypeId.HexadecimalLiteral:
+    case TokenTypeId.Identifier:
+    case TokenTypeId.int:
+    case TokenTypeId.IntegerLiteral:
+    case TokenTypeId.legacy:
+    case TokenTypeId.lengthof:
+    case TokenTypeId.LogicalAnd:
+    case TokenTypeId.LogicalOr:
+    case TokenTypeId.LookAhead:
+    case TokenTypeId.map:
+    case TokenTypeId.Modulus:
+    case TokenTypeId.MultipleCharacterLiteralCharacters:
+    case TokenTypeId.Multiplication:
+    case TokenTypeId.OpenBrace:
+    case TokenTypeId.OpenBracket:
+    case TokenTypeId.OpenParenthesis:
+    case TokenTypeId.Period:
+    case TokenTypeId.PostfixDecrement:
+    case TokenTypeId.PostfixIncrement:
+    case TokenTypeId.RangeOperator:
+    case TokenTypeId.reserved:
+    case TokenTypeId.RelationalEqual:
+    case TokenTypeId.RelationalGreaterThan:
+    case TokenTypeId.RelationalGreaterThanOrEqual:
+    case TokenTypeId.RelationalLessThan:
+    case TokenTypeId.RelationalLessThanOrEqual:
+    case TokenTypeId.RelationalNotEqual:
+    case TokenTypeId.Semicolon:
+    case TokenTypeId.SingleQuote:
+    case TokenTypeId.Subtraction:
+    case TokenTypeId.UnaryNegation:
+    case TokenTypeId.UnaryPlus:
+    case TokenTypeId.unsigned:
+    case TokenTypeId.utf16string:
+    case TokenTypeId.utf8list:
+    case TokenTypeId.utf8string:
+    case TokenTypeId.utfstring:
+    case TokenTypeId.UtfPrefix:
+    case TokenTypeId.UtfStringLiteralCharacters:
+    case TokenTypeId.Whitespace:
+      break;
+    default: {
+      const exhaustiveCheck: never = tokenTypeId;
+      throw new InternalScannerError(
+        "Unreachable code reached, tokenTypeId == " + exhaustiveCheck,
+      );
+    }
+  }
+});

package/src/ast/build/util/binary-operator-kind-by-token-type-id-map.ts

@@ -0,0 +1,152 @@
+import type { BinaryOperatorKind } from "../../node/enum/binary-operator-kind.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../../lezer/parser.terms.ts";
+
+export const binaryOperatorKindByTokenTypeId: Map<number, BinaryOperatorKind> =
+  new Map();
+
+/**
+ * Map parsed token type ID to AST token type enum.
+ * Exhaustive switch statement to ensure all token type IDs are handled.
+ */
+Object.values(TokenTypeId).forEach((tokenTypeId) => {
+  switch (tokenTypeId) {
+    case TokenTypeId.ArrayDefinition:
+    case TokenTypeId.ComputedArrayDefinition:
+    case TokenTypeId.ComputedElementaryTypeDefinition:
+    case TokenTypeId.ClassDeclaration:
+    case TokenTypeId.ClassDefinition:
+    case TokenTypeId.CompoundStatement:
+    case TokenTypeId.DoStatement:
+    case TokenTypeId.ElementaryTypeDefinition:
+    case TokenTypeId.ExpressionStatement:
+    case TokenTypeId.ForStatement:
+    case TokenTypeId.IfStatement:
+    case TokenTypeId.MapDeclaration:
+    case TokenTypeId.MapDefinition:
+    case TokenTypeId.StringDefinition:
+    case TokenTypeId.SwitchStatement:
+    case TokenTypeId.WhileStatement:
+    case TokenTypeId.AlignedModifier:
+    case TokenTypeId.AggregateOutputValue:
+    case TokenTypeId.AssignmentExpression:
+    case TokenTypeId.BinaryExpression:
+    case TokenTypeId.LengthofExpression:
+    case TokenTypeId.UnaryExpression:
+    case TokenTypeId.ArrayElementAccess:
+    case TokenTypeId.Base64StringLiteral:
+    case TokenTypeId.UtfStringLiteral:
+    case TokenTypeId.BitModifier:
+    case TokenTypeId.CaseClause:
+    case TokenTypeId.ClassId:
+    case TokenTypeId.ClassIdRange:
+    case TokenTypeId.ExtendedClassIdRange:
+    case TokenTypeId.ClassMemberAccess:
+    case TokenTypeId.DefaultClause:
+    case TokenTypeId.ElementaryType:
+    case TokenTypeId.ElementaryTypeOutputValue:
+    case TokenTypeId.ExpandableModifier:
+    case TokenTypeId.ExplicitArrayDimension:
+    case TokenTypeId.ImplicitArrayDimension:
+    case TokenTypeId.PartialArrayDimension:
+    case TokenTypeId.ExtendsModifier:
+    case TokenTypeId.LengthAttribute:
+    case TokenTypeId.MapEntry:
+    case TokenTypeId.MultipleCharacterLiteral:
+    case TokenTypeId.Parameter:
+    case TokenTypeId.ParameterList:
+    case TokenTypeId.ParameterValueList:
+    case TokenTypeId.Specification:
+    case TokenTypeId._break:
+    case TokenTypeId._case:
+    case TokenTypeId._class:
+    case TokenTypeId._const:
+    case TokenTypeId._default:
+    case TokenTypeId._do:
+    case TokenTypeId._else:
+    case TokenTypeId._extends:
+    case TokenTypeId._for:
+    case TokenTypeId._if:
+    case TokenTypeId._switch:
+    case TokenTypeId._while:
+    case TokenTypeId.abstract:
+    case TokenTypeId.Addition:
+    case TokenTypeId.AlignmentBitCount8:
+    case TokenTypeId.AlignmentBitCount16:
+    case TokenTypeId.AlignmentBitCount32:
+    case TokenTypeId.AlignmentBitCount64:
+    case TokenTypeId.AlignmentBitCount128:
+    case TokenTypeId.aligned:
+    case TokenTypeId.Assignment:
+    case TokenTypeId.base64string:
+    case TokenTypeId.Base64StringLiteralCharacters:
+    case TokenTypeId.BinaryLiteral:
+    case TokenTypeId.bit:
+    case TokenTypeId.BitwiseAnd:
+    case TokenTypeId.BitwiseOr:
+    case TokenTypeId.BitwiseShiftLeft:
+    case TokenTypeId.BitwiseShiftRight:
+    case TokenTypeId.CloseBrace:
+    case TokenTypeId.CloseBracket:
+    case TokenTypeId.CloseParenthesis:
+    case TokenTypeId.Colon:
+    case TokenTypeId.Comma:
+    case TokenTypeId.Comment:
+    case TokenTypeId.computed:
+    case TokenTypeId.DecimalLiteral:
+    case TokenTypeId.Division:
+    case TokenTypeId.DoubleQuote:
+    case TokenTypeId.expandable:
+    case TokenTypeId.float:
+    case TokenTypeId.FloatingPointLiteral:
+    case TokenTypeId.HexadecimalLiteral:
+    case TokenTypeId.Identifier:
+    case TokenTypeId.int:
+    case TokenTypeId.IntegerLiteral:
+    case TokenTypeId.legacy:
+    case TokenTypeId.lengthof:
+    case TokenTypeId.LogicalAnd:
+    case TokenTypeId.LogicalOr:
+    case TokenTypeId.LookAhead:
+    case TokenTypeId.map:
+    case TokenTypeId.Modulus:
+    case TokenTypeId.MultipleCharacterLiteralCharacters:
+    case TokenTypeId.Multiplication:
+    case TokenTypeId.OpenBrace:
+    case TokenTypeId.OpenBracket:
+    case TokenTypeId.OpenParenthesis:
+    case TokenTypeId.Period:
+    case TokenTypeId.PostfixDecrement:
+    case TokenTypeId.PostfixIncrement:
+    case TokenTypeId.RangeOperator:
+    case TokenTypeId.reserved:
+    case TokenTypeId.RelationalEqual:
+    case TokenTypeId.RelationalGreaterThan:
+    case TokenTypeId.RelationalGreaterThanOrEqual:
+    case TokenTypeId.RelationalLessThan:
+    case TokenTypeId.RelationalLessThanOrEqual:
+    case TokenTypeId.RelationalNotEqual:
+    case TokenTypeId.Semicolon:
+    case TokenTypeId.SingleQuote:
+    case TokenTypeId.Subtraction:
+    case TokenTypeId.UnaryNegation:
+    case TokenTypeId.UnaryPlus:
+    case TokenTypeId.unsigned:
+    case TokenTypeId.utf16string:
+    case TokenTypeId.utf8list:
+    case TokenTypeId.utf8string:
+    case TokenTypeId.utfstring:
+    case TokenTypeId.UtfPrefix:
+    case TokenTypeId.UtfStringLiteralCharacters:
+    case TokenTypeId.Whitespace:
+      break;
+    default: {
+      const exhaustiveCheck: never = tokenTypeId;
+      throw new InternalScannerError(
+        "Unreachable code reached, tokenTypeId == " + exhaustiveCheck,
+      );
+    }
+  }
+});

package/src/ast/build/util/build-context.ts

@@ -0,0 +1,36 @@
+import { Text } from "@codemirror/state";
+import type { TreeCursor } from "@lezer/common";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import type { Trivia } from "../../node/trivia.ts";
+
+/**
+ * State information used during AST construction.
+ *
+ * @interface State
+ * @property {boolean} isEndOfSiblings - Indicates if there are no more sibling nodes to process.
+ * @property {AbstractNode} [nextNode] - The next node to be processed, if applicable.
+ * @property {string} indent - The current indentation level for logging purposes.
+ */
+export interface State {
+  isEndOfSiblings: boolean;
+  nextNode?: AbstractNode;
+  indent: string;
+}
+
+/**
+ * State storage while building the AST.
+ *
+ * @interface BuildContext
+ * @property {TreeCursor} cursor - The current position in the parse tree.
+ * @property {Text} text - The source text that was parsed.
+ * @property {boolean} lenient - Whether to build the AST leniently, allowing error nodes.
+ * @property {State[]} stateStack - A stack of states used during AST construction.
+ * @property {Trivia[]} [unconsumedTrivia] - Any trivia not consumed. This should only be present for trailing trivia after the root node.
+ */
+export interface BuildContext {
+  readonly cursor: TreeCursor;
+  readonly text: Text;
+  readonly lenient: boolean;
+  stateStack: State[];
+  unconsumedTrivia?: Trivia[];
+}

package/src/ast/build/util/class-id-kind-by-token-type-id-map.ts

@@ -0,0 +1,160 @@
+import { ClassIdKind } from "../../node/enum/class-id-kind.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../../lezer/parser.terms.ts";
+
+export const classIdKindByTokenTypeId: Map<number, ClassIdKind> = new Map();
+
+/**
+ * Map parsed token type ID to AST token type enum.
+ * Exhaustive switch statement to ensure all token type IDs are handled.
+ */
+Object.values(TokenTypeId).forEach((tokenTypeId) => {
+  switch (tokenTypeId) {
+    case TokenTypeId.ClassId:
+      classIdKindByTokenTypeId.set(tokenTypeId, ClassIdKind.SINGLE);
+      break;
+    case TokenTypeId.ClassIdRange:
+      classIdKindByTokenTypeId.set(tokenTypeId, ClassIdKind.RANGE);
+      break;
+    case TokenTypeId.ExtendedClassIdRange:
+      classIdKindByTokenTypeId.set(
+        tokenTypeId,
+        ClassIdKind.EXTENDED_RANGE,
+      );
+      break;
+    case TokenTypeId.AlignedModifier:
+    case TokenTypeId.AggregateOutputValue:
+    case TokenTypeId.ArrayDefinition:
+    case TokenTypeId.ClassDefinition:
+    case TokenTypeId.ClassDeclaration:
+    case TokenTypeId.CompoundStatement:
+    case TokenTypeId.ComputedArrayDefinition:
+    case TokenTypeId.ComputedElementaryTypeDefinition:
+    case TokenTypeId.DoStatement:
+    case TokenTypeId.ElementaryTypeDefinition:
+    case TokenTypeId.ExpressionStatement:
+    case TokenTypeId.ForStatement:
+    case TokenTypeId.IfStatement:
+    case TokenTypeId.MapDeclaration:
+    case TokenTypeId.StringDefinition:
+    case TokenTypeId.SwitchStatement:
+    case TokenTypeId.WhileStatement:
+    case TokenTypeId.AssignmentExpression:
+    case TokenTypeId.BinaryExpression:
+    case TokenTypeId.LengthofExpression:
+    case TokenTypeId.UnaryExpression:
+    case TokenTypeId.ArrayElementAccess:
+    case TokenTypeId.Base64StringLiteral:
+    case TokenTypeId.UtfStringLiteral:
+    case TokenTypeId.BitModifier:
+    case TokenTypeId.CaseClause:
+    case TokenTypeId.ClassMemberAccess:
+    case TokenTypeId.DefaultClause:
+    case TokenTypeId.ElementaryType:
+    case TokenTypeId.ElementaryTypeOutputValue:
+    case TokenTypeId.ExpandableModifier:
+    case TokenTypeId.ExplicitArrayDimension:
+    case TokenTypeId.ImplicitArrayDimension:
+    case TokenTypeId.PartialArrayDimension:
+    case TokenTypeId.ExtendsModifier:
+    case TokenTypeId.LengthAttribute:
+    case TokenTypeId.MapDefinition:
+    case TokenTypeId.MapEntry:
+    case TokenTypeId.MultipleCharacterLiteral:
+    case TokenTypeId.Parameter:
+    case TokenTypeId.ParameterList:
+    case TokenTypeId.ParameterValueList:
+    case TokenTypeId.Specification:
+    case TokenTypeId._break:
+    case TokenTypeId._case:
+    case TokenTypeId._class:
+    case TokenTypeId._const:
+    case TokenTypeId._default:
+    case TokenTypeId._do:
+    case TokenTypeId._else:
+    case TokenTypeId._extends:
+    case TokenTypeId._for:
+    case TokenTypeId._if:
+    case TokenTypeId._switch:
+    case TokenTypeId._while:
+    case TokenTypeId.abstract:
+    case TokenTypeId.Addition:
+    case TokenTypeId.AlignmentBitCount8:
+    case TokenTypeId.AlignmentBitCount16:
+    case TokenTypeId.AlignmentBitCount32:
+    case TokenTypeId.AlignmentBitCount64:
+    case TokenTypeId.AlignmentBitCount128:
+    case TokenTypeId.aligned:
+    case TokenTypeId.Assignment:
+    case TokenTypeId.base64string:
+    case TokenTypeId.Base64StringLiteralCharacters:
+    case TokenTypeId.BinaryLiteral:
+    case TokenTypeId.bit:
+    case TokenTypeId.BitwiseAnd:
+    case TokenTypeId.BitwiseOr:
+    case TokenTypeId.BitwiseShiftLeft:
+    case TokenTypeId.BitwiseShiftRight:
+    case TokenTypeId.CloseBrace:
+    case TokenTypeId.CloseBracket:
+    case TokenTypeId.CloseParenthesis:
+    case TokenTypeId.Colon:
+    case TokenTypeId.Comma:
+    case TokenTypeId.Comment:
+    case TokenTypeId.computed:
+    case TokenTypeId.DecimalLiteral:
+    case TokenTypeId.Division:
+    case TokenTypeId.DoubleQuote:
+    case TokenTypeId.expandable:
+    case TokenTypeId.float:
+    case TokenTypeId.FloatingPointLiteral:
+    case TokenTypeId.HexadecimalLiteral:
+    case TokenTypeId.Identifier:
+    case TokenTypeId.int:
+    case TokenTypeId.IntegerLiteral:
+    case TokenTypeId.legacy:
+    case TokenTypeId.lengthof:
+    case TokenTypeId.LogicalAnd:
+    case TokenTypeId.LogicalOr:
+    case TokenTypeId.LookAhead:
+    case TokenTypeId.map:
+    case TokenTypeId.Modulus:
+    case TokenTypeId.MultipleCharacterLiteralCharacters:
+    case TokenTypeId.Multiplication:
+    case TokenTypeId.OpenBrace:
+    case TokenTypeId.OpenBracket:
+    case TokenTypeId.OpenParenthesis:
+    case TokenTypeId.Period:
+    case TokenTypeId.PostfixDecrement:
+    case TokenTypeId.PostfixIncrement:
+    case TokenTypeId.RangeOperator:
+    case TokenTypeId.reserved:
+    case TokenTypeId.RelationalEqual:
+    case TokenTypeId.RelationalGreaterThan:
+    case TokenTypeId.RelationalGreaterThanOrEqual:
+    case TokenTypeId.RelationalLessThan:
+    case TokenTypeId.RelationalLessThanOrEqual:
+    case TokenTypeId.RelationalNotEqual:
+    case TokenTypeId.Semicolon:
+    case TokenTypeId.SingleQuote:
+    case TokenTypeId.Subtraction:
+    case TokenTypeId.UnaryNegation:
+    case TokenTypeId.UnaryPlus:
+    case TokenTypeId.unsigned:
+    case TokenTypeId.utf16string:
+    case TokenTypeId.utf8list:
+    case TokenTypeId.utf8string:
+    case TokenTypeId.utfstring:
+    case TokenTypeId.UtfPrefix:
+    case TokenTypeId.UtfStringLiteralCharacters:
+    case TokenTypeId.Whitespace:
+      break;
+    default: {
+      const exhaustiveCheck: never = tokenTypeId;
+      throw new InternalScannerError(
+        "Unreachable code reached, tokenTypeId == " + exhaustiveCheck,
+      );
+    }
+  }
+});