@mpeggroup/mpeg-sdl-parser 4.0.0

package/src/ast/build/consume/consume-trivia.ts

@@ -0,0 +1,96 @@
+import { Text } from "@codemirror/state";
+import { TreeCursor } from "@lezer/common";
+import type { Trivia } from "../../node/trivia.ts";
+import { getLocationFromTextPosition } from "../../../util/location-utils.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import getLogger from "../../../util/logger.ts";
+
+// Generated by Lezer from the SDL grammar
+import { Comment, Whitespace } from "../../../lezer/parser.terms.ts";
+
+const logger = getLogger("consumeTrivia");
+
+function getCommentTrivia(cursor: TreeCursor, text: Text): Trivia {
+  return {
+    text: text.sliceString(cursor.from, cursor.to),
+    location: getLocationFromTextPosition(text, cursor.from),
+  };
+}
+
+function getBlankLineTrivia(cursor: TreeCursor, text: Text): Trivia {
+  return {
+    text: "\n",
+    location: getLocationFromTextPosition(text, cursor.from),
+  };
+}
+
+/**
+ * Iterate through the syntax tree and convert comments or whitespace to Trivia
+ * until a non-comment or non-whitespace parse node is found.
+ */
+export function consumeTrivia(
+  buildContext: BuildContext,
+  isTrailing: boolean,
+): Trivia[] {
+  const { cursor, text } = buildContext;
+  const currentState =
+    buildContext.stateStack[buildContext.stateStack.length - 1];
+
+  if (currentState.isEndOfSiblings) {
+    return [];
+  }
+
+  const trivia: Trivia[] = [];
+
+  let noSibling = true;
+
+  do {
+    // Save comments
+    if (cursor.type.id === Comment) {
+      trivia.push(getCommentTrivia(cursor, text));
+
+      // If trailing, stop after first comment
+      if (isTrailing) {
+        // advance the cursor as we consumed the comment
+        noSibling = !cursor.nextSibling();
+        break;
+      }
+    } // Skip whitespace except for multiple new lines which are collapsed into a single blank line trivia
+    else if (cursor.type.id === Whitespace) {
+      const whitespace = text.sliceString(cursor.from, cursor.to);
+      const newLineCount = (whitespace.match(/\n/g) || []).length;
+
+      // If trailing, stop at first line break
+      if (isTrailing && (newLineCount > 0)) {
+        noSibling = false;
+        break;
+      }
+
+      if (newLineCount > 1) {
+        trivia.push(getBlankLineTrivia(cursor, text));
+      }
+    } // Otherwise stop at the first non-comment, non-whitespace node
+    else if (
+      (cursor.type.id !== Comment) &&
+      (cursor.type.id !== Whitespace)
+    ) {
+      noSibling = false;
+      break;
+    }
+  } while (cursor.nextSibling());
+
+  currentState.isEndOfSiblings = noSibling;
+
+  if (trivia.length > 0) {
+    logger.debug(
+      currentState.indent +
+        `consumed ${isTrailing ? "trailing" : "leading"} trivia: ` +
+        trivia.map((triviaEntry) => triviaEntry.text.replaceAll("\n", "\\n"))
+          .join(
+            ", ",
+          ),
+    );
+  }
+
+  return trivia;
+}

package/src/ast/build/consume/consume-white-space-and-missing-errors.ts

@@ -0,0 +1,27 @@
+import type { BuildContext } from "../util/build-context.ts";
+
+// Generated by Lezer from the SDL grammar
+import { Whitespace } from "../../../lezer/parser.terms.ts";
+
+export function consumeWhiteSpaceAndMissingErrors(
+  buildContext: BuildContext,
+): void {
+  const { cursor } = buildContext;
+  const currentState =
+    buildContext.stateStack[buildContext.stateStack.length - 1];
+
+  if (currentState.isEndOfSiblings) {
+    return;
+  }
+
+  let noSibling = true;
+
+  do {
+    if ((cursor.type.id !== Whitespace) && !cursor.type.isError) {
+      noSibling = false;
+      break;
+    }
+  } while (cursor.nextSibling());
+
+  currentState.isEndOfSiblings = noSibling;
+}

package/src/ast/build/node/build-aggregate-output-value.ts

@@ -0,0 +1,57 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import { AggregateOutputValue } from "../../node/aggregate-output-value.ts";
+import type { NumberLiteral } from "../../node/number-literal.ts";
+import type { ElementaryTypeOutputValue } from "../../node/elementary-type-output-value.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import type { Token } from "../../node/token.ts";
+import {
+  fetchOneToManyCommaSeparatedList,
+  fetchRequiredNode,
+} from "../util/fetch-node.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+
+export function buildAggregateOutputValue(
+  buildContext: BuildContext,
+): AggregateOutputValue {
+  const children: Array<AbstractNode> = [];
+
+  const openBracePunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.OPEN_BRACE,
+  );
+  children.push(openBracePunctuator);
+
+  const { nodes: outputValues, commaPunctuators } =
+    fetchOneToManyCommaSeparatedList<
+      AggregateOutputValue | ElementaryTypeOutputValue | NumberLiteral
+    >(buildContext, [
+      NodeKind.AGGREGATE_OUTPUT_VALUE,
+      NodeKind.ELEMENTARY_TYPE_OUTPUT_VALUE,
+      NodeKind.NUMBER_LITERAL,
+    ]);
+
+  // now push each output value and comma punctuator to children
+  for (let i = 0; i < outputValues.length; i++) {
+    children.push(outputValues[i]);
+    if (i < commaPunctuators.length) {
+      children.push(commaPunctuators[i]);
+    }
+  }
+
+  const closeBracePunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.CLOSE_BRACE,
+  );
+  children.push(closeBracePunctuator);
+
+  return new AggregateOutputValue(
+    openBracePunctuator,
+    outputValues,
+    commaPunctuators,
+    closeBracePunctuator,
+    children,
+  );
+}

package/src/ast/build/node/build-aligned-modifier.ts

@@ -0,0 +1,99 @@
+import { InternalScannerError } from "../../../scanner-error.ts";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import { AlignedModifier } from "../../node/aligned-modifier.ts";
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+import type { Token } from "../../node/token.ts";
+import { fetchOptionalNode, fetchRequiredNode } from "../util/fetch-node.ts";
+import { isToken, type OptionalNode } from "../../util/types.ts";
+import type { BuildContext } from "../util/build-context.ts";
+
+export function buildAlignedModifier(
+  buildContext: BuildContext,
+): AlignedModifier {
+  const children: Array<AbstractNode> = [];
+  const alignedKeyword = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.ALIGNED,
+  );
+  children.push(alignedKeyword);
+
+  const openParenthesisPunctuator = fetchOptionalNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.OPEN_PARENTHESIS,
+  );
+
+  let bitCountModifierToken: OptionalNode<Token> | undefined = undefined;
+  let closeParenthesisPunctuator: OptionalNode<Token> | undefined = undefined;
+
+  let alignment = 8;
+
+  if (openParenthesisPunctuator) {
+    children.push(openParenthesisPunctuator);
+
+    bitCountModifierToken = fetchOptionalNode<Token>(
+      buildContext,
+      NodeKind.TOKEN,
+      [
+        TokenKind.ALIGNMENT_BIT_COUNT_8,
+        TokenKind.ALIGNMENT_BIT_COUNT_16,
+        TokenKind.ALIGNMENT_BIT_COUNT_32,
+        TokenKind.ALIGNMENT_BIT_COUNT_64,
+        TokenKind.ALIGNMENT_BIT_COUNT_128,
+      ],
+    );
+    if (bitCountModifierToken) {
+      children.push(bitCountModifierToken);
+
+      if (isToken(bitCountModifierToken)) {
+        switch (bitCountModifierToken.tokenKind) {
+          case TokenKind.ALIGNMENT_BIT_COUNT_8:
+            alignment = 8;
+            break;
+          case TokenKind.ALIGNMENT_BIT_COUNT_16:
+            alignment = 16;
+            break;
+          case TokenKind.ALIGNMENT_BIT_COUNT_32:
+            alignment = 32;
+            break;
+          case TokenKind.ALIGNMENT_BIT_COUNT_64:
+            alignment = 64;
+            break;
+          case TokenKind.ALIGNMENT_BIT_COUNT_128:
+            alignment = 128;
+            break;
+        }
+      }
+    } else {
+      throw new InternalScannerError(
+        "Expected alignment bit count modifier token after 'aligned('",
+        (openParenthesisPunctuator as Token).getLocation(),
+      );
+    }
+
+    closeParenthesisPunctuator = fetchOptionalNode<Token>(
+      buildContext,
+      NodeKind.TOKEN,
+      TokenKind.CLOSE_PARENTHESIS,
+    );
+    if (closeParenthesisPunctuator) {
+      children.push(closeParenthesisPunctuator);
+    } else {
+      throw new InternalScannerError(
+        "Expected closing parenthesis ')' after alignment bit count modifier",
+        (bitCountModifierToken as Token).getLocation(),
+      );
+    }
+  }
+
+  return new AlignedModifier(
+    alignedKeyword,
+    openParenthesisPunctuator,
+    bitCountModifierToken,
+    closeParenthesisPunctuator,
+    alignment,
+    children,
+  );
+}

package/src/ast/build/node/build-array-definition.ts

@@ -0,0 +1,131 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import type { Identifier } from "../../node/identifier.ts";
+import { ArrayDefinition } from "../../node/array-definition.ts";
+import { AlignedModifier } from "../../node/aligned-modifier.ts";
+import { ImplicitArrayDimension } from "../../node/implicit-array-dimension.ts";
+import type { ExplicitArrayDimension } from "../../node/explicit-array-dimension.ts";
+import type { PartialArrayDimension } from "../../node/partial-array-dimension.ts";
+import type { ElementaryType } from "../../node/elementary-type.ts";
+import type { LengthAttribute } from "../../node/length-attribute.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import type { Token } from "../../node/token.ts";
+import {
+  fetchOptionalNode,
+  fetchRequiredNode,
+  fetchZeroToManyList,
+} from "../util/fetch-node.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+import type { OptionalNode, ZeroToManyList } from "../../util/types.ts";
+import { ArrayDimensionKind } from "../../node/enum/array-dimension-kind.ts";
+
+export function buildArrayDefinition(
+  buildContext: BuildContext,
+): ArrayDefinition {
+  const children: Array<AbstractNode> = [];
+
+  const reservedKeyword = fetchOptionalNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.RESERVED,
+  );
+  if (reservedKeyword) {
+    children.push(reservedKeyword);
+  }
+
+  const legacyKeyword = fetchOptionalNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.LEGACY,
+  );
+  if (legacyKeyword) {
+    children.push(legacyKeyword);
+  }
+
+  const alignedModifier = fetchOptionalNode<AlignedModifier>(
+    buildContext,
+    NodeKind.ALIGNED_MODIFIER,
+  );
+  if (alignedModifier) {
+    children.push(alignedModifier);
+  }
+
+  const elementaryType = fetchOptionalNode<ElementaryType>(
+    buildContext,
+    NodeKind.ELEMENTARY_TYPE,
+  );
+
+  let lengthAttribute: OptionalNode<LengthAttribute> = undefined;
+  let classIdentifier: OptionalNode<Identifier> = undefined;
+  if (elementaryType) {
+    children.push(elementaryType);
+    lengthAttribute = fetchOptionalNode<LengthAttribute>(
+      buildContext,
+      NodeKind.LENGTH_ATTRIBUTE,
+    );
+    if (lengthAttribute) {
+      children.push(lengthAttribute);
+    } else {
+      // If an elementary type is present, a length attribute must be present
+      throw new InternalScannerError(
+        "Expected LengthAttribute node after ElementaryType node",
+      );
+    }
+  } else {
+    classIdentifier = fetchOptionalNode<Identifier>(
+      buildContext,
+      NodeKind.IDENTIFIER,
+    );
+    if (classIdentifier) {
+      children.push(classIdentifier);
+    }
+  }
+  const identifier = fetchRequiredNode<Identifier>(
+    buildContext,
+    NodeKind.IDENTIFIER,
+  );
+  children.push(identifier);
+
+  let dimensions: ZeroToManyList<
+    ExplicitArrayDimension | PartialArrayDimension
+  > = [];
+  const implicitArrayDimension = fetchOptionalNode<ImplicitArrayDimension>(
+    buildContext,
+    NodeKind.ARRAY_DIMENSION,
+    ArrayDimensionKind.IMPLICIT,
+  );
+  if (implicitArrayDimension) {
+    children.push(implicitArrayDimension);
+  } else {
+    dimensions = fetchZeroToManyList<
+      ExplicitArrayDimension | PartialArrayDimension
+    >(
+      buildContext,
+      NodeKind.ARRAY_DIMENSION,
+      [ArrayDimensionKind.EXPLICIT, ArrayDimensionKind.PARTIAL],
+    );
+    children.push(...dimensions);
+  }
+
+  const semicolonPunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.SEMICOLON,
+  );
+  children.push(semicolonPunctuator);
+
+  return new ArrayDefinition(
+    reservedKeyword,
+    legacyKeyword,
+    alignedModifier,
+    elementaryType,
+    lengthAttribute,
+    classIdentifier,
+    identifier,
+    implicitArrayDimension,
+    dimensions,
+    semicolonPunctuator,
+    children,
+  );
+}

package/src/ast/build/node/build-array-element-access.ts

@@ -0,0 +1,45 @@
+import type { AbstractExpression } from "../../node/abstract-expression.ts";
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import type { NumberLiteral } from "../../node/number-literal.ts";
+import { ArrayElementAccess } from "../../node/array-element-access.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import type { Token } from "../../node/token.ts";
+import { fetchRequiredNode } from "../util/fetch-node.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+import { Identifier } from "../../node/identifier.ts";
+
+export function buildArrayElementAccess(
+  buildContext: BuildContext,
+): ArrayElementAccess {
+  const children: Array<AbstractNode> = [];
+
+  const openBracketPunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.OPEN_BRACKET,
+  );
+  children.push(openBracketPunctuator);
+
+  const index = fetchRequiredNode<
+    AbstractExpression | Identifier | NumberLiteral
+  >(
+    buildContext,
+    [NodeKind.EXPRESSION, NodeKind.IDENTIFIER, NodeKind.NUMBER_LITERAL],
+  );
+  children.push(index);
+
+  const closeBracketPunctuator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    TokenKind.CLOSE_BRACKET,
+  );
+  children.push(closeBracketPunctuator);
+
+  return new ArrayElementAccess(
+    openBracketPunctuator,
+    index,
+    closeBracketPunctuator,
+    children,
+  );
+}

package/src/ast/build/node/build-base64-string-literal.ts

@@ -0,0 +1,41 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import { StringLiteral } from "../../node/string-literal.ts";
+import { StringLiteralKind } from "../../node/enum/string-literal-kind.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { Token } from "../../node/token.ts";
+import { fetchOneToManyList } from "../util/fetch-node.ts";
+import { isToken } from "../../util/types.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+
+const ESCAPED_BACKSLASH_REGEX = /\\\\/g;
+const ESCAPED_DOUBLE_QUOTE_REGEX = /\\"/g;
+
+export function buildBase64StringLiteral(
+  buildContext: BuildContext,
+): StringLiteral {
+  const literals = fetchOneToManyList<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    [TokenKind.DOUBLE_QUOTE, TokenKind.BASE64_STRING_LITERAL_CHARACTERS],
+  );
+
+  let value = "";
+
+  for (const literal of literals) {
+    if (
+      isToken(literal) &&
+      (literal.tokenKind === TokenKind.BASE64_STRING_LITERAL_CHARACTERS)
+    ) {
+      value += literal.text;
+    }
+  }
+  value = value.replaceAll(ESCAPED_DOUBLE_QUOTE_REGEX, "'")
+    .replaceAll(ESCAPED_BACKSLASH_REGEX, "\\");
+
+  return new StringLiteral(
+    StringLiteralKind.BASIC,
+    value,
+    literals,
+    [...literals],
+  );
+}

package/src/ast/build/node/build-binary-expression.ts

@@ -0,0 +1,113 @@
+import { NodeKind } from "../../node/enum/node-kind.ts";
+import { BinaryExpression } from "../../node/binary-expression.ts";
+import type { BuildContext } from "../util/build-context.ts";
+import type { AbstractNode } from "../../node/abstract-node.ts";
+import type { Token } from "../../node/token.ts";
+import { fetchRequiredNode } from "../util/fetch-node.ts";
+import { TokenKind } from "../../node/enum/token-kind.ts";
+import { isToken } from "../../util/types.ts";
+import { BinaryOperatorKind } from "../../node/enum/binary-operator-kind.ts";
+
+function getBinaryOperatorKind(
+  token: Token,
+): BinaryOperatorKind | undefined {
+  const tokenKind = token.tokenKind;
+
+  switch (tokenKind) {
+    case TokenKind.ASSIGNMENT:
+      return BinaryOperatorKind.ASSIGNMENT;
+    case TokenKind.MULTIPLICATION:
+      return BinaryOperatorKind.MULTIPLY;
+    case TokenKind.DIVISION:
+      return BinaryOperatorKind.DIVIDE;
+    case TokenKind.MODULUS:
+      return BinaryOperatorKind.MODULUS;
+    case TokenKind.ADDITION:
+      return BinaryOperatorKind.ADD;
+    case TokenKind.SUBTRACTION:
+      return BinaryOperatorKind.SUBTRACT;
+    case TokenKind.BITWISE_SHIFT_LEFT:
+      return BinaryOperatorKind.SHIFT_LEFT;
+    case TokenKind.BITWISE_SHIFT_RIGHT:
+      return BinaryOperatorKind.SHIFT_RIGHT;
+    case TokenKind.RELATIONAL_LESS_THAN:
+      return BinaryOperatorKind.LESS_THAN;
+    case TokenKind.RELATIONAL_LESS_THAN_OR_EQUAL:
+      return BinaryOperatorKind.LESS_THAN_OR_EQUAL;
+    case TokenKind.RELATIONAL_GREATER_THAN:
+      return BinaryOperatorKind.GREATER_THAN;
+    case TokenKind.RELATIONAL_GREATER_THAN_OR_EQUAL:
+      return BinaryOperatorKind.GREATER_THAN_OR_EQUAL;
+    case TokenKind.RELATIONAL_EQUAL:
+      return BinaryOperatorKind.EQUAL;
+    case TokenKind.RELATIONAL_NOT_EQUAL:
+      return BinaryOperatorKind.NOT_EQUAL;
+    case TokenKind.BITWISE_AND:
+      return BinaryOperatorKind.BITWISE_AND;
+    case TokenKind.BITWISE_OR:
+      return BinaryOperatorKind.BITWISE_OR;
+    case TokenKind.LOGICAL_AND:
+      return BinaryOperatorKind.LOGICAL_AND;
+    case TokenKind.LOGICAL_OR:
+      return BinaryOperatorKind.LOGICAL_OR;
+    default:
+      return undefined;
+  }
+}
+
+export function buildBinaryExpression(
+  buildContext: BuildContext,
+): BinaryExpression {
+  const children: Array<AbstractNode> = [];
+
+  const leftOperand = fetchRequiredNode<Token>(
+    buildContext,
+    [NodeKind.EXPRESSION, NodeKind.IDENTIFIER, NodeKind.NUMBER_LITERAL],
+  );
+  children.push(leftOperand);
+
+  const binaryOperator = fetchRequiredNode<Token>(
+    buildContext,
+    NodeKind.TOKEN,
+    [
+      TokenKind.ASSIGNMENT,
+      TokenKind.ADDITION,
+      TokenKind.SUBTRACTION,
+      TokenKind.MULTIPLICATION,
+      TokenKind.DIVISION,
+      TokenKind.MODULUS,
+      TokenKind.BITWISE_SHIFT_LEFT,
+      TokenKind.BITWISE_SHIFT_RIGHT,
+      TokenKind.RELATIONAL_LESS_THAN,
+      TokenKind.RELATIONAL_LESS_THAN_OR_EQUAL,
+      TokenKind.RELATIONAL_GREATER_THAN,
+      TokenKind.RELATIONAL_GREATER_THAN_OR_EQUAL,
+      TokenKind.RELATIONAL_EQUAL,
+      TokenKind.RELATIONAL_NOT_EQUAL,
+      TokenKind.BITWISE_AND,
+      TokenKind.BITWISE_OR,
+      TokenKind.LOGICAL_AND,
+      TokenKind.LOGICAL_OR,
+    ],
+  );
+  children.push(binaryOperator);
+
+  let binaryOperatorKind: BinaryOperatorKind | undefined;
+  if (isToken(binaryOperator)) {
+    binaryOperatorKind = getBinaryOperatorKind(binaryOperator);
+  }
+
+  const rightOperand = fetchRequiredNode<Token>(
+    buildContext,
+    [NodeKind.EXPRESSION, NodeKind.IDENTIFIER, NodeKind.NUMBER_LITERAL],
+  );
+  children.push(rightOperand);
+
+  return new BinaryExpression(
+    leftOperand,
+    binaryOperator,
+    rightOperand,
+    binaryOperatorKind,
+    children,
+  );
+}

package/src/ast/build/node/build-binary-literal.ts

@@ -0,0 +1,37 @@
+import { NumberLiteral } from "../../node/number-literal.ts";
+import { NumberLiteralKind } from "../../node/enum/number-literal-kind.ts";
+import { InternalScannerError } from "../../../scanner-error.ts";
+import type { Token } from "../../node/token.ts";
+
+const DOT_SEPARATOR_REGEX = /\./g;
+export function buildBinaryLiteral(
+  token: Token,
+): NumberLiteral {
+  const literalText = token.text;
+
+  if (!literalText.startsWith("0b")) {
+    throw new InternalScannerError(
+      `Missing binary literal prefix '0b': ${literalText}`,
+      token.getLocation(),
+    );
+  }
+
+  if (literalText.length === 2) {
+    throw new InternalScannerError(
+      `Missing binary literal value after prefix '0b'`,
+      token.getLocation(),
+    );
+  }
+
+  const stringValue = literalText.substring(2).trim().replaceAll(
+    DOT_SEPARATOR_REGEX,
+    "",
+  );
+  const value = parseInt(stringValue, 2);
+
+  return new NumberLiteral(
+    NumberLiteralKind.BINARY,
+    value,
+    [token],
+  );
+}