@mpeggroup/mpeg-sdl-parser 4.0.0

package/src/completion/rules/specification-rules.ts

@@ -0,0 +1,37 @@
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const specificationRules = [
+  {
+    previous: -1,
+    expected: [
+      TokenTypeId.ClassDeclaration,
+      TokenTypeId.MapDeclaration,
+      TokenTypeId.ComputedElementaryTypeDefinition,
+    ],
+  },
+  {
+    previous: TokenTypeId.ClassDeclaration,
+    expected: [
+      TokenTypeId.ClassDeclaration,
+      TokenTypeId.MapDeclaration,
+      TokenTypeId.ComputedElementaryTypeDefinition,
+    ],
+  },
+  {
+    previous: TokenTypeId.MapDeclaration,
+    expected: [
+      TokenTypeId.ClassDeclaration,
+      TokenTypeId.MapDeclaration,
+      TokenTypeId.ComputedElementaryTypeDefinition,
+    ],
+  },
+  {
+    previous: TokenTypeId.ComputedElementaryTypeDefinition,
+    expected: [
+      TokenTypeId.ClassDeclaration,
+      TokenTypeId.MapDeclaration,
+      TokenTypeId.ComputedElementaryTypeDefinition,
+    ],
+  },
+];

package/src/completion/rules/string-definition-rules.ts

@@ -0,0 +1,103 @@
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const stringDefinitionRules = [
+  {
+    previous: [
+      TokenTypeId.utf8string,
+      TokenTypeId.Identifier,
+      TokenTypeId.Assignment,
+    ],
+    expected: TokenTypeId.UtfStringLiteral,
+  },
+  {
+    previous: [
+      TokenTypeId.utf16string,
+      TokenTypeId.Identifier,
+      TokenTypeId.Assignment,
+    ],
+    expected: TokenTypeId.UtfStringLiteral,
+  },
+  {
+    previous: [
+      TokenTypeId.utfstring,
+      TokenTypeId.Identifier,
+      TokenTypeId.Assignment,
+    ],
+    expected: TokenTypeId.UtfStringLiteral,
+  },
+  {
+    previous: [
+      TokenTypeId.base64string,
+      TokenTypeId.Identifier,
+      TokenTypeId.Assignment,
+    ],
+    expected: TokenTypeId.Base64StringLiteral,
+  },
+  {
+    previous: TokenTypeId.reserved,
+    expected: [
+      TokenTypeId._const,
+      TokenTypeId.AlignedModifier,
+      TokenTypeId.utf8string,
+      TokenTypeId.utf16string,
+      TokenTypeId.utfstring,
+      TokenTypeId.base64string,
+    ],
+  },
+  {
+    previous: TokenTypeId._const,
+    expected: [
+      TokenTypeId.AlignedModifier,
+      TokenTypeId.utf8string,
+      TokenTypeId.utf16string,
+      TokenTypeId.utfstring,
+      TokenTypeId.base64string,
+    ],
+  },
+  {
+    previous: TokenTypeId.AlignedModifier,
+    expected: [
+      TokenTypeId.utf8string,
+      TokenTypeId.utf16string,
+      TokenTypeId.utfstring,
+      TokenTypeId.base64string,
+    ],
+  },
+  {
+    previous: [
+      TokenTypeId.utf8string,
+      TokenTypeId.Identifier,
+    ],
+    expected: TokenTypeId.Semicolon,
+  },
+  {
+    previous: [
+      TokenTypeId.utf16string,
+      TokenTypeId.Identifier,
+    ],
+    expected: TokenTypeId.Semicolon,
+  },
+  {
+    previous: [
+      TokenTypeId.utfstring,
+      TokenTypeId.Identifier,
+    ],
+    expected: TokenTypeId.Semicolon,
+  },
+  {
+    previous: [
+      TokenTypeId.base64string,
+      TokenTypeId.Identifier,
+    ],
+    expected: TokenTypeId.Semicolon,
+  },
+  {
+    previous: TokenTypeId.Base64StringLiteral,
+    expected: TokenTypeId.Semicolon,
+  },
+  {
+    previous: TokenTypeId.UtfStringLiteral,
+    expected: TokenTypeId.Semicolon,
+  },
+];

package/src/completion/rules/switch-statement-rules.ts

@@ -0,0 +1,50 @@
+import { binaryOperatorTypes, expressionTypes } from "./abstract-types.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const switchStatementRules = [
+  {
+    previous: -1,
+    expected: TokenTypeId._switch,
+  },
+  {
+    previous: TokenTypeId._switch,
+    expected: TokenTypeId.OpenParenthesis,
+  },
+  {
+    previous: TokenTypeId.OpenParenthesis,
+    expected: expressionTypes,
+  },
+  ...expressionTypes.map((expressionType) => ({
+    previous: expressionType,
+    expected: [
+      TokenTypeId.CloseParenthesis,
+      ...binaryOperatorTypes,
+    ],
+  })),
+  {
+    previous: TokenTypeId.CloseParenthesis,
+    expected: TokenTypeId.OpenBrace,
+  },
+  {
+    previous: TokenTypeId.OpenBrace,
+    expected: [
+      TokenTypeId.CaseClause,
+      TokenTypeId.DefaultClause,
+      TokenTypeId.CloseBrace,
+    ],
+  },
+  {
+    previous: TokenTypeId.CaseClause,
+    expected: [
+      TokenTypeId.CaseClause,
+      TokenTypeId.DefaultClause,
+      TokenTypeId.CloseBrace,
+    ],
+  },
+  {
+    previous: TokenTypeId.DefaultClause,
+    expected: TokenTypeId.CloseBrace,
+  },
+];

package/src/completion/rules/unary-expression-rules.ts

@@ -0,0 +1,41 @@
+import {
+  binaryOperatorTypes,
+  expressionTypes,
+  numberLiteralTypes,
+  postfixOperatorTypes,
+  unaryOperatorTypes,
+} from "./abstract-types.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const unaryExpressionRules = [
+  {
+    previous: -1,
+    expected: [
+      ...unaryOperatorTypes,
+      TokenTypeId.LengthofExpression,
+      TokenTypeId.Identifier,
+      ...numberLiteralTypes,
+      TokenTypeId.OpenParenthesis,
+    ],
+  },
+  ...expressionTypes.map((expressionType) => ({
+    previous: expressionType,
+    expected: [
+      TokenTypeId.ArrayElementAccess,
+      TokenTypeId.ClassMemberAccess,
+      ...postfixOperatorTypes,
+      ...binaryOperatorTypes,
+      TokenTypeId.CloseParenthesis,
+    ],
+  })),
+  ...unaryOperatorTypes.map((unaryOperatorType) => ({
+    previous: unaryOperatorType,
+    expected: expressionTypes,
+  })),
+  {
+    previous: TokenTypeId.OpenParenthesis,
+    expected: [...expressionTypes],
+  },
+];

package/src/completion/rules/utf-string-literal-rules.ts

@@ -0,0 +1,17 @@
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const utfStringLiteralRules = [
+  {
+    previous: -1,
+    expected: TokenTypeId.UtfPrefix,
+  },
+  {
+    previous: TokenTypeId.UtfStringLiteralCharacters,
+    expected: TokenTypeId.DoubleQuote,
+  },
+  {
+    previous: TokenTypeId.DoubleQuote,
+    expected: TokenTypeId.UtfPrefix,
+  },
+];

package/src/completion/rules/while-statement-rules.ts

@@ -0,0 +1,38 @@
+import {
+  binaryOperatorTypes,
+  expressionTypes,
+  statementTypes,
+} from "./abstract-types.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../lezer/parser.terms.ts";
+
+export const whileStatementRules = [
+  {
+    previous: -1,
+    expected: TokenTypeId._while,
+  },
+  {
+    previous: TokenTypeId._while,
+    expected: TokenTypeId.OpenParenthesis,
+  },
+  {
+    previous: TokenTypeId.OpenParenthesis,
+    expected: expressionTypes,
+  },
+  {
+    previous: TokenTypeId.UnaryExpression,
+    expected: TokenTypeId.CloseParenthesis,
+  },
+  ...expressionTypes.map((expressionType) => ({
+    previous: expressionType,
+    expected: [
+      TokenTypeId.CloseParenthesis,
+      ...binaryOperatorTypes,
+    ],
+  })),
+  {
+    previous: TokenTypeId.CloseParenthesis,
+    expected: statementTypes,
+  },
+];

package/src/lezer/context-tracker.ts

@@ -0,0 +1,45 @@
+import { ContextTracker, LRParser as LezerParser } from "@lezer/lr";
+import getLogger from "../util/logger.ts";
+
+const logger = getLogger("contextTracker");
+
+/*
+ * Define a custom context tracker for debug logging via the framework wide logger
+ */
+export function getContextTracker(parser: LezerParser): ContextTracker<number> {
+  const contextTracker = new ContextTracker<number>({
+    start: 0,
+    shift: (_context, term, stack, _input) => {
+      const termName = parser.getName(term) ?? "unknown";
+
+      logger.debug("shift term: %s at position %d", termName, stack.pos);
+
+      return 0;
+    },
+    reduce: (_context, term, stack, _input) => {
+      const termName = parser.getName(term) ?? "unknown";
+
+      logger.debug("reduce term: %s at position %d", termName, stack.pos);
+
+      return 0;
+    },
+    reuse: (_context, node, stack, _input) => {
+      const nodeName = parser.getName(node.type.id) ?? "unknown";
+
+      logger.debug("reuse node: %s at position %d", nodeName, stack.pos);
+
+      return 0;
+    },
+    strict: false,
+  });
+
+  return contextTracker;
+}
+
+export const defaultContextTracker = new ContextTracker<number>({
+  start: 0,
+  shift: (_context, _term, _stack, _input) => 0,
+  reduce: (_context, _term, _stack, _input) => 0,
+  reuse: (_context, _node, _stack, _input) => 0,
+  strict: false,
+});

package/src/lezer/create-sdl-parser.ts

@@ -0,0 +1,88 @@
+import { LRParser as LezerParser } from "@lezer/lr";
+import { Text } from "@codemirror/state";
+import { type Input } from "@lezer/common";
+import { debugEnabled } from "../util/logger.ts";
+import { getContextTracker } from "./context-tracker.ts";
+import { styleTagsNodePropSource } from "./props/style-tags-node-prop-source.ts";
+import { foldNodePropSource } from "./props/fold-node-prop-source.ts";
+import { primitiveNodePropSource } from "./props/primitive-node-prop-source.ts";
+import { syntacticTokenNodePropSource } from "./props/syntactic-token-node-prop-source.ts";
+import { createSyntaxErrorFromTextAndPosition } from "./create-syntax-error-from-text-and-position.ts";
+
+// Generated by Lezer from the SDL grammar
+import { parser } from "./parser.ts";
+
+let lenientSdlParser: LezerParser | undefined;
+let strictSdlParser: LezerParser | undefined;
+
+/**
+ * Create an in memory lenient Lezer based parser using the SDL grammar and store it as a "singleton".
+ */
+export function createLenientSdlParser(): LezerParser {
+  if (!lenientSdlParser) {
+    if (debugEnabled) {
+      lenientSdlParser = parser.configure({
+        contextTracker: getContextTracker(parser),
+      });
+    } else {
+      lenientSdlParser = parser;
+    }
+
+    // Configure the parser with custom node properties
+    lenientSdlParser = lenientSdlParser.configure({
+      props: [
+        foldNodePropSource,
+        primitiveNodePropSource,
+        styleTagsNodePropSource,
+        syntacticTokenNodePropSource,
+      ],
+    });
+  }
+
+  return lenientSdlParser;
+}
+
+/**
+ * Create an in memory strict Lezer based parser using the SDL grammar and store it as a "singleton".
+ */
+export function createStrictSdlParser(): LezerParser {
+  if (!strictSdlParser) {
+    strictSdlParser = createLenientSdlParser().configure({
+      strict: true,
+    });
+
+    // Wrap the parser's parse method to catch errors and wrap them in SyntaxError
+    const originalParseFunc = strictSdlParser.parse.bind(strictSdlParser);
+
+    strictSdlParser.parse = (input: Input | string, ...args) => {
+      try {
+        return originalParseFunc(input, ...args);
+      } catch (err) {
+        // parse position out of error message which takes the form "No parse at 0"
+        const errorMessage = (err as Error).message;
+        const match = errorMessage.match(/No parse at (\d+)/);
+        let position = 0;
+        if (match) {
+          const parsed = parseInt(match[1], 10);
+          position = isNaN(parsed) ? 0 : parsed;
+        }
+
+        let inputText: string;
+
+        if (typeof input === "string") {
+          inputText = input;
+        } else {
+          inputText = input.read(0, input.length);
+        }
+
+        const text = Text.of(
+          inputText.split("\n"),
+        );
+
+        throw createSyntaxErrorFromTextAndPosition(text, position);
+      }
+    };
+  }
+
+  return strictSdlParser;
+}

package/src/lezer/create-syntax-error-from-text-and-cursor.ts

@@ -0,0 +1,110 @@
+import type { Text } from "@codemirror/state";
+import { TreeCursor } from "@lezer/common";
+import { InternalScannerError, SyntaxError } from "../scanner-error.ts";
+import { syntacticTokenNodeProp } from "./props/syntactic-token-node-prop-source.ts";
+import { getPotentialTokenTypeIds } from "../completion/get-potential-token-type-ids.ts";
+import { createSyntaxErrorFromTextAndPosition } from "./create-syntax-error-from-text-and-position.ts";
+import { createLenientSdlParser } from "./create-sdl-parser.ts";
+
+const lenientSdlParser = createLenientSdlParser();
+
+const nodeSet = lenientSdlParser.nodeSet;
+
+/**
+ * Helper function to create a SyntaxError from the text and a cursor positioned at an error node.
+ * As the cursor from a parse tree is available, details on missing or unexpected tokens can be extracted.
+ *
+ * @param text The text to parse.
+ * @param cursor The cursor position in the text which should be at an error node.
+ */
+export function createSyntaxErrorFromTextAndCursor(
+  text: Text,
+  cursor: TreeCursor,
+): SyntaxError {
+  if (!cursor.type.isError) {
+    throw new InternalScannerError(
+      "Expected cursor to be an error token, but it is not",
+    );
+  }
+
+  // if the error token has a child, the child is an unexpected token
+  const unexpectedNode = cursor.firstChild();
+
+  let message: string = "";
+
+  if (unexpectedNode) {
+    message = "Unexpected: " +
+      text.sliceString(cursor.from, cursor.to);
+
+    return createSyntaxErrorFromTextAndPosition(
+      text,
+      cursor.from,
+      message,
+    );
+  }
+
+  // check if the error is for an uknown token
+  if (cursor.to > cursor.from) {
+    message = "Unknown token: " +
+      text.sliceString(cursor.from, cursor.to);
+
+    return createSyntaxErrorFromTextAndPosition(
+      text,
+      cursor.from,
+      message,
+    );
+  }
+
+  // otherwise it is a missing expected token and we should indicate what token(s) or node(s) could be expected here
+  const potentialTokenTypeIds = getPotentialTokenTypeIds(
+    cursor,
+  );
+
+  if (!potentialTokenTypeIds || potentialTokenTypeIds.length === 0) {
+    throw new InternalScannerError(
+      "Expected token type IDs not found for parse error at position " +
+        cursor.from,
+    );
+  }
+
+  const possibleSyntacticTokens: string[] = [];
+  const possibleNodes: string[] = [];
+  potentialTokenTypeIds.forEach((id) => {
+    const type = nodeSet.types[id];
+
+    if (type.prop(syntacticTokenNodeProp)) {
+      possibleSyntacticTokens.push(type.prop(syntacticTokenNodeProp)!);
+    } else {
+      possibleNodes.push("<" + type.name + ">");
+    }
+  });
+
+  // sort strings taking into account that some strings may be integers
+  possibleSyntacticTokens.sort((a, b) => {
+    const aIsNumber = !isNaN(Number(a));
+    const bIsNumber = !isNaN(Number(b));
+
+    if (aIsNumber && bIsNumber) {
+      return Number(a) - Number(b);
+    }
+    if (aIsNumber) return -1;
+    if (bIsNumber) return 1;
+    return a.localeCompare(b);
+  });
+
+  possibleNodes.sort();
+
+  const possibleSyntacticTokensAndNodes = [
+    ...possibleSyntacticTokens,
+    ...possibleNodes,
+  ];
+
+  message = possibleSyntacticTokensAndNodes.length > 1
+    ? "Expected one of: " + possibleSyntacticTokensAndNodes.join(" ")
+    : "Expected: " + possibleSyntacticTokensAndNodes[0];
+  return createSyntaxErrorFromTextAndPosition(
+    text,
+    cursor.from,
+    message,
+  );
+}

package/src/lezer/create-syntax-error-from-text-and-position.ts

@@ -0,0 +1,36 @@
+import type { Text } from "@codemirror/state";
+import { SyntaxError } from "../scanner-error.ts";
+import { getLocationFromTextPosition } from "../util/location-utils.ts";
+
+/**
+ * Helper function to create a SyntaxError from the text and a position.
+ *
+ * @param text The text to parse.
+ * @param position The position in the text, 0-based.
+ * @param message The error message, defaults to "Syntax error".
+ */
+export function createSyntaxErrorFromTextAndPosition(
+  text: Text,
+  position: number,
+  message: string = "",
+): SyntaxError {
+  const line = text.lineAt(position);
+  const location = getLocationFromTextPosition(text, position);
+  const precedingLines = [];
+  // Collect up to two preceding lines if available
+  if (location.row > 1) {
+    // First preceding line (if exists)
+    precedingLines.push(text.line(location.row - 1).text);
+    // Second preceding line (if exists)
+    if (location.row > 2) {
+      precedingLines.unshift(text.line(location.row - 2).text);
+    }
+  }
+
+  return new SyntaxError(
+    message,
+    location,
+    line.text,
+    precedingLines,
+  );
+}

package/src/lezer/parser.terms.ts

@@ -0,0 +1,129 @@
+// This file was generated by lezer-generator. You probably shouldn't edit it.
+export const Whitespace = 1,
+  Comment = 2,
+  Specification = 3,
+  CloseBrace = 4,
+  ClassDeclaration = 5,
+  AlignedModifier = 6,
+  Identifier = 7,
+  aligned = 8,
+  OpenParenthesis = 9,
+  AlignmentBitCount8 = 10,
+  AlignmentBitCount16 = 11,
+  AlignmentBitCount32 = 12,
+  AlignmentBitCount64 = 13,
+  AlignmentBitCount128 = 14,
+  CloseParenthesis = 15,
+  ExpandableModifier = 16,
+  expandable = 17,
+  IntegerLiteral = 18,
+  abstract = 19,
+  _class = 20,
+  ParameterList = 21,
+  Parameter = 22,
+  ElementaryType = 23,
+  int = 24,
+  unsigned = 25,
+  float = 26,
+  bit = 27,
+  Comma = 28,
+  ExtendsModifier = 29,
+  _extends = 30,
+  ParameterValueList = 31,
+  BinaryExpression = 32,
+  Multiplication = 33,
+  Division = 34,
+  Modulus = 35,
+  Addition = 36,
+  Subtraction = 37,
+  BitwiseShiftLeft = 38,
+  BitwiseShiftRight = 39,
+  RelationalLessThan = 40,
+  RelationalLessThanOrEqual = 41,
+  RelationalGreaterThan = 42,
+  RelationalGreaterThanOrEqual = 43,
+  RelationalEqual = 44,
+  RelationalNotEqual = 45,
+  BitwiseAnd = 46,
+  BitwiseOr = 47,
+  LogicalAnd = 48,
+  LogicalOr = 49,
+  UnaryExpression = 50,
+  CloseBracket = 51,
+  OpenBracket = 52,
+  ArrayElementAccess = 53,
+  ClassMemberAccess = 54,
+  Period = 55,
+  PostfixIncrement = 56,
+  PostfixDecrement = 57,
+  UnaryPlus = 58,
+  UnaryNegation = 59,
+  LengthofExpression = 60,
+  lengthof = 61,
+  BinaryLiteral = 62,
+  HexadecimalLiteral = 63,
+  MultipleCharacterLiteral = 64,
+  SingleQuote = 65,
+  MultipleCharacterLiteralCharacters = 66,
+  DecimalLiteral = 67,
+  FloatingPointLiteral = 68,
+  BitModifier = 69,
+  Colon = 70,
+  Assignment = 71,
+  ExtendedClassIdRange = 72,
+  ClassId = 73,
+  ClassIdRange = 74,
+  RangeOperator = 75,
+  OpenBrace = 76,
+  CompoundStatement = 77,
+  IfStatement = 78,
+  _if = 79,
+  _else = 80,
+  SwitchStatement = 81,
+  _switch = 82,
+  CaseClause = 83,
+  _case = 84,
+  _break = 85,
+  Semicolon = 86,
+  DefaultClause = 87,
+  _default = 88,
+  ForStatement = 89,
+  _for = 90,
+  AssignmentExpression = 91,
+  ComputedElementaryTypeDefinition = 92,
+  computed = 93,
+  _const = 94,
+  DoStatement = 95,
+  _do = 96,
+  _while = 97,
+  WhileStatement = 98,
+  ExpressionStatement = 99,
+  ElementaryTypeDefinition = 100,
+  reserved = 101,
+  legacy = 102,
+  LengthAttribute = 103,
+  LookAhead = 104,
+  MapDefinition = 105,
+  ClassDefinition = 106,
+  StringDefinition = 107,
+  base64string = 108,
+  Base64StringLiteral = 109,
+  DoubleQuote = 110,
+  Base64StringLiteralCharacters = 111,
+  utf16string = 112,
+  UtfStringLiteral = 113,
+  UtfPrefix = 114,
+  UtfStringLiteralCharacters = 115,
+  utf8string = 116,
+  utf8list = 117,
+  utfstring = 118,
+  ArrayDefinition = 119,
+  ImplicitArrayDimension = 120,
+  ExplicitArrayDimension = 121,
+  PartialArrayDimension = 122,
+  ComputedArrayDefinition = 123,
+  MapDeclaration = 124,
+  map = 125,
+  MapEntry = 126,
+  AggregateOutputValue = 127,
+  ElementaryTypeOutputValue = 128;