@mpeggroup/mpeg-sdl-parser 4.0.0

package/tests/ast/build-ast.test.ts

@@ -0,0 +1,463 @@
+import { describe, expect, test } from "bun:test";
+import { buildAst } from "../../src/ast/build-ast.ts";
+import {
+  createLenientSdlParser,
+  createStrictSdlParser,
+} from "../../src/lezer/create-sdl-parser.ts";
+import { SdlStringInput } from "../../src/lezer/sdl-string-input.ts";
+import { Specification } from "../../src/ast/node/specification.ts";
+import { ClassDeclaration } from "../../src/ast/node/class-declaration.ts";
+import { Identifier } from "../../src/ast/node/identifier.ts";
+import { ExpressionStatement } from "../../src/ast/node/expression-statement.ts";
+import { UnaryExpression } from "../../src/ast/node/unary-expression.ts";
+import { Token } from "../../src/ast/node/token.ts";
+import { TokenKind } from "../../src/ast/node/enum/token-kind.ts";
+import { MissingError } from "../../src/ast/node/missing-error.ts";
+import { UnexpectedError } from "../../src/ast/node/unexpected-error.ts";
+
+const lenientSdlParser = createLenientSdlParser();
+const strictSdlParser = createStrictSdlParser();
+
+describe("buildAst Tests", () => {
+  test("Parse tree with parse error of missing token SyntaxError fails with non-lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("int i;");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    expect(() => buildAst(parseTree, sdlStringInput)).toThrow(
+      "SYNTAX ERROR: Expected: computed => { row: 1, column: 1, position: 0 }",
+    );
+  });
+
+  test("Parse tree with parse error of missing expected node SyntaxError fails with non-lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("§");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    expect(() => buildAst(parseTree, sdlStringInput)).toThrow(
+      "SYNTAX ERROR: Unknown token: § => { row: 1, column: 1, position: 0 }",
+    );
+  });
+
+  test("Parse tree with no content works with non-lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    buildAst(parseTree, sdlStringInput);
+  });
+
+  test("Parse tree with parse error of unexpected token SyntaxError fails with non-lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("1");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    expect(() => buildAst(parseTree, sdlStringInput)).toThrow(
+      "SYNTAX ERROR: Unexpected: 1 => { row: 1, column: 1, position: 0 }",
+    );
+  });
+
+  test("Parse tree with parse error of token SyntaxError works with lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("int i;");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    buildAst(parseTree, sdlStringInput, true);
+  });
+
+  test("Parse tree with parse error of missing expected node SyntaxError works with lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("§");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    buildAst(parseTree, sdlStringInput, true);
+  });
+
+  test("Parse tree with no content works with lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    buildAst(parseTree, sdlStringInput, true);
+  });
+
+  test("Parse tree with parse error of unexpected node SyntaxError works with lenient AST", () => {
+    const sdlStringInput = new SdlStringInput("1");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+
+    buildAst(parseTree, sdlStringInput, true);
+  });
+
+  test("Comment only", () => {
+    const sdlStringInput = new SdlStringInput(" // hello world");
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+    const expected = new Specification([]);
+
+    expected.trailingTrivia = [
+      { text: "// hello world", location: { row: 1, column: 2, position: 1 } },
+    ];
+
+    expect(
+      specification,
+    ).toEqual(
+      expected,
+    );
+  });
+
+  test("Simple", () => {
+    const sdlStringInput = new SdlStringInput("class A {}");
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+    const classToken = new Token(
+      TokenKind.CLASS,
+      "class",
+      { row: 1, column: 1, position: 0 },
+    );
+    const identifier = new Identifier(
+      "A",
+      new Token(
+        TokenKind.IDENTIFIER,
+        "A",
+        { row: 1, column: 7, position: 6 },
+      ),
+    );
+    const openBraceToken = new Token(
+      TokenKind.OPEN_BRACE,
+      "{",
+      { row: 1, column: 9, position: 8 },
+    );
+    const closeBraceToken = new Token(
+      TokenKind.CLOSE_BRACE,
+      "}",
+      { row: 1, column: 10, position: 9 },
+    );
+
+    expect(
+      specification,
+    ).toEqual(
+      new Specification(
+        [
+          new ClassDeclaration(
+            undefined,
+            undefined,
+            undefined,
+            classToken,
+            identifier,
+            undefined,
+            undefined,
+            undefined,
+            openBraceToken,
+            [],
+            closeBraceToken,
+            [classToken, identifier, openBraceToken, closeBraceToken],
+          ),
+        ],
+      ),
+    );
+  });
+
+  test("Simple with comments", () => {
+    const sdlStringInput = new SdlStringInput(
+      "// hello\nclass A {// world\n}// again",
+    );
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+
+    const classToken = new Token(
+      TokenKind.CLASS,
+      "class",
+      { row: 2, column: 1, position: 9 },
+    );
+    const identifier = new Identifier(
+      "A",
+      new Token(
+        TokenKind.IDENTIFIER,
+        "A",
+        { row: 2, column: 7, position: 15 },
+      ),
+    );
+    const openBraceToken = new Token(
+      TokenKind.OPEN_BRACE,
+      "{",
+      { row: 2, column: 9, position: 17 },
+    );
+    const closeBraceToken = new Token(
+      TokenKind.CLOSE_BRACE,
+      "}",
+      { row: 3, column: 1, position: 27 },
+    );
+
+    const classDeclaration = new ClassDeclaration(
+      undefined,
+      undefined,
+      undefined,
+      classToken,
+      identifier,
+      undefined,
+      undefined,
+      undefined,
+      openBraceToken,
+      [],
+      closeBraceToken,
+      [classToken, identifier, openBraceToken, closeBraceToken],
+    );
+
+    classDeclaration.leadingTrivia = [
+      { text: "// hello", location: { row: 1, column: 1, position: 0 } },
+    ];
+    classDeclaration.trailingTrivia = [
+      { text: "// again", location: { row: 3, column: 2, position: 28 } },
+    ];
+    openBraceToken.trailingTrivia = [
+      { text: "// world", location: { row: 2, column: 10, position: 18 } },
+    ];
+
+    expect(
+      specification,
+    ).toEqual(
+      new Specification(
+        [
+          classDeclaration,
+        ],
+      ),
+    );
+  });
+
+  test("Lenient with missing identifier", () => {
+    const sdlStringInput = new SdlStringInput("class {}");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+    const classToken = new Token(
+      TokenKind.CLASS,
+      "class",
+      { row: 1, column: 1, position: 0 },
+    );
+    const identifier = new MissingError({ row: 1, column: 7, position: 6 });
+    const openBraceToken = new Token(
+      TokenKind.OPEN_BRACE,
+      "{",
+      { row: 1, column: 7, position: 6 },
+    );
+    const closeBraceToken = new Token(
+      TokenKind.CLOSE_BRACE,
+      "}",
+      { row: 1, column: 8, position: 7 },
+    );
+
+    expect(
+      specification,
+    ).toEqual(
+      new Specification(
+        [
+          new ClassDeclaration(
+            undefined,
+            undefined,
+            undefined,
+            classToken,
+            identifier,
+            undefined,
+            undefined,
+            undefined,
+            openBraceToken,
+            [],
+            closeBraceToken,
+            [classToken, identifier, openBraceToken, closeBraceToken],
+          ),
+        ],
+      ),
+    );
+  });
+
+  test("Lenient with multiple missing tokens", () => {
+    const sdlStringInput = new SdlStringInput("class A { - }");
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    const classToken = new Token(
+      TokenKind.CLASS,
+      "class",
+      { row: 1, column: 1, position: 0 },
+    );
+    const identifier = new Identifier(
+      "A",
+      new Token(
+        TokenKind.IDENTIFIER,
+        "A",
+        { row: 1, column: 7, position: 6 },
+      ),
+    );
+    const openBraceToken = new Token(
+      TokenKind.OPEN_BRACE,
+      "{",
+      { row: 1, column: 9, position: 8 },
+    );
+    const unaryOperatorToken = new Token(
+      TokenKind.UNARY_NEGATION,
+      "-",
+      { row: 1, column: 11, position: 10 },
+    );
+    const missingOperandToken = new MissingError({
+      row: 1,
+      column: 13,
+      position: 12,
+    });
+    const unaryExpression = new UnaryExpression(
+      unaryOperatorToken,
+      undefined,
+      missingOperandToken,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      [unaryOperatorToken, missingOperandToken],
+    );
+    const missingSemicolonPunctuator = new MissingError({
+      row: 1,
+      column: 13,
+      position: 12,
+    });
+
+    const expressionStatement = new ExpressionStatement(
+      unaryExpression,
+      missingSemicolonPunctuator,
+      [unaryExpression, missingSemicolonPunctuator],
+    );
+    const closeBraceToken = new Token(
+      TokenKind.CLOSE_BRACE,
+      "}",
+      { row: 1, column: 13, position: 12 },
+    );
+
+    expect(
+      specification,
+    ).toEqual(
+      new Specification(
+        [
+          new ClassDeclaration(
+            undefined,
+            undefined,
+            undefined,
+            classToken,
+            identifier,
+            undefined,
+            undefined,
+            undefined,
+            openBraceToken,
+            [
+              expressionStatement,
+            ],
+            closeBraceToken,
+            [
+              classToken,
+              identifier,
+              openBraceToken,
+              expressionStatement,
+              closeBraceToken,
+            ],
+          ),
+        ],
+      ),
+    );
+  });
+
+  test("Comment within syntax element", () => {
+    const sdlStringInput = new SdlStringInput("class A {i++// comment\n;}");
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("JSON", () => {
+    const sdlStringInput = new SdlStringInput("computed int i;");
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+    const actual = JSON.stringify(specification);
+
+    expect(actual).toMatchSnapshot();
+  });
+
+  test("Leading comment", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{\n  // Le lorem ipsum est, en imprimerie, une suite de mots sans signification utilisée\n}",
+    );
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("Leading and trailing comments", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A {// trailing 1\ni++; // trailing 2\n//leading 1\nN = N + // trailing 3\n 1;}\n// trailing 4",
+    );
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("Switch statement", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{switch (1) { case 1: break; default: i++; }}",
+    );
+    const parseTree = strictSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("Invalid elementary type definition", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{bit transport_priority;}",
+    );
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("Single invalid token", () => {
+    const sdlStringInput = new SdlStringInput(
+      "§\n",
+    );
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    const unknownToken = new Token(
+      TokenKind.ERROR_UNKNOWN_TOKEN,
+      "§",
+      { row: 1, column: 1, position: 0 },
+    );
+
+    const unexpectedError = new UnexpectedError(unknownToken);
+
+    expect(specification).toEqual(
+      new Specification([unexpectedError]),
+    );
+  });
+
+  test("Invalid various elements 1", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{if(a fie==1) {}}\n",
+    );
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("Invalid various elements 2", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{if(a fie==1){}}\n",
+    );
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    expect(specification).toMatchSnapshot();
+  });
+
+  test("While statement", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A{while(i<2){i++;}}\n",
+    );
+    const parseTree = lenientSdlParser.parse(sdlStringInput);
+    const specification = buildAst(parseTree, sdlStringInput, true);
+
+    expect(specification).toMatchSnapshot();
+  });
+});

package/tests/ast/visitor/traversing-visitor.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from "bun:test";
+import path from "node:path";
+import { type Buffer } from "node:buffer";
+import { promises as fs } from "node:fs";
+import HistoryRecordingNodeHandler, {
+  expectedHistory,
+} from "../../fixtures/history-recording-node-handler.ts";
+import { createStrictSdlParser } from "../../../src/lezer/create-sdl-parser.ts";
+import { TraversingVisitor } from "../../../src/ast/visitor/traversing-visitor.ts";
+import { SdlStringInput } from "../../../src/lezer/sdl-string-input.ts";
+import { buildAst } from "../../../src/ast/build-ast.ts";
+
+describe("TraversingVisitor Tests", () => {
+  test("Test traversing visitor", async () => {
+    const sdlSource = await fs.readFile(
+      path.join(__dirname, "../../sample-specifications/sample.sdl"),
+    ).then((buffer: Buffer) => buffer.toString());
+    const sdlStringInput = new SdlStringInput(sdlSource);
+    const sdlParser = createStrictSdlParser();
+    const sdlParseTree = sdlParser.parse(sdlStringInput);
+    const specification = buildAst(sdlParseTree, sdlStringInput);
+
+    const historyRecordingNodeHandler = new HistoryRecordingNodeHandler();
+    const traversingVisitor = new TraversingVisitor(
+      historyRecordingNodeHandler,
+    );
+
+    traversingVisitor.visit(specification);
+
+    expect(
+      historyRecordingNodeHandler.nodeHistory,
+    ).toEqual(
+      expectedHistory,
+    );
+  });
+});

package/tests/completion/get-potential-syntactic-tokens.test.ts

@@ -0,0 +1,89 @@
+import { describe, test } from "bun:test";
+import { testPotentialSyntacticTokensScenario } from "./test-potential-syntactic-tokens-scenario.ts";
+
+describe("getPotentialSyntacticTokens Tests", () => {
+  test("Test missing length attribute potential syntactic token scenario", () => {
+    testPotentialSyntacticTokensScenario(
+      "class A { bit b; }",
+      14,
+      ["("],
+    );
+  });
+
+  test("Test duplicate identifier potential syntactic token scenario", () => {
+    testPotentialSyntacticTokensScenario(
+      "class A {if (f oo) {bit(8) c;}}",
+      15,
+      [
+        "!=",
+        "%",
+        "&",
+        "&&",
+        "*",
+        "+",
+        "-",
+        "/",
+        "<",
+        "<<",
+        "<=",
+        "==",
+        ">",
+        ">=",
+        ">>",
+        "|",
+        "||",
+      ].sort(),
+    );
+  });
+
+  test("Test complex potential syntactic token scenario", () => {
+    testPotentialSyntacticTokensScenario(
+      "class A {if (a b == 0b01 || c == 0b11) {bit(8) c;}}",
+      15,
+      [
+        "!=",
+        "%",
+        "&",
+        "&&",
+        ")",
+        "*",
+        "+",
+        "-",
+        "/",
+        "<",
+        "<<",
+        "<=",
+        "==",
+        ">",
+        ">=",
+        ">>",
+        "|",
+        "||",
+      ].sort(),
+    );
+  });
+
+  test("Test global scope potential syntactic token scenario", () => {
+    testPotentialSyntacticTokensScenario(
+      "",
+      0,
+      ["abstract", "aligned", "class", "computed", "expandable", "map"].sort(),
+    );
+  });
+
+  test("Test global scope with previous comment potential syntactic token scenario 1", () => {
+    testPotentialSyntacticTokensScenario(
+      "// foo",
+      6,
+      undefined,
+    );
+  });
+
+  test("Test global scope with previous comment potential syntactic token scenario 2", () => {
+    testPotentialSyntacticTokensScenario(
+      "// foo\n ",
+      8,
+      ["abstract", "aligned", "class", "computed", "expandable", "map"].sort(),
+    );
+  });
+});

package/tests/completion/get-potential-token-type-ids.test.ts

@@ -0,0 +1,76 @@
+import { describe, test } from "bun:test";
+import { testPotentialTokenTypeIdsScenario } from "./test-potential-token-type-ids-scenario.ts";
+
+// Generated by Lezer from the SDL grammar
+import * as TokenTypeId from "../../src/lezer/parser.terms.ts";
+
+describe("getPotentialTokenTypeIds Tests", () => {
+  test("Test missing length attribute expected token type scenario", () => {
+    testPotentialTokenTypeIdsScenario(
+      "class A { bit b; }",
+      [TokenTypeId.LengthAttribute],
+    );
+  });
+
+  test("Test duplicate identifier expected token type scenario", () => {
+    testPotentialTokenTypeIdsScenario(
+      "class A {if (f oo) {bit(8) c;}}",
+      [
+        TokenTypeId.Multiplication,
+        TokenTypeId.Division,
+        TokenTypeId.Modulus,
+        TokenTypeId.Addition,
+        TokenTypeId.Subtraction,
+        TokenTypeId.BitwiseShiftLeft,
+        TokenTypeId.BitwiseShiftRight,
+        TokenTypeId.RelationalLessThan,
+        TokenTypeId.RelationalLessThanOrEqual,
+        TokenTypeId.RelationalGreaterThan,
+        TokenTypeId.RelationalGreaterThanOrEqual,
+        TokenTypeId.RelationalEqual,
+        TokenTypeId.RelationalNotEqual,
+        TokenTypeId.BitwiseAnd,
+        TokenTypeId.BitwiseOr,
+        TokenTypeId.LogicalAnd,
+        TokenTypeId.LogicalOr,
+      ].sort((a, b) => a - b),
+    );
+  });
+
+  test("Test complex expected token type scenario", () => {
+    testPotentialTokenTypeIdsScenario(
+      "class A {if (a b == 0b01 || c == 0b11) {bit(8) c;}}",
+      [
+        TokenTypeId.CloseParenthesis,
+        TokenTypeId.Multiplication,
+        TokenTypeId.Division,
+        TokenTypeId.Modulus,
+        TokenTypeId.Addition,
+        TokenTypeId.Subtraction,
+        TokenTypeId.BitwiseShiftLeft,
+        TokenTypeId.BitwiseShiftRight,
+        TokenTypeId.RelationalLessThan,
+        TokenTypeId.RelationalLessThanOrEqual,
+        TokenTypeId.RelationalGreaterThan,
+        TokenTypeId.RelationalGreaterThanOrEqual,
+        TokenTypeId.RelationalEqual,
+        TokenTypeId.RelationalNotEqual,
+        TokenTypeId.BitwiseAnd,
+        TokenTypeId.BitwiseOr,
+        TokenTypeId.LogicalAnd,
+        TokenTypeId.LogicalOr,
+      ].sort((a, b) => a - b),
+    );
+  });
+
+  test("Test global scope expected token type scenario", () => {
+    testPotentialTokenTypeIdsScenario(
+      "",
+      [
+        TokenTypeId.ClassDeclaration,
+        TokenTypeId.ComputedElementaryTypeDefinition,
+        TokenTypeId.MapDeclaration,
+      ].sort((a, b) => a - b),
+    );
+  });
+});

package/tests/completion/test-potential-syntactic-tokens-scenario.ts

@@ -0,0 +1,24 @@
+import { expect } from "bun:test";
+import { createLenientSdlParser } from "../../src/lezer/create-sdl-parser.ts";
+import { SdlStringInput } from "../../index.ts";
+import { getPotentialSyntacticTokens } from "../../src/completion/get-potential-syntactic-tokens.ts";
+
+const lenientSdlParser = createLenientSdlParser();
+
+export function testPotentialSyntacticTokensScenario(
+  source: string,
+  position: number,
+  expectedSyntacticTokens: string[] | undefined,
+) {
+  const sdlStringInput = new SdlStringInput(source);
+  const parseTree = lenientSdlParser.parse(sdlStringInput);
+  const lastNode = parseTree.resolveInner(position, -1);
+
+  const actualSyntacticTokens = getPotentialSyntacticTokens(lastNode.cursor());
+
+  if (!actualSyntacticTokens && expectedSyntacticTokens) {
+    throw new Error("Expected syntactic tokens to be defined");
+  }
+
+  expect(actualSyntacticTokens).toEqual(expectedSyntacticTokens);
+}