@mpeggroup/mpeg-sdl-parser 4.0.0

package/tests/completion/test-potential-token-type-ids-scenario.ts

@@ -0,0 +1,33 @@
+import { expect } from "bun:test";
+import { createLenientSdlParser } from "../../src/lezer/create-sdl-parser.ts";
+import { SdlStringInput } from "../../index.ts";
+import { getPotentialTokenTypeIds } from "../../src/completion/get-potential-token-type-ids.ts";
+
+const lenientSdlParser = createLenientSdlParser();
+
+export function testPotentialTokenTypeIdsScenario(
+  source: string,
+  expectedTokenTypeIds: number[],
+) {
+  const sdlStringInput = new SdlStringInput(source);
+  const parseTree = lenientSdlParser.parse(sdlStringInput);
+  const cursor = parseTree.cursor();
+
+  do {
+    if (cursor.type.isError) {
+      break;
+    }
+  } while (cursor.next());
+
+  const actualTokenTypeIds = getPotentialTokenTypeIds(cursor);
+
+  if (!actualTokenTypeIds) {
+    throw new Error("Expected token type IDs to be defined");
+  }
+
+  // sort and remove duplicates as we have potentially collected from multiple error nodes
+  actualTokenTypeIds.sort((a, b) => a - b);
+  const uniqueActualTokenTypeIds = Array.from(new Set(actualTokenTypeIds));
+
+  expect(uniqueActualTokenTypeIds).toEqual(expectedTokenTypeIds);
+}

package/tests/fixtures/history-recording-node-handler.ts

@@ -0,0 +1,232 @@
+import type { AbstractCompositeNode } from "../../src/ast/node/abstract-composite-node.ts";
+import type { AbstractLeafNode } from "../../src/ast/node/abstract-leaf-node.ts";
+import { NodeKind } from "../../src/ast/node/enum/node-kind.ts";
+import type { NodeHandler } from "../../src/ast/visitor/node-handler.ts";
+
+export default class HistoryRecordingNodeHandler implements NodeHandler {
+  nodeHistory: string[] = [];
+
+  beforeVisit(node: AbstractCompositeNode): void {
+    this.nodeHistory.push(NodeKind[node.nodeKind]);
+  }
+
+  visit(node: AbstractLeafNode): void {
+    this.nodeHistory.push(NodeKind[node.nodeKind]);
+  }
+
+  afterVisit(_node: AbstractCompositeNode): void {
+  }
+}
+
+const expectedHistory = [
+  "SPECIFICATION",
+  "STATEMENT",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "TOKEN",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "TOKEN",
+  "STATEMENT",
+  "IDENTIFIER",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "CLASS_MEMBER_ACCESS",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "STATEMENT",
+  "TOKEN",
+  "STATEMENT",
+  "ELEMENTARY_TYPE",
+  "TOKEN",
+  "LENGTH_ATTRIBUTE",
+  "TOKEN",
+  "EXPRESSION",
+  "NUMBER_LITERAL",
+  "TOKEN",
+  "TOKEN",
+  "IDENTIFIER",
+  "TOKEN",
+  "ARRAY_DIMENSION",
+  "TOKEN",
+  "EXPRESSION",
+  "IDENTIFIER",
+  "TOKEN",
+  "TOKEN",
+  "TOKEN",
+  "TOKEN",
+  "TOKEN",
+];
+
+export { expectedHistory };

package/tests/lezer/create-parse-error-from-text-and-position.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, test } from "bun:test";
+import type { Text } from "@codemirror/state";
+import { SyntaxError } from "../../src/scanner-error.ts";
+import { createSyntaxErrorFromTextAndPosition } from "../../src/lezer/create-syntax-error-from-text-and-position.ts";
+
+// Minimal mock for @codemirror/state Text
+class MockText {
+  private _lines: string[];
+  constructor(text: string) {
+    this._lines = text.split("\n");
+  }
+  get lines(): number {
+    return this._lines.length;
+  }
+  lineAt(pos: number) {
+    let total = 0;
+    for (let i = 0; i < this._lines.length; i++) {
+      const lineLen = this._lines[i].length + 1; // +1 for '\n'
+      if (pos < total + lineLen) {
+        return {
+          number: i + 1,
+          from: total,
+          to: total + this._lines[i].length,
+          text: this._lines[i],
+          length: this._lines[i].length,
+        };
+      }
+      total += lineLen;
+    }
+    // fallback to last line
+    const last = this._lines.length - 1;
+    return {
+      number: this._lines.length,
+      from: total - (this._lines[last].length + 1),
+      to: total,
+      text: this._lines[last],
+      length: this._lines[last].length,
+    };
+  }
+  line(n: number) {
+    // Line numbers in Text interface are 1-indexed, array is 0-indexed
+    const index = n - 1;
+    if (index < 0 || index >= this._lines.length) {
+      throw new Error(`Line ${n} is out of range (1-${this._lines.length})`);
+    }
+    return {
+      number: n,
+      from: this.getLineStart(n),
+      to: this.getLineStart(n) + this._lines[index].length,
+      text: this._lines[index],
+      length: this._lines[index].length,
+    };
+  }
+  getLineStart(n: number) {
+    let pos = 0;
+    for (let i = 0; i < n - 1; i++) {
+      pos += this._lines[i].length + 1;
+    }
+    return pos;
+  }
+}
+
+describe("createSyntaxErrorFromTextAndPosition Tests", () => {
+  test("creates a SyntaxError with correct location and errorLine", () => {
+    const text = new MockText("foo\nbar\nbaz");
+    // Position 5 is in line 2 ("bar"), column 1
+    const err = createSyntaxErrorFromTextAndPosition(
+      text as unknown as Text,
+      4,
+    );
+
+    expect(err).toBeInstanceOf(SyntaxError);
+    expect(err.location).toEqual({ row: 2, column: 1, position: 4 });
+    expect(err.errorLine).toBe("bar");
+    expect(err.precedingLines).toEqual(["foo"]);
+    expect(err.message).toContain("SYNTAX ERROR:");
+  });
+
+  test("creates a SyntaxError handling position at start of text", () => {
+    const text = new MockText("abc\ndef");
+    const err = createSyntaxErrorFromTextAndPosition(
+      text as unknown as Text,
+      0,
+    );
+
+    expect(err.location).toEqual({ row: 1, column: 1, position: 0 });
+    expect(err.errorLine).toBe("abc");
+    expect(err.precedingLines).toEqual([]);
+  });
+
+  test("creates a SyntaxError handling position at start of a later line", () => {
+    const text = new MockText("abc\ndef\nghi");
+    // Position 4 is start of line 2 ("def")
+    const err = createSyntaxErrorFromTextAndPosition(
+      text as unknown as Text,
+      4,
+    );
+
+    expect(err.location).toEqual({ row: 2, column: 1, position: 4 });
+    expect(err.errorLine).toBe("def");
+    expect(err.precedingLines).toEqual(["abc"]);
+  });
+
+  test("creates a SyntaxError including up to two preceding lines", () => {
+    const text = new MockText("a\nb\nc\nd");
+    // Position 6 is in line 4 ("d")
+    const err = createSyntaxErrorFromTextAndPosition(
+      text as unknown as Text,
+      6,
+    );
+
+    expect(err.location?.row).toBe(4);
+    expect(err.errorLine).toBe("d");
+    expect(err.precedingLines).toEqual(["b", "c"]);
+  });
+
+  test("creates a SyntaxError handling position at last line", () => {
+    const text = new MockText("x\ny\nz");
+    // Position 4 is in line 3 ("z")
+    const err = createSyntaxErrorFromTextAndPosition(
+      text as unknown as Text,
+      4,
+    );
+
+    expect(err.location?.row).toBe(3);
+    expect(err.errorLine).toBe("z");
+    expect(err.precedingLines).toEqual(["x", "y"]);
+  });
+});

package/tests/lezer/create-sdl-parser.test.ts

@@ -0,0 +1,15 @@
+import { describe, test } from "bun:test";
+import {
+  createLenientSdlParser,
+  createStrictSdlParser,
+} from "../../src/lezer/create-sdl-parser.ts";
+
+describe("createSdlParser Tests", () => {
+  test("Test lenient SDL parser creation", () => {
+    createLenientSdlParser();
+  });
+
+  test("Test strict SDL parser creation", () => {
+    createStrictSdlParser();
+  });
+});

package/tests/lezer/sdl-parser-scenarios.test.ts

@@ -0,0 +1,24 @@
+import fs from "node:fs";
+import * as path from "path";
+import { describe, test } from "bun:test";
+import { createLenientSdlParser } from "../../src/lezer/create-sdl-parser.ts";
+import { fileTests } from "@lezer/generator/dist/test";
+
+const sdlParser = createLenientSdlParser();
+const testCaseDir = path.join(__dirname, "./test-cases");
+
+for (const filename of fs.readdirSync(testCaseDir)) {
+  if (!/\.txt$/.test(filename)) {
+    continue;
+  }
+
+  const scenarioName = /^[^\.]*/.exec(filename)?.[0] || "??";
+
+  describe(`SDL Parser ${scenarioName} Tests`, () => {
+    const testCases = fs.readFileSync(path.join(testCaseDir, filename), "utf8");
+
+    for (const { name, run } of fileTests(testCases, filename)) {
+      test(`Test SDL Parser ${scenarioName} - ${name}`, () => run(sdlParser));
+    }
+  });
+}

package/tests/lezer/sdl-string-input.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, test } from "bun:test";
+import { SdlStringInput } from "../../src/lezer/sdl-string-input.ts";
+
+describe("SdlStringInput Tests", () => {
+  test("Test creation and basic behavior", () => {
+    const input = new SdlStringInput("test 1\ntest 2\n");
+    expect(input.length).toBe(14);
+    expect(input.chunk(0)).toBe("test 1\ntest 2\n");
+    expect(input.read(0, 5)).toBe("test ");
+    expect(input.read(5, 12)).toBe("1\ntest ");
+  });
+});

package/tests/lezer/syntax-error-scenarios.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+
+import { createStrictSdlParser } from "../../src/lezer/create-sdl-parser.ts";
+import { SdlStringInput } from "../../src/lezer/sdl-string-input.ts";
+
+const sdlParser = createStrictSdlParser();
+
+describe("Syntax Error Scenario Tests", () => {
+  test("Empty declarations parses", () => {
+    const sdlStringInput = new SdlStringInput("");
+
+    sdlParser.parse(sdlStringInput);
+  });
+
+  test("Declarations with single invalid token fails to parse", () => {
+    const sdlStringInput = new SdlStringInput("§");
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 1, position: 0 }",
+    );
+  });
+
+  test("No class parameter values in parenthesis fails to parse", () => {
+    const sdlStringInput = new SdlStringInput("class A {ClassD d();}");
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 19, position: 18 }",
+    );
+  });
+
+  test("Unexpected token fails to parse", () => {
+    const sdlStringInput = new SdlStringInput("class A {ClassD computed d;}");
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 17, position: 16 }",
+    );
+  });
+
+  test("Trailing comma in class parameter values fails to parse", () => {
+    const sdlStringInput = new SdlStringInput("class A {ClassD d(3,);}");
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 21, position: 20 }",
+    );
+  });
+
+  test("Mix of concatenated string literal types fails to parse", () => {
+    let sdlStringInput = new SdlStringInput(
+      'class A {utf8string d = "hello" u"world";}',
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 25, position: 24 }",
+    );
+
+    sdlStringInput = new SdlStringInput(
+      'class A {utf8string d = u"hello" "world";}',
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 34, position: 33 }",
+    );
+  });
+
+  test("Both legacy and reserved together fails to parse", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A {reserved legacy utfstring foo;}",
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 19, position: 18 }",
+    );
+  });
+
+  test("Invalid UTF string literal type fails to parse", () => {
+    const sdlStringInput = new SdlStringInput(
+      'class A {base64string foo = u"aGVsbG8K";}',
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 29, position: 28 }",
+    );
+  });
+
+  test("Invalid basic string literal type fails to parse", () => {
+    const sdlStringInput = new SdlStringInput(
+      'class A {utf8string foo = "hello";}',
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 27, position: 26 }",
+    );
+  });
+
+  test("Invalid prefix for string literal", () => {
+    const sdlStringInput = new SdlStringInput(
+      'class A {utf8string foo = u8"hello";}',
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 28, position: 27 }",
+    );
+  });
+
+  test("Illegal alignment bit count value fails to parse", () => {
+    const sdlStringInput = new SdlStringInput(
+      "class A {aligned(17) utf8string foo;}",
+    );
+
+    expect(() => sdlParser.parse(sdlStringInput)).toThrow(
+      "SYNTAX ERROR: => { row: 1, column: 18, position: 17 }",
+    );
+  });
+});

package/tests/lezer/test-cases/array_definition.txt

@@ -0,0 +1,109 @@
+# Simple
+
+class A {int(3) a[3];}
+
+==>
+
+Specification(
+    ClassDeclaration(
+        class
+        Whitespace
+        Identifier
+        Whitespace
+        OpenBrace
+        ArrayDefinition(
+            ElementaryType(
+                int
+            )
+            LengthAttribute(
+                OpenParenthesis
+                UnaryExpression(
+                    IntegerLiteral
+                )
+                CloseParenthesis
+            )
+            Whitespace
+            Identifier
+            ExplicitArrayDimension(
+                OpenBracket
+                UnaryExpression(
+                    IntegerLiteral
+                )
+                CloseBracket
+            )
+            Semicolon
+        )
+        CloseBrace
+    )
+)
+
+# Reserved multiple dimensions with mixed explicit and partial dimensions
+
+class A {reserved MyClass a[3][[2]];}
+
+==>
+
+Specification(
+    ClassDeclaration(
+        class
+        Whitespace
+        Identifier
+        Whitespace
+        OpenBrace
+        ArrayDefinition(
+            reserved
+            Whitespace
+            Identifier
+            Whitespace
+            Identifier
+            ExplicitArrayDimension(
+                OpenBracket
+                UnaryExpression(
+                    IntegerLiteral
+                )
+                CloseBracket
+            )
+            PartialArrayDimension(
+                OpenBracket
+                OpenBracket
+                UnaryExpression(
+                    IntegerLiteral
+                )
+                CloseBracket
+                CloseBracket
+            )
+            Semicolon
+        )
+        CloseBrace
+    )
+)
+
+# Implicit dimension
+
+class A {MyClass a[1..3];}
+
+==>
+
+Specification(
+    ClassDeclaration(
+        class
+        Whitespace
+        Identifier
+        Whitespace
+        OpenBrace
+        ArrayDefinition(
+            Identifier
+            Whitespace
+            Identifier
+            ImplicitArrayDimension(
+                OpenBracket
+                IntegerLiteral
+                RangeOperator
+                IntegerLiteral
+                CloseBracket
+            )
+            Semicolon
+        )
+        CloseBrace
+    )
+)