@scinorandex/sparse 0.0.8 → 0.1.2

package/test/grammar.test.ts

@@ -0,0 +1,243 @@
+import { describe, expect, it } from "vitest";
+import { dehydateProduction, hydrateProduction } from "../src/meta/common";
+import {
+  buildProductions,
+  generateStates,
+  namedProductions,
+  tryBuildProductions,
+  validateProductions,
+  type Production,
+  type ProductionWarning,
+} from "../src/index";
+import { EXAMPLE_GRAMMARS, readFile } from "./helpers";
+
+const MATH_GRAMMAR = `<S>: <PROGRAM>;
+<PROGRAM: program>: [EOF];
+`;
+
+const kleeneGrammar = (extra: string) => `<S>: <PROGRAM>;
+<PROGRAM: program>: [NUMBER] ([PLUS] [NUMBER])* [EOF];${extra}
+`;
+
+describe("grammar parsing", () => {
+  it.each(EXAMPLE_GRAMMARS)("builds productions for the %s grammar", async (_name, grammarPath) => {
+    const grammar = await readFile(grammarPath);
+    const result = tryBuildProductions(grammar);
+    if (result.success === false) throw new Error(`${result.reason} at ${result.token.line}:${result.token.column}`);
+    expect(result.value.length).toBeGreaterThan(0);
+  });
+
+  it("keeps production names and identifiers", () => {
+    const productions = buildProductions(MATH_GRAMMAR);
+    expect(productions[0].identifier).toBe("<S>");
+    expect(productions[0].name).toBe(null);
+    expect(productions[1].identifier).toBe("<PROGRAM>");
+    expect(productions[1].name).toBe("program");
+    expect(namedProductions(productions)).toEqual(["program"]);
+  });
+
+  it("unrolls * into an autogenerated list production that the user has to reduce", () => {
+    const productions = buildProductions(kleeneGrammar(""));
+    expect(productions[0].identifier).toBe("<S>");
+
+    const kleene = productions.filter((production) => production.name === "autogenerated-kleene");
+    expect(kleene.length).toBeGreaterThan(0);
+    expect(kleene.every((production) => production.identifier.startsWith("<autogen-"))).toBe(true);
+    expect(namedProductions(productions)).toContain("autogenerated-kleene");
+  });
+
+  it("unrolls + into a list production that contains at least one item", () => {
+    const productions = buildProductions(`<S>: <PROGRAM>;
+<PROGRAM: program>: ([PLUS] [NUMBER])+ [EOF];
+`);
+    const kleene = productions.filter((production) => production.name === "autogenerated-kleene");
+    expect(kleene.length).toBeGreaterThan(0);
+    expect(kleene.some((production) => production.rhs.length > 1)).toBe(true);
+  });
+
+  it("unrolls ? into one production with the group and one without", () => {
+    const productions = buildProductions(`<S>: <PROGRAM>;
+<PROGRAM: program>: [NUMBER] ([PLUS] [NUMBER])? [EOF];
+`);
+    const program = productions.filter((production) => production.identifier === "<PROGRAM>");
+    expect(program.length).toBe(2);
+    expect(program[0].rhs.map((item) => item.identifier)).toEqual(["[NUMBER]", "[EOF]"]);
+    expect(program[1].rhs.map((item) => item.identifier)).toEqual(["[NUMBER]", "[PLUS]", "[NUMBER]", "[EOF]"]);
+  });
+
+  it("unrolls terminal alternatives into one production per alternative", () => {
+    const productions = buildProductions(`<S>: <PROGRAM>;
+<PROGRAM: program>: [PLUS | MINUS] [EOF];
+`);
+    const program = productions.filter((production) => production.identifier === "<PROGRAM>");
+    expect(program.length).toBe(2);
+    expect(program[0].rhs.map((item) => item.identifier)).toEqual(["[PLUS]", "[EOF]"]);
+    expect(program[1].rhs.map((item) => item.identifier)).toEqual(["[MINUS]", "[EOF]"]);
+  });
+
+  it("unrolls variable alternatives into one production per alternative", () => {
+    const productions = buildProductions(`<S>: <PROGRAM>;
+<PROGRAM: program>: <A | B> [EOF];
+<A>: [EOF];
+<B>: [EOF];
+`);
+    const program = productions.filter((production) => production.identifier === "<PROGRAM>");
+    expect(program.length).toBe(2);
+    expect(program.map((production) => production.rhs[0].identifier)).toEqual(["<A>", "<B>"]);
+  });
+
+  it("ignores comments", () => {
+    const withComments = `// a comment
+<S>: <PROGRAM>; /* another
+comment */
+<PROGRAM: program>: [EOF];
+`;
+    expect(buildProductions(withComments).length).toBe(2);
+  });
+
+  describe("reports problems as Result failures instead of throwing", () => {
+    const cases: [string, string, string][] = [
+      ["empty grammar", ``, "does not contain any productions"],
+      [
+        "missing semicolon",
+        `<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER]\n`,
+        "expected one of",
+      ],
+      [
+        "undefined variable in the first position",
+        `<S>: <PROGRAM>;\n<PROGRAM: program>: <NOPE> [EOF];\n`,
+        `Variable "<NOPE>" is used on the right hand side`,
+      ],
+      [
+        "undefined variable in a later position",
+        `<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER] <NOPE> [EOF];\n`,
+        `Variable "<NOPE>" is used on the right hand side`,
+      ],
+      [
+        "group that expands to nothing",
+        `<S>: <PROGRAM>;\n<PROGRAM: program>: ([NUMBER])?;\n`,
+        "empty right hand side",
+      ],
+      [
+        "start symbol used on the right hand side",
+        `<S>: <PROGRAM>;\n<PROGRAM: program>: <S> [EOF];\n`,
+        "cannot be used on the right hand side",
+      ],
+    ];
+
+    it.each(cases)("rejects a %s", (_name, grammar, expectedReason) => {
+      const result = tryBuildProductions(grammar);
+      expect(result.success).toBe(false);
+      if (result.success === false) {
+        expect(result.reason).toContain(expectedReason);
+        expect(result.token.line).toBeGreaterThan(0);
+        expect(result.token.column).toBeGreaterThanOrEqual(0);
+      }
+    });
+
+    it("points at the offending token", () => {
+      const result = tryBuildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER] <NOPE> [EOF];\n`);
+      if (result.success === false) {
+        expect(result.token.line).toBe(2);
+        expect(result.token.lexeme).toBe("NOPE");
+      } else throw new Error("expected a failure");
+    });
+
+    it("throws from buildProductions with the position in the message", () => {
+      expect(() => buildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: <NOPE> [EOF];\n`)).toThrow(
+        /Encountered error .* at 2:\d+/,
+      );
+    });
+  });
+});
+
+describe("validateProductions", () => {
+  it("accepts every grammar checked into the repository", async () => {
+    for (const [name, grammarPath] of EXAMPLE_GRAMMARS) {
+      const grammar = await readFile(grammarPath);
+      const result = validateProductions(buildProductions(grammar));
+      expect(result.success, `${name} should be valid`).toBe(true);
+    }
+  });
+
+  it("warns about duplicate right hand side names but still succeeds", () => {
+    const warnings: ProductionWarning[] = [];
+    const result = validateProductions(
+      buildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER: n] [EOF: n];\n`),
+      { onWarning: (warning) => warnings.push(warning) },
+    );
+    expect(result.success).toBe(true);
+    expect(warnings.map((warning) => warning.kind)).toContain("duplicate-rhs-name");
+  });
+
+  it("warns about unreachable productions but still succeeds", () => {
+    const warnings: ProductionWarning[] = [];
+    const result = validateProductions(
+      buildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: [EOF];\n<ORPHAN: orphan>: [EOF];\n`),
+      { onWarning: (warning) => warnings.push(warning) },
+    );
+    expect(result.success).toBe(true);
+    expect(warnings.map((warning) => warning.kind)).toContain("unreachable-production");
+  });
+
+  it("rejects an empty production list", () => {
+    const result = validateProductions([]);
+    expect(result.success).toBe(false);
+    if (result.success === false) expect(result.token.line).toBe(1);
+  });
+});
+
+describe("generateStates validation", () => {
+  it("refuses to generate states from an empty grammar", () => {
+    const result = generateStates([]);
+    expect(result.success).toBe(false);
+  });
+
+  it("refuses a grammar with an undefined variable instead of building a broken table", () => {
+    const grammar = `<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER] <NOPE> [EOF];\n`;
+
+    // Rejected while reading the grammar...
+    expect(() => buildProductions(grammar)).toThrow(/Variable "<NOPE>"/);
+
+    // ...and again when hand-built productions are handed straight to the generator, which used to
+    // happily produce a table that crashed every parse with a TypeError.
+    const valid = buildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: [NUMBER] [EOF];\n`);
+    const withDanglingVariable: Production[] = [
+      valid[0],
+      valid[1],
+      {
+        ...valid[1],
+        rhs: [...valid[1].rhs.slice(0, 1), { type: "variable", token: valid[1].rhs[0].token, identifier: "<NOPE>", name: null }],
+      },
+    ];
+
+    const result = generateStates(withDanglingVariable);
+    expect(result.success).toBe(false);
+    if (result.success === false) expect(result.reason).toContain(`Variable "<NOPE>"`);
+  });
+
+  it("reports how many states it produced through onProgress", () => {
+    let last = 0;
+    generateStates(buildProductions(MATH_GRAMMAR), { onProgress: (states) => (last = states) });
+    expect(last).toBeGreaterThan(0);
+  });
+
+  it("forwards grammar warnings", () => {
+    const warnings: ProductionWarning[] = [];
+    generateStates(buildProductions(`<S>: <PROGRAM>;\n<PROGRAM: program>: [EOF];\n<ORPHAN: orphan>: [EOF];\n`), {
+      onWarning: (warning) => warnings.push(warning),
+    });
+    expect(warnings.map((warning) => warning.kind)).toEqual(["unreachable-production"]);
+  });
+});
+
+describe("grammar round trip", () => {
+  it("keeps every production's identifier and name through hydrate/dehydrate", () => {
+    const productions: Production[] = buildProductions(kleeneGrammar(""));
+    const round = productions.map((production) => hydrateProduction(dehydateProduction(production)));
+    expect(round.map((production) => production.identifier)).toEqual(
+      productions.map((production) => production.identifier),
+    );
+    expect(round.map((production) => production.name)).toEqual(productions.map((production) => production.name));
+  });
+});

package/test/helpers.ts

@@ -0,0 +1,59 @@
+import path from "path";
+import fs from "fs/promises";
+import { Slex } from "@scinorandex/slex";
+
+// prettier-ignore
+export enum MathTokenType {
+  PLUS, MINUS, STAR, SLASH, NUMBER, EOF
+}
+
+export type MathMetadata = {};
+
+export const mathLexerGenerator = new Slex<MathTokenType, MathMetadata>({
+  EOF_TYPE: MathTokenType.EOF,
+  isHigherPrecedence: () => false,
+});
+
+mathLexerGenerator.addRule("plus", "$+", MathTokenType.PLUS);
+mathLexerGenerator.addRule("minus", "$-", MathTokenType.MINUS);
+mathLexerGenerator.addRule("star", "$*", MathTokenType.STAR);
+mathLexerGenerator.addRule("forward_slash", "$/", MathTokenType.SLASH);
+mathLexerGenerator.addRule("digit", "0|1|2|3|4|5|6|7|8|9");
+mathLexerGenerator.addRule("float_number", "(${digit})+$.(${digit})+");
+mathLexerGenerator.addRule("decimal_number", "(${digit})+");
+mathLexerGenerator.addRule("number_literal", "${float_number}|${decimal_number}", MathTokenType.NUMBER);
+
+export const mathLexer = (source: string) => mathLexerGenerator.generate(source, () => ({}));
+
+// prettier-ignore
+export enum SemiTokenType {
+  NUMBER, SEMICOLON, EOF
+}
+
+/** A lexer whose grammar mentions a SEMICOLON, for the error recovery tests. */
+export const semiLexer = (source: string) => {
+  const generator = new Slex<SemiTokenType, {}>({
+    EOF_TYPE: SemiTokenType.EOF,
+    isHigherPrecedence: () => false,
+  });
+  generator.addRule("semicolon", "$;", SemiTokenType.SEMICOLON);
+  generator.addRule("digit", "0|1|2|3|4|5|6|7|8|9");
+  generator.addRule("number", "(${digit})+", SemiTokenType.NUMBER);
+  return generator.generate(source, () => ({}));
+};
+
+export const root = path.join(__dirname, "..");
+
+export const examplePath = (...segments: string[]) => path.join(root, "example", ...segments);
+
+export const readExample = (...segments: string[]) => fs.readFile(examplePath(...segments), "utf8");
+
+export const readFile = (filePath: string) => fs.readFile(path.resolve(root, "example", filePath), "utf8");
+
+export const EXAMPLE_GRAMMARS: [string, string][] = [
+  ["math", examplePath("math/grammar.txt")],
+  ["kleene-test", examplePath("kleene-test/grammar.txt")],
+  ["LoLang", examplePath("LoLang/grammar.txt")],
+  ["selfhosted meta", examplePath("selfhosted/01-meta-grammar.txt")],
+  ["selfhosted table", examplePath("selfhosted/02-table-grammar.txt")],
+];

package/test/lalr.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from "vitest";
+import fs from "fs/promises";
+import {
+  LR1StackSymbol,
+  Sparse,
+  buildProductions,
+  buildStates,
+  enumToString,
+  generateStates,
+} from "../src/index";
+import { MathTokenType, mathLexer, readFile } from "./helpers";
+import { lexerGenerator as loLangLexerGenerator, LoLangTokenType } from "../example/LoLang/example";
+
+type StackSymbol = LR1StackSymbol<any, any, AstNode>;
+
+class AstNode {
+  constructor(public readonly nodes: StackSymbol[]) {}
+  toObject(): unknown[] {
+    return this.nodes.map((node) => (node.type === "token" ? node.token.lexeme : (node.node as AstNode).toObject()));
+  }
+}
+
+const reducer = (_newInput: any, oldInput: { input: LR1StackSymbol<any, any, AstNode>[] }) =>
+  new AstNode(oldInput.input);
+
+const toStringifiedTokenType = enumToString<MathTokenType>(MathTokenType);
+const MATH_GRAMMAR = `${__dirname}/../example/math/grammar.txt`;
+const EXPRESSIONS = ["1 + 2", "42", "3 * 4 - 5 / 6", "1.5 + 2.25 * 3"];
+
+const mathTable = async () => buildStates(await fs.readFile(`${__dirname}/../example/math/table.txt`, "utf8"));
+
+describe("LALR(1)", () => {
+  it("produces the same trees as the committed LR(1) table for math expressions", async () => {
+    const productions = buildProductions(await fs.readFile(MATH_GRAMMAR, "utf8"));
+    const states = await mathTable();
+
+    const fromTable = new Sparse<MathTokenType, {}, AstNode>({ productions, states, toStringifiedTokenType });
+    const fromLalr = Sparse.fromProductions<MathTokenType, {}, AstNode>({
+      productions,
+      toStringifiedTokenType,
+      mode: "lalr1",
+      quiet: true,
+    });
+
+    for (const expression of EXPRESSIONS) {
+      const lr1 = fromTable.generate(mathLexer(expression), { reducer }).parse().result?.toObject();
+      const lalr = fromLalr.generate(mathLexer(expression), { reducer }).parse().result?.toObject();
+      expect(lalr, `parsing "${expression}"`).toEqual(lr1);
+    }
+  });
+
+  it("agrees with LR(1) on the LoLang example", async () => {
+    const productions = buildProductions(await readFile("LoLang/grammar.txt"));
+    const source = await readFile("LoLang/Features_Array_Methods.lol");
+
+    const lr1 = generateStates(productions);
+    const lalr = generateStates(productions, { mode: "lalr1" });
+    if (lr1.success === false || lalr.success === false) throw new Error("generation failed");
+
+    const parse = (states: any[]) =>
+      new Sparse<LoLangTokenType, {}, AstNode>({
+        productions,
+        states,
+        toStringifiedTokenType: enumToString<LoLangTokenType>(LoLangTokenType),
+        validate: true,
+      })
+        .generate(loLangLexerGenerator.generate(source, () => ({})), { reducer })
+        .parse().result?.toObject();
+
+    const fromLr1 = parse(lr1.value.toStates());
+    const fromLalr = parse(lalr.value.toStates());
+
+    expect(fromLalr).toEqual(fromLr1);
+  }, 120_000);
+
+  it("is never larger than LR(1)", async () => {
+    const productions = buildProductions(await readFile("LoLang/grammar.txt"));
+
+    const lr1 = generateStates(productions);
+    const lalr = generateStates(productions, { mode: "lalr1" });
+    if (lr1.success === false || lalr.success === false) throw new Error("generation failed");
+
+    expect(lalr.value.toStates().length).toBeLessThan(lr1.value.toStates().length);
+  }, 120_000);
+
+  it("builds the same table in both modes for an unambiguous grammar", () => {
+    const productions = buildProductions(`<S>: <A>;
+<A: a>: [A] [PLUS] [A];
+<A: a>: [ID];
+`);
+
+    const lr1 = generateStates(productions);
+    const lalr = generateStates(productions, { mode: "lalr1" });
+    if (lr1.success === false || lalr.success === false) throw new Error("generation failed");
+
+    expect(lalr.value.toTable()).toBe(lr1.value.toTable());
+  });
+
+  it("refuses to serialize a grammar that has a state without actions", () => {
+    const productions = buildProductions(`<S>: <A>;
+<A>: <A> <C>;
+<B>: <A> [d] <C>;
+<B>: <C> <C>;
+<C>: <A>;
+`);
+    const result = generateStates(productions);
+    if (result.success === false) throw new Error(result.reason);
+
+    // The states are usable in memory...
+    expect(result.value.toStates().some((state) => state.actions.size === 0)).toBe(true);
+    // ...but the table file format has no way to express a state without actions.
+    expect(() => result.value.toTable()).toThrow(/has no actions/);
+  });
+});