@scinorandex/sparse 0.0.2 → 0.0.3

package/src/generator.ts

@@ -1,5 +1,5 @@
-import { GrammarToken, Production } from "./grammarParser";
 import { TableState } from "./parser";
+import { GrammarToken, Production } from "./meta/common";
 import { Result } from "./utils/Result";
 
 const xContainsAllOfY = <T>(xs: Set<T>, ys: Set<T>) => [...ys].every((x) => xs.has(x));
@@ -9,7 +9,7 @@ const EOF_STRING = "[EOF]";
 function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<string>>> {
   // Initialize the hashmap
   const ret = new Map<string, Set<string>>();
-  for (const production of allProductions) ret.set(production.lhs.lexeme, new Set());
+  for (const production of allProductions) ret.set(production.identifier, new Set());
 
   // Keep iterating through all productions until no edits are made
   let wasEdited = true;
@@ -17,16 +17,16 @@ function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<
     wasEdited = false;
 
     for (const production of allProductions) {
-      const toBeModified = ret.get(production.lhs.lexeme)!;
+      const toBeModified = ret.get(production.identifier)!;
 
       const firstRhsToken = production.rhs[0];
       if (firstRhsToken.type === "variable") {
-        const add = ret.get(firstRhsToken.token.lexeme);
+        const add = ret.get(firstRhsToken.identifier);
 
         if (add == null) {
           return {
             success: false,
-            reason: `Variable "${firstRhsToken.token.lexeme}" doesn't have a corresponding left hand side`,
+            reason: `Variable "${firstRhsToken.identifier}" doesn't have a corresponding left hand side`,
             token: firstRhsToken.token,
           };
         }
@@ -35,13 +35,13 @@ function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<
         // if not then add them and set wasEdited to true so we iterate one more time
         if (!xContainsAllOfY(toBeModified, add)) {
           wasEdited = true;
-          ret.set(production.lhs.lexeme, new Set([...toBeModified, ...add]));
+          ret.set(production.identifier, new Set([...toBeModified, ...add]));
         }
       } else if (firstRhsToken.type === "terminal") {
         // add terminal to toBeModified and set wasEdited to true
-        if (toBeModified.has(firstRhsToken.token.lexeme) == false) {
+        if (toBeModified.has(firstRhsToken.identifier) == false) {
           wasEdited = true;
-          ret.set(production.lhs.lexeme, new Set([...toBeModified, firstRhsToken.token.lexeme]));
+          ret.set(production.identifier, new Set([...toBeModified, firstRhsToken.identifier]));
         }
       }
     }
@@ -51,7 +51,7 @@ function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<
 }
 
 function computeFollowSets(
-  allProductions: Production[]
+  allProductions: Production[],
 ): Result<{ firstSets: Map<string, Set<string>>; followSets: Map<string, Set<string>> }> {
   const firstSetsResult = computeFirstSets(allProductions);
   if (firstSetsResult.success === false) return firstSetsResult;
@@ -60,8 +60,8 @@ function computeFollowSets(
 
   // create a list of follow sets and add EOF to the initial production
   const ret = new Map<string, Set<string>>();
-  for (const production of allProductions) ret.set(production.lhs.lexeme, new Set());
-  ret.get(allProductions[0].lhs.lexeme)!.add(EOF_STRING);
+  for (const production of allProductions) ret.set(production.identifier, new Set());
+  ret.get(allProductions[0].identifier)!.add(EOF_STRING);
 
   let wasEdited = true;
   while (wasEdited) {
@@ -75,11 +75,11 @@ function computeFollowSets(
         for (let i = 0; i < rhs.length; i++) {
           const currentRhsToken = rhs[i];
 
-          if (currentRhsToken.type === "variable" && currentRhsToken.token.lexeme === variable) {
+          if (currentRhsToken.type === "variable" && currentRhsToken.identifier === variable) {
             if (i === rhs.length - 1) {
               // we are at the end of rhs, so whatever is in currentProduction
               // we need to also need to add to variable
-              const toBeAdded = ret.get(currentProduction.lhs.lexeme)!;
+              const toBeAdded = ret.get(currentProduction.identifier)!;
               const receiver = ret.get(variable)!;
               if (!xContainsAllOfY(receiver, toBeAdded)) {
                 wasEdited = true;
@@ -91,18 +91,18 @@ function computeFollowSets(
               if (nextRhsToken.type === "terminal") {
                 // next token is a terminal, check if its in existng follow set and add if it doesn't exist
                 const existing = ret.get(variable)!;
-                if (!existing.has(nextRhsToken.token.lexeme)) {
+                if (!existing.has(nextRhsToken.identifier)) {
                   wasEdited = true;
-                  ret.set(variable, new Set([...existing, nextRhsToken.token.lexeme]));
+                  ret.set(variable, new Set([...existing, nextRhsToken.identifier]));
                 }
               } else if (nextRhsToken.type === "variable") {
                 // next token is a variable, get the first set of the variable
                 // check if it's not in the followset and add if not
-                const firstSet = firstSets.get(nextRhsToken.token.lexeme);
+                const firstSet = firstSets.get(nextRhsToken.identifier);
                 if (firstSet == null)
                   return {
                     success: false,
-                    reason: `Variable "${nextRhsToken.token.lexeme}" doesn't have a corresponding left hand side`,
+                    reason: `Variable "${nextRhsToken.identifier}" doesn't have a corresponding left hand side`,
                     token: nextRhsToken.token,
                   };
 
@@ -134,27 +134,28 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   // The implementation of this function ensures that the table is an LR(1) parsing table
   const determineNextLookAhead = (
     lhs: GrammarToken,
-    array: { type: "terminal" | "variable"; token: GrammarToken }[]
+    identifier: string,
+    array: { type: "terminal" | "variable"; token: GrammarToken; identifier: string }[],
   ): Result<string[]> => {
     if (array.length === 0) {
-      const followSet = followSets.get(lhs.lexeme);
+      const followSet = followSets.get(identifier);
       if (followSet != undefined) return { success: true, value: [...followSet] };
 
       return {
         success: false,
-        reason: `Variable ${lhs.lexeme} not present in computed follow sets`,
+        reason: `Variable ${identifier} not present in computed follow sets`,
         token: lhs,
       };
     } else {
       const next = array[0];
 
-      if (next.type === "terminal") return { success: true, value: [next.token.lexeme] };
-      const testing = firstSets.get(next.token.lexeme);
+      if (next.type === "terminal") return { success: true, value: [next.identifier] };
+      const testing = firstSets.get(next.identifier);
       if (testing !== undefined) return { success: true, value: [...testing] };
 
       return {
         success: false,
-        reason: `Variable ${next.token.lexeme} not present in computed first sets`,
+        reason: `Variable ${next.identifier} not present in computed first sets`,
         token: next.token,
       };
     }
@@ -163,7 +164,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   // This fnuction creates the initial item set based on the
   // production provided. It creating the initial item and expands it
   function generateInitialItemSet(production: Production) {
-    const initialItem = { lhs: production.lhs, rhs: production.rhs, dot: 0, lookahead: [EOF_STRING] } as Item;
+    const initialItem = {
+      lhs: production.lhs,
+      identifier: production.identifier,
+      rhs: production.rhs,
+      dot: 0,
+      lookahead: [EOF_STRING],
+    } as Item;
+
     return expandItemSet([initialItem]);
   }
 
@@ -182,19 +190,25 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
 
       if (after.type === "variable") {
         // Find prodctions whose left hand side is the symbol after the dot
-        const newProductions = productions.filter((p) => p.lhs.lexeme === after.token.lexeme);
+        const newProductions = productions.filter((p) => p.identifier === after.identifier);
 
         // Compute the lookahead for the new productions to be added to the item set
         const rest = currentItem.rhs.slice(currentItem.dot + 1);
 
-        const lookaheadResult = determineNextLookAhead(currentItem.lhs, rest);
+        const lookaheadResult = determineNextLookAhead(currentItem.lhs, currentItem.identifier, rest);
         if (lookaheadResult.success === false) return lookaheadResult;
         const lookahead = lookaheadResult.value;
 
         for (const newProduction of newProductions) {
           // Create the new item and check if it already exists in the item set
           // If it doesn't exist, add it to the item set and the queue of unprocessed items
-          const newItem = { lhs: newProduction.lhs, rhs: newProduction.rhs, dot: 0, lookahead } as Item;
+          const newItem = {
+            lhs: newProduction.lhs,
+            identifier: newProduction.identifier,
+            rhs: newProduction.rhs,
+            dot: 0,
+            lookahead,
+          } as Item;
           const encoding = JSON.stringify(newItem);
 
           if (itemSet.some((i) => JSON.stringify(i) === encoding) == false) {
@@ -247,7 +261,7 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           const productionToReduceTo = productions
             .map((p, i) => [p, i] as const)
             .find(
-              ([p, _]) => JSON.stringify(p.rhs) === JSON.stringify(item.rhs) && p.lhs.lexeme === item.lhs.lexeme
+              ([p, _]) => JSON.stringify(p.rhs) === JSON.stringify(item.rhs) && p.identifier === item.identifier,
             )![1];
 
           // for each lookahead in the item, create a new entry in the action table to reduce to the production found
@@ -266,14 +280,13 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         if (nextSymbol.type === "variable") {
           // Next symbol is a variable so we need to add it to the GOTO table
           // Check if the GOTO table already has an entry for the next symbola and either push or create the array
-          if (nextStates_Goto.has(nextSymbol.token.lexeme)) nextStates_Goto.get(nextSymbol.token.lexeme)!.push(newItem);
-          else nextStates_Goto.set(nextSymbol.token.lexeme, [newItem]);
+          if (nextStates_Goto.has(nextSymbol.identifier)) nextStates_Goto.get(nextSymbol.identifier)!.push(newItem);
+          else nextStates_Goto.set(nextSymbol.identifier, [newItem]);
         } else if (nextSymbol.type === "terminal") {
           // Next symbol is a terminal so we need to add it to the SHIFT table
           // Check if the SHIFT table already has an entry for the next symbola and either push or create the array
-          if (nextStates_Shift.has(nextSymbol.token.lexeme))
-            nextStates_Shift.get(nextSymbol.token.lexeme)!.push(newItem);
-          else nextStates_Shift.set(nextSymbol.token.lexeme, [newItem]);
+          if (nextStates_Shift.has(nextSymbol.identifier)) nextStates_Shift.get(nextSymbol.identifier)!.push(newItem);
+          else nextStates_Shift.set(nextSymbol.identifier, [newItem]);
         }
       }
 
@@ -367,7 +380,7 @@ export class GeneratorResult {
   constructor(
     public readonly states: State[],
     public readonly ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[],
-    public readonly GotoTable: Map<string, number>[]
+    public readonly GotoTable: Map<string, number>[],
   ) {}
 
   toTable(): string {
@@ -382,6 +395,7 @@ export class GeneratorResult {
   toStates(): TableState[] {
     return this.ActionTable.map((actions, idx) => {
       const tableState = new TableState();
+
       for (const [k, { action, value }] of actions.entries())
         tableState.actions.set(k, { type: action === "reduce" ? "reduce" : "shift", value });
 
@@ -391,4 +405,8 @@ export class GeneratorResult {
       return tableState;
     });
   }
+
+  toJSObject() {
+    return this.toStates().map((tableState) => tableState.toJSObject());
+  }
 }

package/src/index.ts

@@ -1,5 +1,5 @@
 export { GeneratorResult, generateStates } from "./generator";
-export { Production, buildProductions, tryBuildProductions } from "./grammarParser";
+export { buildProductions, tryBuildProductions } from "./meta/selfhosted";
 export { Result } from "./utils/Result";
 export {
   ParserRecoveryFunction,
@@ -9,4 +9,5 @@ export {
   ParserResult,
   ParserError,
 } from "./parser";
-export { buildStates } from "./tableParser";
+export { buildStates } from "./table/selfhosted";
+export { Production } from "./meta/common";

package/src/meta/common.ts

@@ -0,0 +1,74 @@
+import { ColumnAndRow, Token } from "@scinorandex/slex";
+
+export type Production = {
+  lhs: GrammarToken;
+  identifier: string;
+  originalProductionIndex: number;
+  name: string | null;
+  rhs: { type: "terminal" | "variable"; token: GrammarToken; identifier: string; name: string | null }[];
+};
+
+const dehydateGrammarToken = ({ column, lexeme, line, type }: GrammarToken) => {
+  return { column, lexeme, line, type };
+};
+
+const hydrateGrammarToken = (
+  token: ReturnType<typeof dehydateGrammarToken>,
+): Token<GrammarTokenType, GrammarTokenMetadata> => {
+  return new Token(token.type, token.lexeme, new ColumnAndRow(token.line, token.column), {});
+};
+
+export const dehydateProduction = (opts: Production) => {
+  const { lhs, identifier, rhs, originalProductionIndex, name } = opts;
+  return {
+    lhs: dehydateGrammarToken(lhs),
+    identifier,
+    name,
+    originalProductionIndex,
+    rhs: rhs.map(({ type, token, identifier, name }) => ({
+      type,
+      token: dehydateGrammarToken(token),
+      identifier,
+      name,
+    })),
+  };
+};
+
+export const hydrateProduction = (production: ReturnType<typeof dehydateProduction>): Production => {
+  return {
+    lhs: hydrateGrammarToken(production.lhs),
+    identifier: production.identifier,
+    originalProductionIndex: production.originalProductionIndex,
+    name: production.name,
+    rhs: production.rhs.map(({ type, token, name }) => ({
+      type,
+      token: hydrateGrammarToken(token),
+      identifier: token.lexeme,
+      name,
+    })),
+  };
+};
+
+export type GrammarTokenMetadata = {};
+export enum GrammarTokenType {
+  IDENTIFIER,
+  L_ANGLE,
+  R_ANGLE,
+  L_BRACKET,
+  R_BRACKET,
+  L_PAREN,
+  R_PAREN,
+  PIPE,
+  QUESTION_MARK,
+  COLON,
+  SEMICOLON,
+  NUMBER,
+  EQUALS,
+  COMMA,
+  EOF,
+
+  PRODUCTION_NAME,
+  TOKEN_NAME,
+}
+
+export type GrammarToken = Token<GrammarTokenType, GrammarTokenMetadata>;

package/src/{grammarParser.ts → meta/handwritten.ts}

@@ -1,18 +1,6 @@
-import { Slex, Token } from "@scinorandex/slex";
-import { Result } from "./utils/Result";
-
-export type Production = { lhs: GrammarToken; rhs: { type: "terminal" | "variable"; token: GrammarToken }[] };
-
-export type GrammarTokenMetadata = {};
-export enum GrammarTokenType {
-  PRODUCTION_NAME,
-  TOKEN_NAME,
-  COLON,
-  SEMICOLON,
-  EOF,
-}
-
-export type GrammarToken = Token<GrammarTokenType, GrammarTokenMetadata>;
+import { Slex } from "@scinorandex/slex";
+import { Result } from "../utils/Result";
+import { GrammarToken, GrammarTokenMetadata, GrammarTokenType, Production } from "./common";
 
 export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMetadata>({
   EOF_TYPE: GrammarTokenType.EOF,
@@ -21,11 +9,11 @@ export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMeta
 
 grammarLexerGenerator.addRule(
   "lowercase",
-  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z"
+  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z",
 );
 grammarLexerGenerator.addRule(
   "uppercase",
-  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z"
+  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z",
 );
 grammarLexerGenerator.addRule("letter", "${lowercase} | ${uppercase}");
 grammarLexerGenerator.addRule("digit", "0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9");
@@ -53,19 +41,27 @@ export const tryBuildProductions = (lexer: LexerInterface | string): Result<Prod
     lexer.getNextToken();
   };
 
+  let productionIndex = 0;
+
   const buildProduction = (): Result<Production> => {
     const productionLHS = lexer.getNextToken();
     expect(GrammarTokenType.COLON);
 
-    const production: Production = { lhs: productionLHS, rhs: [] };
+    const production: Production = {
+      lhs: productionLHS,
+      identifier: productionLHS.lexeme,
+      rhs: [],
+      name: null,
+      originalProductionIndex: productionIndex++,
+    };
 
     while (lexer.peekNextToken().type != GrammarTokenType.SEMICOLON) {
       const productionRHS = lexer.getNextToken();
 
       if (productionRHS.type == GrammarTokenType.PRODUCTION_NAME)
-        production.rhs.push({ type: "variable", token: productionRHS });
+        production.rhs.push({ type: "variable", token: productionRHS, identifier: productionRHS.lexeme, name: null });
       else if (productionRHS.type == GrammarTokenType.TOKEN_NAME)
-        production.rhs.push({ type: "terminal", token: productionRHS });
+        production.rhs.push({ type: "terminal", token: productionRHS, identifier: productionRHS.lexeme, name: null });
       else
         return {
           success: false,
@@ -91,7 +87,7 @@ export const buildProductions = (lexer: LexerInterface | string): Production[] =
   const result = tryBuildProductions(lexer);
   if (result.success === false)
     throw new Error(
-      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`
+      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`,
     );
   return result.value;
 };

package/src/meta/selfhosted.ts

@@ -0,0 +1,272 @@
+import { RegexEngine, Slex } from "@scinorandex/slex";
+import { Result, Sparse } from "../index";
+import { selfhosted } from "./states";
+import { TableState } from "../parser";
+import { GrammarToken, GrammarTokenMetadata, GrammarTokenType, hydrateProduction, Production } from "./common";
+
+export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMetadata>({
+  EOF_TYPE: GrammarTokenType.EOF,
+  isHigherPrecedence: () => false,
+});
+
+grammarLexerGenerator.addRule(
+  "lowercase",
+  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z",
+);
+grammarLexerGenerator.addRule(
+  "uppercase",
+  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z",
+);
+grammarLexerGenerator.addRule("letter", "${lowercase} | ${uppercase}");
+grammarLexerGenerator.addRule("digit", "0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9");
+grammarLexerGenerator.addRule("alphanumeric", "${letter} | ${digit}");
+grammarLexerGenerator.addRule(
+  "identifier",
+  "(${letter} | $_) (${letter} | ${digit} | $_)*",
+  GrammarTokenType.IDENTIFIER,
+);
+grammarLexerGenerator.addRule("colon", "$:", GrammarTokenType.COLON);
+grammarLexerGenerator.addRule("semicolon", "$;", GrammarTokenType.SEMICOLON);
+grammarLexerGenerator.addRule("langle", "$<", GrammarTokenType.L_ANGLE);
+grammarLexerGenerator.addRule("rangle", "$>", GrammarTokenType.R_ANGLE);
+grammarLexerGenerator.addRule("lbracket", "$[", GrammarTokenType.L_BRACKET);
+grammarLexerGenerator.addRule("rbracket", "$]", GrammarTokenType.R_BRACKET);
+grammarLexerGenerator.addRule("lparen", "$(", GrammarTokenType.L_PAREN);
+grammarLexerGenerator.addRule("rparen", "$)", GrammarTokenType.R_PAREN);
+grammarLexerGenerator.addRule("pipe", "$|", GrammarTokenType.PIPE);
+grammarLexerGenerator.addRule("equals", "$=", GrammarTokenType.EQUALS);
+grammarLexerGenerator.addRule("comma", "$,", GrammarTokenType.COMMA);
+grammarLexerGenerator.addRule("question_mark", "$?", GrammarTokenType.QUESTION_MARK);
+grammarLexerGenerator.addRule("number", "(${digit})+", GrammarTokenType.NUMBER);
+
+export abstract class BaseNode {}
+
+export class ListNode<T> extends BaseNode {
+  private items: T[];
+
+  constructor(items: T | T[]) {
+    super();
+
+    if (Array.isArray(items)) this.items = items;
+    else this.items = [items];
+  }
+
+  add(item: T) {
+    this.items.push(item);
+    return this;
+  }
+
+  getItems() {
+    return this.items;
+  }
+
+  getItemsReversed() {
+    return this.items.toReversed();
+  }
+}
+
+class ProductionNode extends BaseNode {
+  constructor(
+    public readonly lhs: GrammarToken,
+    public readonly rhs: ListNode<TokenNode | GroupedTokenNode>,
+    public originalProductionIndex: number,
+    public readonly name?: string | undefined,
+  ) {
+    super();
+  }
+
+  toStruct(): Production {
+    if (this.originalProductionIndex === -1) throw new Error("ProductionNode has no originalProductionIndex");
+
+    const hasGrouped = this.rhs.getItemsReversed().some((node) => node instanceof GroupedTokenNode);
+    if (hasGrouped) throw new Error("Need to unroll before serializing");
+
+    return {
+      lhs: this.lhs,
+      identifier: `<${this.lhs.lexeme}>`,
+      name: this.name ?? null,
+      originalProductionIndex: this.originalProductionIndex,
+      rhs: this.rhs.getItemsReversed().map((node) => (node as TokenNode).toStruct()),
+    };
+  }
+
+  setOriginalProductionIndex(index: number) {
+    this.originalProductionIndex = index;
+  }
+}
+
+function expandProduction(production: ProductionNode): ProductionNode[] {
+  const rhs = production.rhs.getItemsReversed();
+
+  for (let i = 0; i < rhs.length; i++) {
+    const item = rhs[i];
+
+    const beforeItems = rhs.slice(0, i);
+    const afterItems = rhs.slice(i + 1);
+
+    if (item instanceof GroupedTokenNode) {
+      // need to create two new productions, one optional and the other not
+      // then apply unrolling on both
+
+      const withoutCurrentItem = new ProductionNode(
+        production.lhs,
+        new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
+        production.originalProductionIndex,
+        production.name,
+      );
+
+      const withCurrentItem = new ProductionNode(
+        production.lhs,
+        new ListNode<TokenNode | GroupedTokenNode>(
+          [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
+        ),
+        production.originalProductionIndex,
+        production.name,
+      );
+
+      const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
+      return newProductions.flatMap((prod) => expandProduction(prod));
+    } else if (item instanceof TokenNode) {
+      if (item.variables.getItems().length > 1) {
+        // this token node has many variants and we should unroll it
+        const variables = item.variables.getItemsReversed();
+
+        const newProductions = variables.map((variable) => {
+          return new ProductionNode(
+            production.lhs,
+            new ListNode<TokenNode | GroupedTokenNode>([
+              ...beforeItems,
+              new TokenNode("variable", new ListNode<GrammarToken>([variable]), item.name),
+            ]),
+            production.originalProductionIndex,
+            production.name,
+          );
+        });
+
+        return newProductions.flatMap((prod) => expandProduction(prod));
+      }
+    } else {
+      throw new Error("shoudn't reach here, ProductionNode RHS is neither GroupedTokenNode nor TokenNode");
+    }
+  }
+
+  // if it reached here then its normal
+  return [production];
+}
+
+class TokenNode extends BaseNode {
+  constructor(
+    public readonly type: "terminal" | "variable",
+    public readonly variables: ListNode<GrammarToken>,
+    public readonly name: string | undefined,
+  ) {
+    super();
+  }
+
+  toStruct() {
+    const items = this.variables.getItems();
+    if (items.length > 1) throw new Error("Not yet unrolled");
+
+    const lexeme = items[0].lexeme;
+    const identifier = this.type === "variable" ? `<${lexeme}>` : `[${lexeme}]`;
+    const name = this.name ?? null;
+
+    return { type: this.type, token: items[0], identifier, name };
+  }
+}
+
+class GroupedTokenNode extends BaseNode {
+  constructor(public readonly inside: ListNode<TokenNode>) {
+    super();
+  }
+}
+
+class ProgramNode extends BaseNode {
+  constructor(public readonly productions: ListNode<ProductionNode>) {
+    super();
+  }
+
+  unrollProductions() {
+    const productions = this.productions.getItemsReversed();
+    for (let i = 0; i < productions.length; i++) productions[i].setOriginalProductionIndex(i);
+    return productions.flatMap((node) => expandProduction(node));
+  }
+
+  getProductions() {
+    return this.unrollProductions().map((node) => node.toStruct());
+  }
+}
+
+export function getSelfHostedParserGenerator(e: typeof selfhosted = selfhosted) {
+  return new Sparse<GrammarTokenType, GrammarTokenMetadata, BaseNode>({
+    productions: e.productions.map((production) => hydrateProduction(production as any)),
+    states: e.states.map((state) => TableState.fromJSObject(state as any)),
+    toStringifiedTokenType: (type: GrammarTokenType) => GrammarTokenType[type],
+  });
+}
+
+type Reducer = (bag: any) => BaseNode;
+const reducers: { [key: string]: Reducer } = {
+  program: (bag: { productions?: ListNode<ProductionNode> }) =>
+    new ProgramNode(bag.productions ?? new ListNode<ProductionNode>([])),
+
+  productions: (bag: { production: ProductionNode; rest?: ListNode<ProductionNode> }) => {
+    if (bag.rest == null) return new ListNode<ProductionNode>([bag.production]);
+    else return bag.rest.add(bag.production);
+  },
+
+  production: (bag: {
+    production_name: GrammarToken;
+    tokens: ListNode<TokenNode | GroupedTokenNode>;
+    uuid?: GrammarToken;
+  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.uuid?.lexeme),
+
+  tokens: (bag: { token: TokenNode | GroupedTokenNode; rest?: ListNode<TokenNode | GroupedTokenNode> }) => {
+    if (bag.rest == null) return new ListNode<TokenNode | GroupedTokenNode>([bag.token]);
+    else return bag.rest.add(bag.token);
+  },
+
+  token: (bag: { token: TokenNode }) => bag.token,
+  grouped_token: (bag: { tokens: ListNode<TokenNode> }) => new GroupedTokenNode(bag.tokens),
+
+  variable: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
+    new TokenNode("variable", bag.inside, bag.token_name?.lexeme),
+
+  terminal: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
+    new TokenNode("terminal", bag.inside, bag.token_name?.lexeme),
+
+  inside: (bag: { identifier: GrammarToken; rest?: ListNode<GrammarToken> }) => {
+    if (bag.rest == null) return new ListNode<GrammarToken>([bag.identifier]);
+    else return bag.rest.add(bag.identifier);
+  },
+};
+
+export const tryBuildProductions = (lexer: LexerInterface | string): Result<Production[]> => {
+  if (typeof lexer === "string") lexer = grammarLexerGenerator.generate(lexer, () => ({}));
+
+  const parserGenerator = getSelfHostedParserGenerator();
+  const parser = parserGenerator.generate(lexer as RegexEngine<GrammarTokenType, GrammarTokenMetadata>, {
+    reducer: ({ bag, name }) => {
+      return reducers[name ?? ""](bag);
+    },
+  });
+
+  const parsingResult = parser.parse().result as ProgramNode;
+  const productions = parsingResult.getProductions();
+  return { success: true, value: productions };
+};
+
+interface LexerInterface {
+  peekNextToken(): GrammarToken;
+  getNextToken(): GrammarToken;
+  hasNextToken(): boolean;
+}
+
+export const buildProductions = (lexer: LexerInterface | string): Production[] => {
+  const result = tryBuildProductions(lexer);
+  if (result.success === false)
+    throw new Error(
+      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`,
+    );
+  return result.value;
+};