@scinorandex/sparse 0.0.7 → 0.1.1

package/src/generator.ts

@@ -121,10 +121,14 @@ function computeFollowSets(
   return { success: true, value: { firstSets, followSets: ret } };
 }
 
-type Item = Production & { dot: number; lookahead: string[] };
+const KEY_SEP = "\x00";
+const FIELD_SEP = "\x01";
+
+type Item = Production & { dot: number; lookahead: string[]; prodIndex: number; key: string };
 type State = { itemSet: Item[]; kernel: Item; count: number };
 
-export const generateStates = (productions: Production[]): Result<GeneratorResult> => {
+export type GeneratorOptions = { mode?: "lr1" | "lalr1" };
+export const generateStates = (productions: Production[], options: GeneratorOptions = {}): Result<GeneratorResult> => {
   const followSetsResult = computeFollowSets(productions);
   if (followSetsResult.success == false) return followSetsResult;
 
@@ -161,15 +165,45 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     }
   };
 
+  const productionIndicesByLhs = new Map<string, number[]>();
+  for (let i = 0; i < productions.length; i++) {
+    const indexList = productionIndicesByLhs.get(productions[i].identifier);
+    if (indexList == undefined) productionIndicesByLhs.set(productions[i].identifier, [i]);
+    else indexList.push(i);
+  }
+
+  const rhsEntrySignature = (rhsToken: Production["rhs"][number]) =>
+    `${rhsToken.type}${FIELD_SEP}${rhsToken.identifier}${FIELD_SEP}${rhsToken.name ?? ""}${FIELD_SEP}${rhsToken.token.line}${FIELD_SEP}${rhsToken.token.column}`;
+
+  const reduceValue: number[] = [];
+  {
+    const lowestIndexBySignature = new Map<string, number>();
+    for (let i = 0; i < productions.length; i++) {
+      const production = productions[i];
+      const signature = `${production.identifier}${FIELD_SEP}${production.rhs.map(rhsEntrySignature).join(FIELD_SEP)}`;
+      const lowest = lowestIndexBySignature.get(signature);
+      if (lowest == undefined) lowestIndexBySignature.set(signature, i);
+      reduceValue[i] = lowestIndexBySignature.get(signature)!;
+    }
+  }
+
+  const itemKey = (prodIndex: number, dot: number, lookahead: string[]) =>
+    `${prodIndex}${KEY_SEP}${dot}${KEY_SEP}${lookahead.join(FIELD_SEP)}`;
+
+  const itemSetSignature = (itemSet: Item[]) => itemSet.map((item) => item.key).join(FIELD_SEP);
+
   // This fnuction creates the initial item set based on the
   // production provided. It creating the initial item and expands it
-  function generateInitialItemSet(production: Production) {
+  function generateInitialItemSet(production: Production, prodIndex: number) {
+    const lookahead = [EOF_STRING];
     const initialItem = {
       lhs: production.lhs,
       identifier: production.identifier,
       rhs: production.rhs,
       dot: 0,
-      lookahead: [EOF_STRING],
+      lookahead,
+      prodIndex,
+      key: itemKey(prodIndex, 0, lookahead),
     } as Item;
 
     return expandItemSet([initialItem]);
@@ -178,20 +212,19 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   // This function expands the item sets provided to it
   function expandItemSet(items: Item[]): Result<{ itemSet: Item[]; kernel: Item }> {
     const itemSet: Item[] = [...items];
+    const knownItems = new Set(items.map((item) => item.key));
 
-    // Create a queue of unprocessed items and keep shifting until there are no more items left
+    // Create a queue of unprocessed items and keep going until there are no more items left
     const unprocesssedItems: Item[] = [...items];
-    while (unprocesssedItems.length > 0) {
-      const currentItem = unprocesssedItems.shift()!;
+    let queuePosition = 0;
+    while (queuePosition < unprocesssedItems.length) {
+      const currentItem = unprocesssedItems[queuePosition++];
 
       // Check if the symbol after the dot is a non terminal
       const after = currentItem.rhs[currentItem.dot];
       if (after == null) continue; // TODO: check if this should be here
 
       if (after.type === "variable") {
-        // Find prodctions whose left hand side is the symbol after the dot
-        const newProductions = productions.filter((p) => p.identifier === after.identifier);
-
         // Compute the lookahead for the new productions to be added to the item set
         const rest = currentItem.rhs.slice(currentItem.dot + 1);
 
@@ -199,19 +232,22 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         if (lookaheadResult.success === false) return lookaheadResult;
         const lookahead = lookaheadResult.value;
 
-        for (const newProduction of newProductions) {
-          // Create the new item and check if it already exists in the item set
-          // If it doesn't exist, add it to the item set and the queue of unprocessed items
-          const newItem = {
-            lhs: newProduction.lhs,
-            identifier: newProduction.identifier,
-            rhs: newProduction.rhs,
-            dot: 0,
-            lookahead,
-          } as Item;
-          const encoding = JSON.stringify(newItem);
-
-          if (itemSet.some((i) => JSON.stringify(i) === encoding) == false) {
+        const productionIndices = productionIndicesByLhs.get(after.identifier) ?? [];
+        for (const prodIndex of productionIndices) {
+          const newProduction = productions[prodIndex];
+          const key = itemKey(prodIndex, 0, lookahead);
+
+          if (knownItems.has(key) == false) {
+            knownItems.add(key);
+            const newItem = {
+              lhs: newProduction.lhs,
+              identifier: newProduction.identifier,
+              rhs: newProduction.rhs,
+              dot: 0,
+              lookahead,
+              prodIndex,
+              key,
+            } as Item;
             unprocesssedItems.push(newItem);
             itemSet.push(newItem);
           }
@@ -223,7 +259,7 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     return { success: true, value: { itemSet, kernel: items[0] } };
   }
 
-  const initialItemSetResult = generateInitialItemSet(productions[0]);
+  const initialItemSetResult = generateInitialItemSet(productions[0], 0);
   if (initialItemSetResult.success === false) return initialItemSetResult;
   const initialItemSet = initialItemSetResult.value;
 
@@ -242,9 +278,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     // array of maps whose keys is a terminal is a value of either to shift or reduce
     const ActionTable = [] as Map<string, { action: "shift" | "reduce"; value: number }>[];
 
+    // map from an item set signature to the index of the state holding it
+    const stateIndexBySignature = new Map<string, number>();
+    stateIndexBySignature.set(itemSetSignature(initialState.itemSet), 0);
+
     // While there is an unprocessed state, visit it
-    while (unprocessedStates.length > 0) {
-      const currentState = unprocessedStates.shift()!;
+    let statePosition = 0;
+    while (statePosition < unprocessedStates.length) {
+      const currentState = unprocessedStates[statePosition++];
       const nextStates_Goto = new Map<string, Item[]>();
       const nextStates_Shift = new Map<string, Item[]>();
 
@@ -256,25 +297,21 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
 
         // no next symbol is available, therefore we need to create a reduction
         if (nextSymbol == null) {
-          // Find the prodction that this item reduces to using the symbols on its right hand side
-          // TODO: check if the find can fail, honestly this might be invariant
-          const productionToReduceTo = productions
-            .map((p, i) => [p, i] as const)
-            .find(
-              ([p, _]) => JSON.stringify(p.rhs) === JSON.stringify(item.rhs) && p.identifier === item.identifier,
-            )![1];
-
           // for each lookahead in the item, create a new entry in the action table to reduce to the production found
           for (const lookahead of item.lookahead) {
             if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
-            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: productionToReduceTo });
+            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: reduceValue[item.prodIndex] });
           }
 
           continue calculateNextItem;
         }
 
         // Create the next item by shifting the dot by 1 place
-        const newItem: Item = { ...item, dot: item.dot + 1 };
+        const newItem: Item = {
+          ...item,
+          dot: item.dot + 1,
+          key: itemKey(item.prodIndex, item.dot + 1, item.lookahead),
+        };
 
         // Determine if we're going to create a GOTO or SHIFT based on the type of the symbol after the dot
         if (nextSymbol.type === "variable") {
@@ -300,19 +337,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         if (GotoTable[currentState.count] === undefined) GotoTable[currentState.count] = new Map();
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -320,10 +349,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        GotoTable[currentState.count]!.set(gotoTransition, existingState.count);
+        GotoTable[currentState.count]!.set(gotoTransition, existingStateIndex);
       }
 
       // For each SHIFT transition from the current state, expand  the item set that it points to
@@ -336,22 +366,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         // If there is no entry in the ActionTable for the current state, create one
         if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
 
         // If no such state exists, create a new state and add it to the list of states
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -359,10 +381,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingState.count });
+        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingStateIndex });
       }
     }
 
@@ -373,9 +396,115 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   const StatesResult = generateStates(initialItemSet);
   if (StatesResult.success === false) return StatesResult;
   const { states, ActionTable, GotoTable } = StatesResult.value;
+
+  if (options.mode === "lalr1") {
+    const merged = mergeStatesForLALR(states, ActionTable, GotoTable);
+    if (merged.success === false) return merged;
+    return {
+      success: true,
+      value: new GeneratorResult(merged.value.states, merged.value.ActionTable, merged.value.GotoTable),
+    };
+  }
+
   return { success: true, value: new GeneratorResult(states, ActionTable, GotoTable) };
 };
 
+// Merges the LR(1) states into LALR(1) states by grouping states that have the same
+// "same-core" item set (items compared without their lookahead sets) and unioning
+// their item sets, action tables and goto tables. Returns an error if merging any
+// group produces a shift/reduce, reduce/reduce, shift or goto conflict, which
+// means the grammar is not LALR(1).
+function mergeStatesForLALR(
+  states: State[],
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[],
+  GotoTable: Map<string, number>[],
+): Result<{
+  states: State[];
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[];
+  GotoTable: Map<string, number>[];
+}> {
+  type Action = { action: "shift" | "reduce"; value: number };
+
+  const itemSkeleton = (item: Item) => `${item.prodIndex}${KEY_SEP}${item.dot}`;
+  const stateSkeleton = (state: State) => state.itemSet.map(itemSkeleton).sort().join(FIELD_SEP);
+
+  // Group states by same-core signature, assigning merged indices in first-occurrence order
+  const groupIndices: number[][] = [];
+  const groupBySkeleton = new Map<string, number>();
+  const oldToNew = new Array<number>(states.length);
+  for (let i = 0; i < states.length; i++) {
+    const skeleton = stateSkeleton(states[i]);
+    let group = groupBySkeleton.get(skeleton);
+    if (group == undefined) {
+      group = groupIndices.length;
+      groupBySkeleton.set(skeleton, group);
+      groupIndices.push([]);
+    }
+    groupIndices[group].push(i);
+    oldToNew[i] = group;
+  }
+
+  const describeAction = (a: Action) =>
+    a.action === "reduce" ? `reduce by production ${a.value}` : `shift to state ${a.value}`;
+  const conflictToken = (members: number[]) => states[members[0]].kernel.lhs;
+  const conflict = (g: number, members: number[], reason: string): Result<never> => ({
+    success: false,
+    reason: `LALR(1) conflict in state ${g} on ${reason} between LR(1) states ${members.join(", ")}: the grammar is not LALR(1)`,
+    token: conflictToken(members),
+  });
+
+  const newStates: State[] = [];
+  const newActionTable: Map<string, Action>[] = [];
+  const newGotoTable: Map<string, number>[] = [];
+
+  for (let g = 0; g < groupIndices.length; g++) {
+    const members = groupIndices[g];
+    const actions = new Map<string, Action>();
+    const gotos = new Map<string, number>();
+
+    for (const member of members) {
+      for (const [key, entry] of ActionTable[member].entries()) {
+        const value: Action = {
+          action: entry.action,
+          value: entry.action === "shift" ? oldToNew[entry.value] : entry.value,
+        };
+        const existing = actions.get(key);
+        if (existing == undefined) actions.set(key, value);
+        else if (existing.action !== value.action || existing.value !== value.value)
+          return conflict(g, members, `"${key}": ${describeAction(existing)} conflicts with ${describeAction(value)}`);
+      }
+
+      if (GotoTable[member] !== undefined)
+        for (const [key, target] of GotoTable[member]!.entries()) {
+          const value = oldToNew[target];
+          const existing = gotos.get(key);
+          if (existing == undefined) gotos.set(key, value);
+          else if (existing !== value)
+            return conflict(g, members, `"${key}": goto to state ${existing} conflicts with goto to state ${value}`);
+        }
+    }
+
+    // Union the item sets of all members, merging the lookahead sets of
+    // items that differ only in their lookahead
+    const itemsBySkeleton = new Map<string, Item>();
+    for (const member of members)
+      for (const item of states[member].itemSet) {
+        const skeleton = itemSkeleton(item);
+        const existing = itemsBySkeleton.get(skeleton);
+        if (existing == undefined) itemsBySkeleton.set(skeleton, { ...item, lookahead: [...item.lookahead] });
+        else
+          for (const lookahead of item.lookahead)
+            if (!existing.lookahead.includes(lookahead)) existing.lookahead.push(lookahead);
+      }
+
+    newStates.push({ itemSet: [...itemsBySkeleton.values()], kernel: states[members[0]].kernel, count: g });
+    newActionTable.push(actions);
+    newGotoTable.push(gotos);
+  }
+
+  return { success: true, value: { states: newStates, ActionTable: newActionTable, GotoTable: newGotoTable } };
+}
+
 export class GeneratorResult {
   constructor(
     public readonly states: State[],

package/src/index.ts

@@ -1,4 +1,5 @@
 export { GeneratorResult, generateStates } from "./generator";
+export type { GeneratorOptions } from "./generator";
 export { buildProductions, tryBuildProductions } from "./meta/selfhosted";
 export { Result } from "./utils/Result";
 export {

package/src/meta/common.ts

@@ -60,6 +60,8 @@ export enum GrammarTokenType {
   R_PAREN,
   PIPE,
   QUESTION_MARK,
+  STAR,
+  PLUS,
   COLON,
   SEMICOLON,
   NUMBER,

package/src/meta/selfhosted.ts

@@ -38,6 +38,8 @@ grammarLexerGenerator.addRule("pipe", "$|", GrammarTokenType.PIPE);
 grammarLexerGenerator.addRule("equals", "$=", GrammarTokenType.EQUALS);
 grammarLexerGenerator.addRule("comma", "$,", GrammarTokenType.COMMA);
 grammarLexerGenerator.addRule("question_mark", "$?", GrammarTokenType.QUESTION_MARK);
+grammarLexerGenerator.addRule("star", "$*", GrammarTokenType.STAR);
+grammarLexerGenerator.addRule("plus", "$+", GrammarTokenType.PLUS);
 grammarLexerGenerator.addRule("number", "(${digit})+", GrammarTokenType.NUMBER);
 
 grammarLexerGenerator.addRule("single_line_comment", "$/$/(($\n)!)*", GrammarTokenType.SINGLE_LINE_COMMENT);
@@ -74,11 +76,16 @@ class ProductionNode extends BaseNode {
     public readonly lhs: GrammarToken,
     public readonly rhs: ListNode<TokenNode | GroupedTokenNode>,
     public originalProductionIndex: number,
+    public readonly identifier: string,
     public readonly name?: string | undefined,
   ) {
     super();
   }
 
+  fork(rhs: ListNode<TokenNode | GroupedTokenNode>) {
+    return new ProductionNode(this.lhs, rhs, this.originalProductionIndex, this.identifier, this.name);
+  }
+
   toStruct(): Production {
     if (this.originalProductionIndex === -1) throw new Error("ProductionNode has no originalProductionIndex");
 
@@ -87,7 +94,7 @@ class ProductionNode extends BaseNode {
 
     return {
       lhs: this.lhs,
-      identifier: `<${this.lhs.lexeme}>`,
+      identifier: `<${this.identifier}>`,
       name: this.name ?? null,
       originalProductionIndex: this.originalProductionIndex,
       rhs: this.rhs.getItemsReversed().map((node) => (node as TokenNode).toStruct()),
@@ -99,7 +106,21 @@ class ProductionNode extends BaseNode {
   }
 }
 
-function expandProduction(production: ProductionNode): ProductionNode[] {
+class ExpanderContext {
+  current = 0;
+
+  constructor(public currentProductionIndex: number) {}
+
+  getNextProductionIndex() {
+    return this.currentProductionIndex++;
+  }
+
+  getAutogeneratedId() {
+    return `autogen-${this.current++}`;
+  }
+}
+
+function expandProduction(production: ProductionNode, ctx: ExpanderContext): ProductionNode[] {
   const rhs = production.rhs.getItemsReversed();
 
   for (let i = 0; i < rhs.length; i++) {
@@ -109,35 +130,53 @@ function expandProduction(production: ProductionNode): ProductionNode[] {
     const afterItems = rhs.slice(i + 1);
 
     if (item instanceof GroupedTokenNode) {
-      // need to create two new productions, one optional and the other not
-      // then apply unrolling on both
-
-      const withoutCurrentItem = new ProductionNode(
-        production.lhs,
-        new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
-        production.originalProductionIndex,
-        production.name,
-      );
-
-      const withCurrentItem = new ProductionNode(
-        production.lhs,
-        new ListNode<TokenNode | GroupedTokenNode>(
-          [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
-        ),
-        production.originalProductionIndex,
-        production.name,
-      );
-
-      const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
-      return newProductions.flatMap((prod) => expandProduction(prod));
+      // The unrolling rules here depend on the modifier in the GroupedTokenNode
+
+      if (item.modifier === GrammarTokenType.QUESTION_MARK) {
+        const withoutCurrentItem = production.fork(
+          new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
+        );
+
+        const withCurrentItem = production.fork(
+          new ListNode<TokenNode | GroupedTokenNode>(
+            [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
+          ),
+        );
+
+        const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
+        return newProductions.flatMap((prod) => expandProduction(prod, ctx));
+      } else if (item.modifier === GrammarTokenType.STAR || item.modifier === GrammarTokenType.PLUS) {
+        // need to handle kleene-star and kleene-plus, which create entirely unique new productions
+        const inside = item.inside.getItemsReversed();
+        const newProductionIdentifier = ctx.getAutogeneratedId();
+
+        // create a dummy production that contains the contents of inside
+        const raw = new StubTokenNode("variable", new ListNode([production.lhs]), "rest", newProductionIdentifier);
+        const pump = new GroupedTokenNode(new ListNode([raw]), GrammarTokenType.QUESTION_MARK);
+
+        const stub = new ProductionNode(
+          production.lhs,
+          new ListNode<TokenNode | GroupedTokenNode>([...inside, pump].toReversed()),
+          ctx.getNextProductionIndex(),
+          newProductionIdentifier,
+          "autogenerated-kleene", // the user needs to implement a reducer named "autogenerated-kleene"
+        );
+
+        const currentReplacement = production.fork(
+          new ListNode(
+            [...beforeItems, item.modifier === GrammarTokenType.STAR ? pump : raw, ...afterItems].toReversed(),
+          ),
+        );
+
+        return [currentReplacement, stub].flatMap((prod) => expandProduction(prod, ctx));
+      } else throw new Error(`Unknown modifier ${GrammarTokenType[item.modifier]}`);
     } else if (item instanceof TokenNode) {
       if (item.variables.getItems().length > 1) {
         // this token node has many variants and we should unroll it
         const variables = item.variables.getItemsReversed();
 
         const newProductions = variables.map((variable) => {
-          return new ProductionNode(
-            production.lhs,
+          return production.fork(
             new ListNode<TokenNode | GroupedTokenNode>(
               [
                 ...beforeItems,
@@ -145,12 +184,10 @@ function expandProduction(production: ProductionNode): ProductionNode[] {
                 ...afterItems,
               ].toReversed(),
             ),
-            production.originalProductionIndex,
-            production.name,
           );
         });
 
-        return newProductions.flatMap((prod) => expandProduction(prod));
+        return newProductions.flatMap((prod) => expandProduction(prod, ctx));
       }
     } else {
       throw new Error("shoudn't reach here, ProductionNode RHS is neither GroupedTokenNode nor TokenNode");
@@ -170,11 +207,17 @@ class TokenNode extends BaseNode {
     super();
   }
 
+  public getIdentifier() {
+    const items = this.variables.getItems();
+    if (items.length > 1) throw new Error("Not yet unrolled");
+    return items[0].lexeme;
+  }
+
   toStruct() {
     const items = this.variables.getItems();
     if (items.length > 1) throw new Error("Not yet unrolled");
 
-    const lexeme = items[0].lexeme;
+    const lexeme = this.getIdentifier();
     const identifier = this.type === "variable" ? `<${lexeme}>` : `[${lexeme}]`;
     const name = this.name ?? null;
 
@@ -182,8 +225,26 @@ class TokenNode extends BaseNode {
   }
 }
 
+class StubTokenNode extends TokenNode {
+  constructor(
+    public readonly type: "terminal" | "variable",
+    public readonly variables: ListNode<GrammarToken>,
+    public readonly name: string | undefined,
+    public readonly identifier: string,
+  ) {
+    super(type, variables, name);
+  }
+
+  public getIdentifier() {
+    return this.identifier;
+  }
+}
+
 class GroupedTokenNode extends BaseNode {
-  constructor(public readonly inside: ListNode<TokenNode>) {
+  constructor(
+    public readonly inside: ListNode<TokenNode>,
+    public readonly modifier: GrammarTokenType,
+  ) {
     super();
   }
 }
@@ -195,8 +256,10 @@ class ProgramNode extends BaseNode {
 
   unrollProductions() {
     const productions = this.productions.getItemsReversed();
+    const expanderContext = new ExpanderContext(this.productions.getItems().length);
+
     for (let i = 0; i < productions.length; i++) productions[i].setOriginalProductionIndex(i);
-    return productions.flatMap((node) => expandProduction(node));
+    return productions.flatMap((node) => expandProduction(node, expanderContext));
   }
 
   getProductions() {
@@ -226,7 +289,7 @@ const reducers: { [key: string]: Reducer } = {
     production_name: GrammarToken;
     tokens: ListNode<TokenNode | GroupedTokenNode>;
     uuid?: GrammarToken;
-  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.uuid?.lexeme),
+  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.production_name.lexeme, bag.uuid?.lexeme),
 
   tokens: (bag: { token: TokenNode | GroupedTokenNode; rest?: ListNode<TokenNode | GroupedTokenNode> }) => {
     if (bag.rest == null) return new ListNode<TokenNode | GroupedTokenNode>([bag.token]);
@@ -234,7 +297,8 @@ const reducers: { [key: string]: Reducer } = {
   },
 
   token: (bag: { token: TokenNode }) => bag.token,
-  grouped_token: (bag: { tokens: ListNode<TokenNode> }) => new GroupedTokenNode(bag.tokens),
+  grouped_token: (bag: { tokens: ListNode<TokenNode>; modifier: GrammarToken }) =>
+    new GroupedTokenNode(bag.tokens, bag.modifier.type),
 
   variable: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
     new TokenNode("variable", bag.inside, bag.token_name?.lexeme),