@scinorandex/sparse 0.0.8 → 0.1.1

package/package.json

@@ -1,11 +1,15 @@
 {
   "name": "@scinorandex/sparse",
-  "version": "0.0.8",
+  "version": "0.1.1",
   "description": "Yet another parser generator",
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
-    "watch": "tsc -w"
+    "watch": "tsc -w",
+    "tc": "tsc --noEmit -p tsconfig.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "bench": "vitest bench"
   },
   "bin": {
     "sparse": "dist/cli.js"
@@ -27,7 +31,9 @@
   "devDependencies": {
     "@types/minimist": "^1.2.5",
     "@types/node": "^26.0.0",
-    "@scinorandex/slex": "^0.0.4"
+    "@scinorandex/slex": "^0.0.4",
+    "typescript": "^5.6.0",
+    "vitest": "^3.0.0"
   },
   "peerDependencies": {
     "@scinorandex/slex": "*"

package/src/cli.ts

@@ -18,7 +18,7 @@ function checkFileExists(filepath: string) {
 
 async function main() {
   if (typeof args.input !== "string" || typeof args.output !== "string") {
-    console.log("Usage: npx @scinorandex/sparse --input=<input> --output=<output>");
+    console.log("Usage: npx @scinorandex/sparse --input=<input> --output=<output> [--lalr]");
     return;
   }
 
@@ -36,7 +36,7 @@ async function main() {
     return;
   }
 
-  const generatorResult = generateStates(productionsResult.value);
+  const generatorResult = generateStates(productionsResult.value, { mode: args.lalr === true ? "lalr1" : "lr1" });
   if (generatorResult.success === false) {
     console.log(generatorResult.reason);
     console.log(buildErrorWindow(grammar, generatorResult.token));

package/src/generator.ts

@@ -121,10 +121,14 @@ function computeFollowSets(
   return { success: true, value: { firstSets, followSets: ret } };
 }
 
-type Item = Production & { dot: number; lookahead: string[] };
+const KEY_SEP = "\x00";
+const FIELD_SEP = "\x01";
+
+type Item = Production & { dot: number; lookahead: string[]; prodIndex: number; key: string };
 type State = { itemSet: Item[]; kernel: Item; count: number };
 
-export const generateStates = (productions: Production[]): Result<GeneratorResult> => {
+export type GeneratorOptions = { mode?: "lr1" | "lalr1" };
+export const generateStates = (productions: Production[], options: GeneratorOptions = {}): Result<GeneratorResult> => {
   const followSetsResult = computeFollowSets(productions);
   if (followSetsResult.success == false) return followSetsResult;
 
@@ -161,15 +165,45 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     }
   };
 
+  const productionIndicesByLhs = new Map<string, number[]>();
+  for (let i = 0; i < productions.length; i++) {
+    const indexList = productionIndicesByLhs.get(productions[i].identifier);
+    if (indexList == undefined) productionIndicesByLhs.set(productions[i].identifier, [i]);
+    else indexList.push(i);
+  }
+
+  const rhsEntrySignature = (rhsToken: Production["rhs"][number]) =>
+    `${rhsToken.type}${FIELD_SEP}${rhsToken.identifier}${FIELD_SEP}${rhsToken.name ?? ""}${FIELD_SEP}${rhsToken.token.line}${FIELD_SEP}${rhsToken.token.column}`;
+
+  const reduceValue: number[] = [];
+  {
+    const lowestIndexBySignature = new Map<string, number>();
+    for (let i = 0; i < productions.length; i++) {
+      const production = productions[i];
+      const signature = `${production.identifier}${FIELD_SEP}${production.rhs.map(rhsEntrySignature).join(FIELD_SEP)}`;
+      const lowest = lowestIndexBySignature.get(signature);
+      if (lowest == undefined) lowestIndexBySignature.set(signature, i);
+      reduceValue[i] = lowestIndexBySignature.get(signature)!;
+    }
+  }
+
+  const itemKey = (prodIndex: number, dot: number, lookahead: string[]) =>
+    `${prodIndex}${KEY_SEP}${dot}${KEY_SEP}${lookahead.join(FIELD_SEP)}`;
+
+  const itemSetSignature = (itemSet: Item[]) => itemSet.map((item) => item.key).join(FIELD_SEP);
+
   // This fnuction creates the initial item set based on the
   // production provided. It creating the initial item and expands it
-  function generateInitialItemSet(production: Production) {
+  function generateInitialItemSet(production: Production, prodIndex: number) {
+    const lookahead = [EOF_STRING];
     const initialItem = {
       lhs: production.lhs,
       identifier: production.identifier,
       rhs: production.rhs,
       dot: 0,
-      lookahead: [EOF_STRING],
+      lookahead,
+      prodIndex,
+      key: itemKey(prodIndex, 0, lookahead),
     } as Item;
 
     return expandItemSet([initialItem]);
@@ -178,20 +212,19 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   // This function expands the item sets provided to it
   function expandItemSet(items: Item[]): Result<{ itemSet: Item[]; kernel: Item }> {
     const itemSet: Item[] = [...items];
+    const knownItems = new Set(items.map((item) => item.key));
 
-    // Create a queue of unprocessed items and keep shifting until there are no more items left
+    // Create a queue of unprocessed items and keep going until there are no more items left
     const unprocesssedItems: Item[] = [...items];
-    while (unprocesssedItems.length > 0) {
-      const currentItem = unprocesssedItems.shift()!;
+    let queuePosition = 0;
+    while (queuePosition < unprocesssedItems.length) {
+      const currentItem = unprocesssedItems[queuePosition++];
 
       // Check if the symbol after the dot is a non terminal
       const after = currentItem.rhs[currentItem.dot];
       if (after == null) continue; // TODO: check if this should be here
 
       if (after.type === "variable") {
-        // Find prodctions whose left hand side is the symbol after the dot
-        const newProductions = productions.filter((p) => p.identifier === after.identifier);
-
         // Compute the lookahead for the new productions to be added to the item set
         const rest = currentItem.rhs.slice(currentItem.dot + 1);
 
@@ -199,19 +232,22 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         if (lookaheadResult.success === false) return lookaheadResult;
         const lookahead = lookaheadResult.value;
 
-        for (const newProduction of newProductions) {
-          // Create the new item and check if it already exists in the item set
-          // If it doesn't exist, add it to the item set and the queue of unprocessed items
-          const newItem = {
-            lhs: newProduction.lhs,
-            identifier: newProduction.identifier,
-            rhs: newProduction.rhs,
-            dot: 0,
-            lookahead,
-          } as Item;
-          const encoding = JSON.stringify(newItem);
-
-          if (itemSet.some((i) => JSON.stringify(i) === encoding) == false) {
+        const productionIndices = productionIndicesByLhs.get(after.identifier) ?? [];
+        for (const prodIndex of productionIndices) {
+          const newProduction = productions[prodIndex];
+          const key = itemKey(prodIndex, 0, lookahead);
+
+          if (knownItems.has(key) == false) {
+            knownItems.add(key);
+            const newItem = {
+              lhs: newProduction.lhs,
+              identifier: newProduction.identifier,
+              rhs: newProduction.rhs,
+              dot: 0,
+              lookahead,
+              prodIndex,
+              key,
+            } as Item;
             unprocesssedItems.push(newItem);
             itemSet.push(newItem);
           }
@@ -223,7 +259,7 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     return { success: true, value: { itemSet, kernel: items[0] } };
   }
 
-  const initialItemSetResult = generateInitialItemSet(productions[0]);
+  const initialItemSetResult = generateInitialItemSet(productions[0], 0);
   if (initialItemSetResult.success === false) return initialItemSetResult;
   const initialItemSet = initialItemSetResult.value;
 
@@ -242,9 +278,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     // array of maps whose keys is a terminal is a value of either to shift or reduce
     const ActionTable = [] as Map<string, { action: "shift" | "reduce"; value: number }>[];
 
+    // map from an item set signature to the index of the state holding it
+    const stateIndexBySignature = new Map<string, number>();
+    stateIndexBySignature.set(itemSetSignature(initialState.itemSet), 0);
+
     // While there is an unprocessed state, visit it
-    while (unprocessedStates.length > 0) {
-      const currentState = unprocessedStates.shift()!;
+    let statePosition = 0;
+    while (statePosition < unprocessedStates.length) {
+      const currentState = unprocessedStates[statePosition++];
       const nextStates_Goto = new Map<string, Item[]>();
       const nextStates_Shift = new Map<string, Item[]>();
 
@@ -256,25 +297,21 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
 
         // no next symbol is available, therefore we need to create a reduction
         if (nextSymbol == null) {
-          // Find the prodction that this item reduces to using the symbols on its right hand side
-          // TODO: check if the find can fail, honestly this might be invariant
-          const productionToReduceTo = productions
-            .map((p, i) => [p, i] as const)
-            .find(
-              ([p, _]) => JSON.stringify(p.rhs) === JSON.stringify(item.rhs) && p.identifier === item.identifier,
-            )![1];
-
           // for each lookahead in the item, create a new entry in the action table to reduce to the production found
           for (const lookahead of item.lookahead) {
             if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
-            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: productionToReduceTo });
+            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: reduceValue[item.prodIndex] });
           }
 
           continue calculateNextItem;
         }
 
         // Create the next item by shifting the dot by 1 place
-        const newItem: Item = { ...item, dot: item.dot + 1 };
+        const newItem: Item = {
+          ...item,
+          dot: item.dot + 1,
+          key: itemKey(item.prodIndex, item.dot + 1, item.lookahead),
+        };
 
         // Determine if we're going to create a GOTO or SHIFT based on the type of the symbol after the dot
         if (nextSymbol.type === "variable") {
@@ -300,19 +337,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         if (GotoTable[currentState.count] === undefined) GotoTable[currentState.count] = new Map();
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -320,10 +349,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        GotoTable[currentState.count]!.set(gotoTransition, existingState.count);
+        GotoTable[currentState.count]!.set(gotoTransition, existingStateIndex);
       }
 
       // For each SHIFT transition from the current state, expand  the item set that it points to
@@ -336,22 +366,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         // If there is no entry in the ActionTable for the current state, create one
         if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
 
         // If no such state exists, create a new state and add it to the list of states
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -359,10 +381,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingState.count });
+        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingStateIndex });
       }
     }
 
@@ -373,9 +396,115 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   const StatesResult = generateStates(initialItemSet);
   if (StatesResult.success === false) return StatesResult;
   const { states, ActionTable, GotoTable } = StatesResult.value;
+
+  if (options.mode === "lalr1") {
+    const merged = mergeStatesForLALR(states, ActionTable, GotoTable);
+    if (merged.success === false) return merged;
+    return {
+      success: true,
+      value: new GeneratorResult(merged.value.states, merged.value.ActionTable, merged.value.GotoTable),
+    };
+  }
+
   return { success: true, value: new GeneratorResult(states, ActionTable, GotoTable) };
 };
 
+// Merges the LR(1) states into LALR(1) states by grouping states that have the same
+// "same-core" item set (items compared without their lookahead sets) and unioning
+// their item sets, action tables and goto tables. Returns an error if merging any
+// group produces a shift/reduce, reduce/reduce, shift or goto conflict, which
+// means the grammar is not LALR(1).
+function mergeStatesForLALR(
+  states: State[],
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[],
+  GotoTable: Map<string, number>[],
+): Result<{
+  states: State[];
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[];
+  GotoTable: Map<string, number>[];
+}> {
+  type Action = { action: "shift" | "reduce"; value: number };
+
+  const itemSkeleton = (item: Item) => `${item.prodIndex}${KEY_SEP}${item.dot}`;
+  const stateSkeleton = (state: State) => state.itemSet.map(itemSkeleton).sort().join(FIELD_SEP);
+
+  // Group states by same-core signature, assigning merged indices in first-occurrence order
+  const groupIndices: number[][] = [];
+  const groupBySkeleton = new Map<string, number>();
+  const oldToNew = new Array<number>(states.length);
+  for (let i = 0; i < states.length; i++) {
+    const skeleton = stateSkeleton(states[i]);
+    let group = groupBySkeleton.get(skeleton);
+    if (group == undefined) {
+      group = groupIndices.length;
+      groupBySkeleton.set(skeleton, group);
+      groupIndices.push([]);
+    }
+    groupIndices[group].push(i);
+    oldToNew[i] = group;
+  }
+
+  const describeAction = (a: Action) =>
+    a.action === "reduce" ? `reduce by production ${a.value}` : `shift to state ${a.value}`;
+  const conflictToken = (members: number[]) => states[members[0]].kernel.lhs;
+  const conflict = (g: number, members: number[], reason: string): Result<never> => ({
+    success: false,
+    reason: `LALR(1) conflict in state ${g} on ${reason} between LR(1) states ${members.join(", ")}: the grammar is not LALR(1)`,
+    token: conflictToken(members),
+  });
+
+  const newStates: State[] = [];
+  const newActionTable: Map<string, Action>[] = [];
+  const newGotoTable: Map<string, number>[] = [];
+
+  for (let g = 0; g < groupIndices.length; g++) {
+    const members = groupIndices[g];
+    const actions = new Map<string, Action>();
+    const gotos = new Map<string, number>();
+
+    for (const member of members) {
+      for (const [key, entry] of ActionTable[member].entries()) {
+        const value: Action = {
+          action: entry.action,
+          value: entry.action === "shift" ? oldToNew[entry.value] : entry.value,
+        };
+        const existing = actions.get(key);
+        if (existing == undefined) actions.set(key, value);
+        else if (existing.action !== value.action || existing.value !== value.value)
+          return conflict(g, members, `"${key}": ${describeAction(existing)} conflicts with ${describeAction(value)}`);
+      }
+
+      if (GotoTable[member] !== undefined)
+        for (const [key, target] of GotoTable[member]!.entries()) {
+          const value = oldToNew[target];
+          const existing = gotos.get(key);
+          if (existing == undefined) gotos.set(key, value);
+          else if (existing !== value)
+            return conflict(g, members, `"${key}": goto to state ${existing} conflicts with goto to state ${value}`);
+        }
+    }
+
+    // Union the item sets of all members, merging the lookahead sets of
+    // items that differ only in their lookahead
+    const itemsBySkeleton = new Map<string, Item>();
+    for (const member of members)
+      for (const item of states[member].itemSet) {
+        const skeleton = itemSkeleton(item);
+        const existing = itemsBySkeleton.get(skeleton);
+        if (existing == undefined) itemsBySkeleton.set(skeleton, { ...item, lookahead: [...item.lookahead] });
+        else
+          for (const lookahead of item.lookahead)
+            if (!existing.lookahead.includes(lookahead)) existing.lookahead.push(lookahead);
+      }
+
+    newStates.push({ itemSet: [...itemsBySkeleton.values()], kernel: states[members[0]].kernel, count: g });
+    newActionTable.push(actions);
+    newGotoTable.push(gotos);
+  }
+
+  return { success: true, value: { states: newStates, ActionTable: newActionTable, GotoTable: newGotoTable } };
+}
+
 export class GeneratorResult {
   constructor(
     public readonly states: State[],

package/src/index.ts

@@ -1,4 +1,5 @@
 export { GeneratorResult, generateStates } from "./generator";
+export type { GeneratorOptions } from "./generator";
 export { buildProductions, tryBuildProductions } from "./meta/selfhosted";
 export { Result } from "./utils/Result";
 export {

package/src/parser.ts

@@ -51,8 +51,9 @@ export class Sparse<TokenType, Metadata, Node> {
   public static tryFromProductions<TokenType, Metadata, Node>(options: {
     productions: Production[];
     toStringifiedTokenType: (tokenType: TokenType) => string;
+    mode?: "lr1" | "lalr1";
   }): Result<Sparse<TokenType, Metadata, Node>> {
-    const statesResult = generateStates(options.productions);
+    const statesResult = generateStates(options.productions, { mode: options.mode });
     if (statesResult.success === false) return statesResult;
     const states = statesResult.value;
 
@@ -68,6 +69,7 @@ export class Sparse<TokenType, Metadata, Node> {
   public static fromProductions<TokenType, Metadata, Node>(options: {
     productions: Production[];
     toStringifiedTokenType: (tokenType: TokenType) => string;
+    mode?: "lr1" | "lalr1";
   }) {
     const result = Sparse.tryFromProductions<TokenType, Metadata, Node>(options);
     if (result.success === false)