@scinorandex/sparse 0.0.8 → 0.1.2

package/src/generator.ts

@@ -1,5 +1,5 @@
 import { TableState } from "./parser";
-import { GrammarToken, Production } from "./meta/common";
+import { GrammarToken, Production, ProductionWarning, validateProductions } from "./meta/common";
 import { Result } from "./utils/Result";
 
 const xContainsAllOfY = <T>(xs: Set<T>, ys: Set<T>) => [...ys].every((x) => xs.has(x));
@@ -20,6 +20,13 @@ function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<
       const toBeModified = ret.get(production.identifier)!;
 
       const firstRhsToken = production.rhs[0];
+      if (firstRhsToken == null)
+        return {
+          success: false,
+          reason: `Production "${production.identifier}" has an empty right hand side, which Sparse does not support`,
+          token: production.lhs,
+        };
+
       if (firstRhsToken.type === "variable") {
         const add = ret.get(firstRhsToken.identifier);
 
@@ -121,10 +128,26 @@ function computeFollowSets(
   return { success: true, value: { firstSets, followSets: ret } };
 }
 
-type Item = Production & { dot: number; lookahead: string[] };
+const KEY_SEP = "\x00";
+const FIELD_SEP = "\x01";
+
+type Item = Production & { dot: number; lookahead: string[]; prodIndex: number; key: string };
 type State = { itemSet: Item[]; kernel: Item; count: number };
 
-export const generateStates = (productions: Production[]): Result<GeneratorResult> => {
+export type GeneratorOptions = {
+  mode?: "lr1" | "lalr1";
+  onWarning?: (warning: ProductionWarning) => void;
+  /** Called every time a new state is discovered, useful to show progress on big grammars. */
+  onProgress?: (statesGenerated: number) => void;
+};
+
+export const generateStates = (
+  productions: Production[],
+  options: GeneratorOptions = {},
+): Result<GeneratorResult> => {
+  const validationResult = validateProductions(productions, { onWarning: options.onWarning });
+  if (validationResult.success === false) return validationResult;
+
   const followSetsResult = computeFollowSets(productions);
   if (followSetsResult.success == false) return followSetsResult;
 
@@ -161,15 +184,45 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     }
   };
 
+  const productionIndicesByLhs = new Map<string, number[]>();
+  for (let i = 0; i < productions.length; i++) {
+    const indexList = productionIndicesByLhs.get(productions[i].identifier);
+    if (indexList == undefined) productionIndicesByLhs.set(productions[i].identifier, [i]);
+    else indexList.push(i);
+  }
+
+  const rhsEntrySignature = (rhsToken: Production["rhs"][number]) =>
+    `${rhsToken.type}${FIELD_SEP}${rhsToken.identifier}${FIELD_SEP}${rhsToken.name ?? ""}${FIELD_SEP}${rhsToken.token.line}${FIELD_SEP}${rhsToken.token.column}`;
+
+  const reduceValue: number[] = [];
+  {
+    const lowestIndexBySignature = new Map<string, number>();
+    for (let i = 0; i < productions.length; i++) {
+      const production = productions[i];
+      const signature = `${production.identifier}${FIELD_SEP}${production.rhs.map(rhsEntrySignature).join(FIELD_SEP)}`;
+      const lowest = lowestIndexBySignature.get(signature);
+      if (lowest == undefined) lowestIndexBySignature.set(signature, i);
+      reduceValue[i] = lowestIndexBySignature.get(signature)!;
+    }
+  }
+
+  const itemKey = (prodIndex: number, dot: number, lookahead: string[]) =>
+    `${prodIndex}${KEY_SEP}${dot}${KEY_SEP}${lookahead.join(FIELD_SEP)}`;
+
+  const itemSetSignature = (itemSet: Item[]) => itemSet.map((item) => item.key).join(FIELD_SEP);
+
   // This fnuction creates the initial item set based on the
   // production provided. It creating the initial item and expands it
-  function generateInitialItemSet(production: Production) {
+  function generateInitialItemSet(production: Production, prodIndex: number) {
+    const lookahead = [EOF_STRING];
     const initialItem = {
       lhs: production.lhs,
       identifier: production.identifier,
       rhs: production.rhs,
       dot: 0,
-      lookahead: [EOF_STRING],
+      lookahead,
+      prodIndex,
+      key: itemKey(prodIndex, 0, lookahead),
     } as Item;
 
     return expandItemSet([initialItem]);
@@ -178,20 +231,19 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   // This function expands the item sets provided to it
   function expandItemSet(items: Item[]): Result<{ itemSet: Item[]; kernel: Item }> {
     const itemSet: Item[] = [...items];
+    const knownItems = new Set(items.map((item) => item.key));
 
-    // Create a queue of unprocessed items and keep shifting until there are no more items left
+    // Create a queue of unprocessed items and keep going until there are no more items left
     const unprocesssedItems: Item[] = [...items];
-    while (unprocesssedItems.length > 0) {
-      const currentItem = unprocesssedItems.shift()!;
+    let queuePosition = 0;
+    while (queuePosition < unprocesssedItems.length) {
+      const currentItem = unprocesssedItems[queuePosition++];
 
       // Check if the symbol after the dot is a non terminal
       const after = currentItem.rhs[currentItem.dot];
       if (after == null) continue; // TODO: check if this should be here
 
       if (after.type === "variable") {
-        // Find prodctions whose left hand side is the symbol after the dot
-        const newProductions = productions.filter((p) => p.identifier === after.identifier);
-
         // Compute the lookahead for the new productions to be added to the item set
         const rest = currentItem.rhs.slice(currentItem.dot + 1);
 
@@ -199,19 +251,22 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         if (lookaheadResult.success === false) return lookaheadResult;
         const lookahead = lookaheadResult.value;
 
-        for (const newProduction of newProductions) {
-          // Create the new item and check if it already exists in the item set
-          // If it doesn't exist, add it to the item set and the queue of unprocessed items
-          const newItem = {
-            lhs: newProduction.lhs,
-            identifier: newProduction.identifier,
-            rhs: newProduction.rhs,
-            dot: 0,
-            lookahead,
-          } as Item;
-          const encoding = JSON.stringify(newItem);
-
-          if (itemSet.some((i) => JSON.stringify(i) === encoding) == false) {
+        const productionIndices = productionIndicesByLhs.get(after.identifier) ?? [];
+        for (const prodIndex of productionIndices) {
+          const newProduction = productions[prodIndex];
+          const key = itemKey(prodIndex, 0, lookahead);
+
+          if (knownItems.has(key) == false) {
+            knownItems.add(key);
+            const newItem = {
+              lhs: newProduction.lhs,
+              identifier: newProduction.identifier,
+              rhs: newProduction.rhs,
+              dot: 0,
+              lookahead,
+              prodIndex,
+              key,
+            } as Item;
             unprocesssedItems.push(newItem);
             itemSet.push(newItem);
           }
@@ -223,7 +278,7 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     return { success: true, value: { itemSet, kernel: items[0] } };
   }
 
-  const initialItemSetResult = generateInitialItemSet(productions[0]);
+  const initialItemSetResult = generateInitialItemSet(productions[0], 0);
   if (initialItemSetResult.success === false) return initialItemSetResult;
   const initialItemSet = initialItemSetResult.value;
 
@@ -242,9 +297,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
     // array of maps whose keys is a terminal is a value of either to shift or reduce
     const ActionTable = [] as Map<string, { action: "shift" | "reduce"; value: number }>[];
 
+    // map from an item set signature to the index of the state holding it
+    const stateIndexBySignature = new Map<string, number>();
+    stateIndexBySignature.set(itemSetSignature(initialState.itemSet), 0);
+
     // While there is an unprocessed state, visit it
-    while (unprocessedStates.length > 0) {
-      const currentState = unprocessedStates.shift()!;
+    let statePosition = 0;
+    while (statePosition < unprocessedStates.length) {
+      const currentState = unprocessedStates[statePosition++];
       const nextStates_Goto = new Map<string, Item[]>();
       const nextStates_Shift = new Map<string, Item[]>();
 
@@ -256,25 +316,21 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
 
         // no next symbol is available, therefore we need to create a reduction
         if (nextSymbol == null) {
-          // Find the prodction that this item reduces to using the symbols on its right hand side
-          // TODO: check if the find can fail, honestly this might be invariant
-          const productionToReduceTo = productions
-            .map((p, i) => [p, i] as const)
-            .find(
-              ([p, _]) => JSON.stringify(p.rhs) === JSON.stringify(item.rhs) && p.identifier === item.identifier,
-            )![1];
-
           // for each lookahead in the item, create a new entry in the action table to reduce to the production found
           for (const lookahead of item.lookahead) {
             if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
-            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: productionToReduceTo });
+            ActionTable[currentState.count]!.set(lookahead, { action: "reduce", value: reduceValue[item.prodIndex] });
           }
 
           continue calculateNextItem;
         }
 
         // Create the next item by shifting the dot by 1 place
-        const newItem: Item = { ...item, dot: item.dot + 1 };
+        const newItem: Item = {
+          ...item,
+          dot: item.dot + 1,
+          key: itemKey(item.prodIndex, item.dot + 1, item.lookahead),
+        };
 
         // Determine if we're going to create a GOTO or SHIFT based on the type of the symbol after the dot
         if (nextSymbol.type === "variable") {
@@ -300,19 +356,11 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         if (GotoTable[currentState.count] === undefined) GotoTable[currentState.count] = new Map();
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -320,10 +368,12 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        GotoTable[currentState.count]!.set(gotoTransition, existingState.count);
+        options.onProgress?.(states.length);
+        GotoTable[currentState.count]!.set(gotoTransition, existingStateIndex);
       }
 
       // For each SHIFT transition from the current state, expand  the item set that it points to
@@ -336,22 +386,14 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
         const expandedItems = expandedItemsResult.value;
 
         // check if a state exist with the same item sets
-        let existingState = states.find((s) => {
-          return (
-            s.kernel.lhs === expandedItems.kernel.lhs &&
-            s.kernel.rhs.length === expandedItems.kernel.rhs.length &&
-            JSON.stringify(s.kernel.rhs) === JSON.stringify(expandedItems.kernel.rhs) &&
-            s.kernel.lookahead.length === expandedItems.kernel.lookahead.length &&
-            JSON.stringify(s.kernel.lookahead) === JSON.stringify(expandedItems.kernel.lookahead) &&
-            JSON.stringify(s.itemSet) === JSON.stringify(expandedItems.itemSet)
-          );
-        });
+        const signature = itemSetSignature(expandedItems.itemSet);
+        let existingStateIndex = stateIndexBySignature.get(signature);
 
         // If there is no entry in the ActionTable for the current state, create one
         if (ActionTable[currentState.count] === undefined) ActionTable[currentState.count] = new Map();
 
         // If no such state exists, create a new state and add it to the list of states
-        if (existingState == undefined) {
+        if (existingStateIndex == undefined) {
           const newState: State = {
             itemSet: expandedItems.itemSet,
             kernel: expandedItems.kernel,
@@ -359,10 +401,12 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
           };
           states.push(newState);
           unprocessedStates.push(newState);
-          existingState = newState;
+          stateIndexBySignature.set(signature, newState.count);
+          existingStateIndex = newState.count;
         }
 
-        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingState.count });
+        options.onProgress?.(states.length);
+        ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingStateIndex });
       }
     }
 
@@ -373,9 +417,117 @@ export const generateStates = (productions: Production[]): Result<GeneratorResul
   const StatesResult = generateStates(initialItemSet);
   if (StatesResult.success === false) return StatesResult;
   const { states, ActionTable, GotoTable } = StatesResult.value;
+
+  if (options.mode === "lalr1") {
+    const merged = mergeStatesForLALR(states, ActionTable, GotoTable);
+    if (merged.success === false) return merged;
+    return {
+      success: true,
+      value: new GeneratorResult(merged.value.states, merged.value.ActionTable, merged.value.GotoTable),
+    };
+  }
+
   return { success: true, value: new GeneratorResult(states, ActionTable, GotoTable) };
 };
 
+// Merges the LR(1) states into LALR(1) states by grouping states that have the same
+// "same-core" item set (items compared without their lookahead sets) and unioning
+// their item sets, action tables and goto tables. Returns an error if merging any
+// group produces a shift/reduce, reduce/reduce, shift or goto conflict, which
+// means the grammar is not LALR(1).
+function mergeStatesForLALR(
+  states: State[],
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[],
+  GotoTable: Map<string, number>[],
+): Result<{
+  states: State[];
+  ActionTable: Map<string, { action: "shift" | "reduce"; value: number }>[];
+  GotoTable: Map<string, number>[];
+}> {
+  type Action = { action: "shift" | "reduce"; value: number };
+
+  const itemSkeleton = (item: Item) => `${item.prodIndex}${KEY_SEP}${item.dot}`;
+  const stateSkeleton = (state: State) => state.itemSet.map(itemSkeleton).sort().join(FIELD_SEP);
+
+  // Group states by same-core signature, assigning merged indices in first-occurrence order
+  const groupIndices: number[][] = [];
+  const groupBySkeleton = new Map<string, number>();
+  const oldToNew = new Array<number>(states.length);
+  for (let i = 0; i < states.length; i++) {
+    const skeleton = stateSkeleton(states[i]);
+    let group = groupBySkeleton.get(skeleton);
+    if (group == undefined) {
+      group = groupIndices.length;
+      groupBySkeleton.set(skeleton, group);
+      groupIndices.push([]);
+    }
+    groupIndices[group].push(i);
+    oldToNew[i] = group;
+  }
+
+  const describeAction = (a: Action) =>
+    a.action === "reduce" ? `reduce by production ${a.value}` : `shift to state ${a.value}`;
+  const conflictToken = (members: number[]) => states[members[0]].kernel.lhs;
+  const conflict = (g: number, members: number[], reason: string): Result<never> => ({
+    success: false,
+    reason: `LALR(1) conflict in state ${g} on ${reason} between LR(1) states ${members.join(", ")}: the grammar is not LALR(1)`,
+    token: conflictToken(members),
+  });
+
+  const newStates: State[] = [];
+  const newActionTable: Map<string, Action>[] = [];
+  const newGotoTable: Map<string, number>[] = [];
+
+  for (let g = 0; g < groupIndices.length; g++) {
+    const members = groupIndices[g];
+    const actions = new Map<string, Action>();
+    const gotos = new Map<string, number>();
+
+    for (const member of members) {
+      // A state whose items only move to other states (no shifts, no completed items) has no
+      // entry in the action table at all.
+      for (const [key, entry] of (ActionTable[member] ?? new Map()).entries()) {
+        const value: Action = {
+          action: entry.action,
+          value: entry.action === "shift" ? oldToNew[entry.value] : entry.value,
+        };
+        const existing = actions.get(key);
+        if (existing == undefined) actions.set(key, value);
+        else if (existing.action !== value.action || existing.value !== value.value)
+          return conflict(g, members, `"${key}": ${describeAction(existing)} conflicts with ${describeAction(value)}`);
+      }
+
+      if (GotoTable[member] !== undefined)
+        for (const [key, target] of GotoTable[member]!.entries()) {
+          const value = oldToNew[target];
+          const existing = gotos.get(key);
+          if (existing == undefined) gotos.set(key, value);
+          else if (existing !== value)
+            return conflict(g, members, `"${key}": goto to state ${existing} conflicts with goto to state ${value}`);
+        }
+    }
+
+    // Union the item sets of all members, merging the lookahead sets of
+    // items that differ only in their lookahead
+    const itemsBySkeleton = new Map<string, Item>();
+    for (const member of members)
+      for (const item of states[member].itemSet) {
+        const skeleton = itemSkeleton(item);
+        const existing = itemsBySkeleton.get(skeleton);
+        if (existing == undefined) itemsBySkeleton.set(skeleton, { ...item, lookahead: [...item.lookahead] });
+        else
+          for (const lookahead of item.lookahead)
+            if (!existing.lookahead.includes(lookahead)) existing.lookahead.push(lookahead);
+      }
+
+    newStates.push({ itemSet: [...itemsBySkeleton.values()], kernel: states[members[0]].kernel, count: g });
+    newActionTable.push(actions);
+    newGotoTable.push(gotos);
+  }
+
+  return { success: true, value: { states: newStates, ActionTable: newActionTable, GotoTable: newGotoTable } };
+}
+
 export class GeneratorResult {
   constructor(
     public readonly states: State[],
@@ -384,7 +536,15 @@ export class GeneratorResult {
   ) {}
 
   toTable(): string {
-    return this.ActionTable.map((actions, idx) => {
+    const actionless = this.ActionTable.findIndex((actions) => actions === undefined || actions.size === 0);
+    if (actionless !== -1)
+      throw new Error(
+        `State ${actionless} of this grammar has no actions, so it cannot be written in the parsing table format. ` +
+          `Use Sparse.fromProductions to keep the states in memory instead.`,
+      );
+
+    // Array.from, not map: the action table is sparse and map would keep the holes.
+    return Array.from(this.ActionTable, (actions, idx) => {
       return [
         ...[...actions.entries()].map(([k, { action, value }]) => `${k}=${action === "reduce" ? "r" : "s"}${value}`),
         ...(this.GotoTable[idx] === undefined ? [] : [...this.GotoTable[idx].entries()].map(([k, v]) => `${k}=${v}`)),
@@ -393,14 +553,24 @@ export class GeneratorResult {
   }
 
   toStates(): TableState[] {
-    return this.ActionTable.map((actions, idx) => {
+    // Array.from, not map: a state without actions leaves a hole in the action table, and map would
+    // hand back an array that is missing that state and shifts every later state number.
+    return Array.from(this.ActionTable, (actions, idx) => {
       const tableState = new TableState();
 
-      for (const [k, { action, value }] of actions.entries())
+      // See toTable(): a state can legitimately have no actions, it just cannot be serialized.
+      if (actions === undefined) return tableState;
+
+      for (const [k, { action, value }] of actions.entries()) {
         tableState.actions.set(k, { type: action === "reduce" ? "reduce" : "shift", value });
+        tableState.rawActions.set(k, `${action === "reduce" ? "r" : "s"}${value}`);
+      }
 
       if (this.GotoTable[idx] !== undefined)
-        for (const [k, value] of this.GotoTable[idx].entries()) tableState.actions.set(k, { type: "goto", value });
+        for (const [k, value] of this.GotoTable[idx].entries()) {
+          tableState.actions.set(k, { type: "goto", value });
+          tableState.rawActions.set(k, `${value}`);
+        }
 
       return tableState;
     });

package/src/index.ts

@@ -1,13 +1,66 @@
 export { GeneratorResult, generateStates } from "./generator";
-export { buildProductions, tryBuildProductions } from "./meta/selfhosted";
+export type { GeneratorOptions } from "./generator";
+
+export {
+  buildProductions,
+  tryBuildProductions,
+  grammarLexerGenerator,
+  getSelfHostedParserGenerator,
+} from "./meta/selfhosted";
+export type { LexerInterface } from "./meta/selfhosted";
+
 export { Result } from "./utils/Result";
 export {
   ParserRecoveryFunction,
   Sparse,
+  SparseGrammarError,
   LR1ParserGraveError,
   LR1StackSymbol,
+  LR1Parser,
   ParserResult,
   ParserError,
+  TableAction,
+  TableState,
 } from "./parser";
-export { buildStates } from "./table/selfhosted";
-export { Production } from "./meta/common";
+export type { ReducerType, SparseOptions, FromProductionsOptions } from "./parser";
+
+export { buildStates, tryBuildStates } from "./table/selfhosted";
+export type { BuildStatesOptions } from "./table/selfhosted";
+export { validateTable, validateTableStates } from "./table/validate";
+export type { ValidateTableOptions, ValidateTableStatesOptions, TableFormatWarning } from "./table/validate";
+
+export {
+  dehydateProduction,
+  hydrateProduction,
+  isVariableIdentifier,
+  syntheticGrammarToken,
+  terminalIdentifier,
+  validateProductions,
+  variableIdentifier,
+  GrammarTokenType,
+} from "./meta/common";
+export type {
+  GrammarToken,
+  GrammarTokenMetadata,
+  Production,
+  ProductionRhsItem,
+  ProductionWarning,
+  ProductionWarningKind,
+  ValidateProductionsOptions,
+} from "./meta/common";
+
+export {
+  assertReducersCoverGrammar,
+  defineReducers,
+  enumToString,
+  missingReducerMessage,
+  missingReducerNames,
+  namedProductions,
+  KLEENE_REDUCER_NAME,
+} from "./utils/reducers";
+export type { ReducerMap } from "./utils/reducers";
+
+export { loadGrammar, loadTable } from "./utils/loadFiles";
+export type { LoadGrammarOptions, LoadTableOptions } from "./utils/loadFiles";
+export { buildErrorWindow } from "./utils/errorWindowBuilder";
+export { Stack } from "./utils/Stack";

package/src/meta/common.ts

@@ -1,4 +1,5 @@
 import { ColumnAndRow, Token } from "@scinorandex/slex";
+import { Result } from "../utils/Result";
 
 export type Production = {
   lhs: GrammarToken;
@@ -8,6 +9,8 @@ export type Production = {
   rhs: { type: "terminal" | "variable"; token: GrammarToken; identifier: string; name: string | null }[];
 };
 
+export type ProductionRhsItem = Production["rhs"][number];
+
 const dehydateGrammarToken = ({ column, lexeme, line, type }: GrammarToken) => {
   return { column, lexeme, line, type };
 };
@@ -50,6 +53,119 @@ export const hydrateProduction = (production: ReturnType<typeof dehydateProducti
 };
 
 export type GrammarTokenMetadata = {};
+
+/** A readable one-liner for an unknown thrown value, used to turn it into a `Result` failure. */
+export const describeThrowable = (err: unknown) =>
+  err instanceof Error ? err.message : `Unexpected error: ${String(err)}`;
+
+/** A stand-in token used to report errors that do not belong to a specific place in a grammar. */
+export const syntheticGrammarToken = () =>
+  new Token(GrammarTokenType.EOF, "", new ColumnAndRow(0, 0), {}) as GrammarToken;
+
+export type ProductionWarningKind = "duplicate-rhs-name" | "unreachable-production";
+
+export type ProductionWarning = {
+  kind: ProductionWarningKind;
+  reason: string;
+  token: GrammarToken;
+};
+
+export type ValidateProductionsOptions = {
+  /** Called once per warning. Warnings never stop the parser generator, they only flag suspicious grammars. */
+  onWarning?: (warning: ProductionWarning) => void;
+};
+
+const at = (token: GrammarToken) => `${token.line}:${token.column}`;
+
+/**
+ * Checks a set of unrolled productions for the mistakes that would otherwise surface as an
+ * inscrutable `TypeError` deep inside state generation or parsing:
+ *
+ * - a variable used on the right hand side that no production defines
+ * - a production with an empty right hand side (Sparse has no support for empty productions)
+ * - the start symbol being used on the right hand side (it must only appear on the left hand side)
+ *
+ * Suspicious-but-legal grammars (duplicate symbol names, unreachable productions) are reported
+ * through `onWarning` instead of failing.
+ */
+export const validateProductions = (
+  productions: Production[],
+  options: ValidateProductionsOptions = {},
+): Result<Production[]> => {
+  const warn = (kind: ProductionWarningKind, reason: string, token: GrammarToken) => {
+    options.onWarning?.({ kind, reason, token });
+  };
+
+  if (productions.length === 0)
+    return {
+      success: false,
+      reason: "The grammar does not contain any productions. Is the grammar file empty?",
+      token: syntheticGrammarToken(),
+    };
+
+  const lhsIdentifiers = new Set(productions.map((production) => production.identifier));
+  const referencedIdentifiers = new Set<string>();
+
+  for (const production of productions) {
+    if (production.rhs.length === 0)
+      return {
+        success: false,
+        reason: `Production "${production.identifier}" (line ${at(production.lhs)}) has an empty right hand side. Sparse does not support empty productions, so a lone "(...)? " group such as "<X: x>: ([A])?;" is not a valid production.`,
+        token: production.lhs,
+      };
+
+    const seenNames = new Set<string>();
+    for (const rhsItem of production.rhs) {
+      if (rhsItem.type === "variable") {
+        referencedIdentifiers.add(rhsItem.identifier);
+        if (lhsIdentifiers.has(rhsItem.identifier) === false)
+          return {
+            success: false,
+            reason: `Variable "${rhsItem.identifier}" is used on the right hand side of production "${production.identifier}" (line ${at(production.lhs)}) but no production defines it as its left hand side.`,
+            token: rhsItem.token,
+          };
+      }
+
+      if (rhsItem.name != null) {
+        if (seenNames.has(rhsItem.name))
+          warn(
+            "duplicate-rhs-name",
+            `Production "${production.identifier}" (line ${at(production.lhs)}) names more than one symbol "${rhsItem.name}". Only the last one ends up on the reducer's "bag", use the positional input to reach the others.`,
+            rhsItem.token,
+          );
+        seenNames.add(rhsItem.name);
+      }
+    }
+  }
+
+  // The first production is the accept production: its left hand side is the start symbol, which
+  // by definition cannot be referenced from the right hand side of anything.
+  const startProduction = productions[0];
+  if (referencedIdentifiers.has(startProduction.identifier))
+    return {
+      success: false,
+      reason: `"${startProduction.identifier}" is the start symbol of the grammar, so it cannot be used on the right hand side of a production. It is used at line ${at(startProduction.lhs)}.`,
+      token: startProduction.lhs,
+    };
+
+  for (let i = 1; i < productions.length; i++) {
+    const production = productions[i];
+    if (referencedIdentifiers.has(production.identifier)) continue;
+    warn(
+      "unreachable-production",
+      `Production "${production.identifier}" (line ${at(production.lhs)}) is never used on the right hand side of another production, so the parser can never reach it.`,
+      production.lhs,
+    );
+  }
+
+  return { success: true, value: productions };
+};
+
+/** True when the identifier is a variable (`<FOO>`) rather than a terminal (`[FOO]`). */
+export const isVariableIdentifier = (identifier: string) => identifier.startsWith("<") && identifier.endsWith(">");
+
+export const terminalIdentifier = (name: string) => `[${name}]`;
+export const variableIdentifier = (name: string) => `<${name}>`;
 export enum GrammarTokenType {
   IDENTIFIER,
   L_ANGLE,