@scinorandex/sparse 0.1.1 → 0.1.2

package/src/meta/common.ts

@@ -1,4 +1,5 @@
 import { ColumnAndRow, Token } from "@scinorandex/slex";
+import { Result } from "../utils/Result";
 
 export type Production = {
   lhs: GrammarToken;
@@ -8,6 +9,8 @@ export type Production = {
   rhs: { type: "terminal" | "variable"; token: GrammarToken; identifier: string; name: string | null }[];
 };
 
+export type ProductionRhsItem = Production["rhs"][number];
+
 const dehydateGrammarToken = ({ column, lexeme, line, type }: GrammarToken) => {
   return { column, lexeme, line, type };
 };
@@ -50,6 +53,119 @@ export const hydrateProduction = (production: ReturnType<typeof dehydateProducti
 };
 
 export type GrammarTokenMetadata = {};
+
+/** A readable one-liner for an unknown thrown value, used to turn it into a `Result` failure. */
+export const describeThrowable = (err: unknown) =>
+  err instanceof Error ? err.message : `Unexpected error: ${String(err)}`;
+
+/** A stand-in token used to report errors that do not belong to a specific place in a grammar. */
+export const syntheticGrammarToken = () =>
+  new Token(GrammarTokenType.EOF, "", new ColumnAndRow(0, 0), {}) as GrammarToken;
+
+export type ProductionWarningKind = "duplicate-rhs-name" | "unreachable-production";
+
+export type ProductionWarning = {
+  kind: ProductionWarningKind;
+  reason: string;
+  token: GrammarToken;
+};
+
+export type ValidateProductionsOptions = {
+  /** Called once per warning. Warnings never stop the parser generator, they only flag suspicious grammars. */
+  onWarning?: (warning: ProductionWarning) => void;
+};
+
+const at = (token: GrammarToken) => `${token.line}:${token.column}`;
+
+/**
+ * Checks a set of unrolled productions for the mistakes that would otherwise surface as an
+ * inscrutable `TypeError` deep inside state generation or parsing:
+ *
+ * - a variable used on the right hand side that no production defines
+ * - a production with an empty right hand side (Sparse has no support for empty productions)
+ * - the start symbol being used on the right hand side (it must only appear on the left hand side)
+ *
+ * Suspicious-but-legal grammars (duplicate symbol names, unreachable productions) are reported
+ * through `onWarning` instead of failing.
+ */
+export const validateProductions = (
+  productions: Production[],
+  options: ValidateProductionsOptions = {},
+): Result<Production[]> => {
+  const warn = (kind: ProductionWarningKind, reason: string, token: GrammarToken) => {
+    options.onWarning?.({ kind, reason, token });
+  };
+
+  if (productions.length === 0)
+    return {
+      success: false,
+      reason: "The grammar does not contain any productions. Is the grammar file empty?",
+      token: syntheticGrammarToken(),
+    };
+
+  const lhsIdentifiers = new Set(productions.map((production) => production.identifier));
+  const referencedIdentifiers = new Set<string>();
+
+  for (const production of productions) {
+    if (production.rhs.length === 0)
+      return {
+        success: false,
+        reason: `Production "${production.identifier}" (line ${at(production.lhs)}) has an empty right hand side. Sparse does not support empty productions, so a lone "(...)? " group such as "<X: x>: ([A])?;" is not a valid production.`,
+        token: production.lhs,
+      };
+
+    const seenNames = new Set<string>();
+    for (const rhsItem of production.rhs) {
+      if (rhsItem.type === "variable") {
+        referencedIdentifiers.add(rhsItem.identifier);
+        if (lhsIdentifiers.has(rhsItem.identifier) === false)
+          return {
+            success: false,
+            reason: `Variable "${rhsItem.identifier}" is used on the right hand side of production "${production.identifier}" (line ${at(production.lhs)}) but no production defines it as its left hand side.`,
+            token: rhsItem.token,
+          };
+      }
+
+      if (rhsItem.name != null) {
+        if (seenNames.has(rhsItem.name))
+          warn(
+            "duplicate-rhs-name",
+            `Production "${production.identifier}" (line ${at(production.lhs)}) names more than one symbol "${rhsItem.name}". Only the last one ends up on the reducer's "bag", use the positional input to reach the others.`,
+            rhsItem.token,
+          );
+        seenNames.add(rhsItem.name);
+      }
+    }
+  }
+
+  // The first production is the accept production: its left hand side is the start symbol, which
+  // by definition cannot be referenced from the right hand side of anything.
+  const startProduction = productions[0];
+  if (referencedIdentifiers.has(startProduction.identifier))
+    return {
+      success: false,
+      reason: `"${startProduction.identifier}" is the start symbol of the grammar, so it cannot be used on the right hand side of a production. It is used at line ${at(startProduction.lhs)}.`,
+      token: startProduction.lhs,
+    };
+
+  for (let i = 1; i < productions.length; i++) {
+    const production = productions[i];
+    if (referencedIdentifiers.has(production.identifier)) continue;
+    warn(
+      "unreachable-production",
+      `Production "${production.identifier}" (line ${at(production.lhs)}) is never used on the right hand side of another production, so the parser can never reach it.`,
+      production.lhs,
+    );
+  }
+
+  return { success: true, value: productions };
+};
+
+/** True when the identifier is a variable (`<FOO>`) rather than a terminal (`[FOO]`). */
+export const isVariableIdentifier = (identifier: string) => identifier.startsWith("<") && identifier.endsWith(">");
+
+export const terminalIdentifier = (name: string) => `[${name}]`;
+export const variableIdentifier = (name: string) => `<${name}>`;
 export enum GrammarTokenType {
   IDENTIFIER,
   L_ANGLE,

package/src/meta/selfhosted.ts

@@ -1,8 +1,17 @@
 import { RegexEngine, Slex } from "@scinorandex/slex";
+import { describeThrowable, syntheticGrammarToken } from "../meta/common";
 import { Result, Sparse } from "../index";
 import { selfhosted } from "./states";
-import { TableState } from "../parser";
-import { GrammarToken, GrammarTokenMetadata, GrammarTokenType, hydrateProduction, Production } from "./common";
+import { LR1ParserGraveError, TableState } from "../parser";
+import {
+  GrammarToken,
+  GrammarTokenMetadata,
+  GrammarTokenType,
+  hydrateProduction,
+  Production,
+  ValidateProductionsOptions,
+  validateProductions,
+} from "./common";
 
 export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMetadata>({
   EOF_TYPE: GrammarTokenType.EOF,
@@ -21,9 +30,12 @@ grammarLexerGenerator.addRule(
 grammarLexerGenerator.addRule("letter", "${lowercase} | ${uppercase}");
 grammarLexerGenerator.addRule("digit", "0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9");
 grammarLexerGenerator.addRule("alphanumeric", "${letter} | ${digit}");
+// Dashes are allowed inside identifiers (but not first) so that the autogenerated productions of a
+// "*" or "+" group, which are named "autogen-0", "autogen-1", ... survive a round trip through a
+// table file.
 grammarLexerGenerator.addRule(
   "identifier",
-  "(${letter} | $_) (${letter} | ${digit} | $_)*",
+  "(${letter} | $_) (${letter} | ${digit} | $_ | $-)*",
   GrammarTokenType.IDENTIFIER,
 );
 grammarLexerGenerator.addRule("colon", "$:", GrammarTokenType.COLON);
@@ -312,8 +324,15 @@ const reducers: { [key: string]: Reducer } = {
   },
 };
 
-export const tryBuildProductions = (lexer: LexerInterface | string): Result<Production[]> => {
-  if (typeof lexer === "string") lexer = grammarLexerGenerator.generate(lexer, () => ({}));
+export const tryBuildProductions = (
+  lexer: LexerInterface | string,
+  options: ValidateProductionsOptions = {},
+): Result<Production[]> => {
+  try {
+    if (typeof lexer === "string") lexer = grammarLexerGenerator.generate(lexer, () => ({}));
+  } catch (err) {
+    return { success: false, reason: describeThrowable(err), token: syntheticGrammarToken() };
+  }
 
   const parserGenerator = getSelfHostedParserGenerator();
   const parser = parserGenerator.generate(lexer as RegexEngine<GrammarTokenType, GrammarTokenMetadata>, {
@@ -322,19 +341,31 @@ export const tryBuildProductions = (lexer: LexerInterface | string): Result<Prod
     },
   });
 
-  const parsingResult = parser.parse().result as ProgramNode;
-  const productions = parsingResult.getProductions();
-  return { success: true, value: productions };
+  let programNode: ProgramNode;
+  try {
+    programNode = parser.parse().result as ProgramNode;
+  } catch (err) {
+    // The grammar parser has no error recovery, so a malformed grammar surfaces as a
+    // LR1ParserGraveError. Translate it into the Result failure every caller already handles.
+    if (err instanceof LR1ParserGraveError && err.currentToken != null)
+      return { success: false, reason: err.reason, token: err.currentToken as GrammarToken };
+    return { success: false, reason: describeThrowable(err), token: syntheticGrammarToken() };
+  }
+
+  return validateProductions(programNode.getProductions(), options);
 };
 
-interface LexerInterface {
+export interface LexerInterface {
   peekNextToken(): GrammarToken;
   getNextToken(): GrammarToken;
   hasNextToken(): boolean;
 }
 
-export const buildProductions = (lexer: LexerInterface | string): Production[] => {
-  const result = tryBuildProductions(lexer);
+export const buildProductions = (
+  lexer: LexerInterface | string,
+  options: ValidateProductionsOptions = {},
+): Production[] => {
+  const result = tryBuildProductions(lexer, options);
   if (result.success === false)
     throw new Error(
       `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`,

package/src/parser.ts

@@ -2,15 +2,27 @@ import { Slex, Token } from "@scinorandex/slex";
 import { generateStates } from "./generator";
 import { Stack } from "./utils/Stack";
 import { Result } from "./utils/Result";
-import { Production } from "./meta/common";
+import { GrammarToken, Production, ProductionWarning } from "./meta/common";
+import { loadGrammar, loadTable } from "./utils/loadFiles";
+import { validateTable } from "./table/validate";
 
 export type TableAction = { type: "shift" | "reduce" | "goto"; value: number };
 
 export class TableState {
   public readonly actions: Map<string, TableAction>;
 
-  constructor(actions = new Map<string, TableAction>()) {
+  /**
+   * The actions exactly as they were written in a table file (`"s12"`, `"r3"`, `"7"`), kept so that
+   * malformed entries can be reported with the text the user actually typed.
+   */
+  public readonly rawActions: Map<string, string>;
+
+  constructor(
+    actions = new Map<string, TableAction>(),
+    rawActions = new Map<string, string>(),
+  ) {
     this.actions = actions;
+    this.rawActions = rawActions;
   }
 
   public getTerminalAction(terminal: string): TableAction | undefined {
@@ -21,16 +33,41 @@ export class TableState {
     return this.actions.get(variable);
   }
 
+  /** Every terminal this state can consume, without the surrounding brackets, sorted. */
+  public getTerminalKeys(): string[] {
+    return this.getKeys("[", "]");
+  }
+
+  /** Every variable this state can reduce past, without the surrounding angle brackets, sorted. */
+  public getVariableKeys(): string[] {
+    return this.getKeys("<", ">");
+  }
+
+  private getKeys(prefix: string, suffix: string): string[] {
+    const keys: string[] = [];
+    for (const key of this.actions.keys())
+      if (key.startsWith(prefix) && key.endsWith(suffix))
+        keys.push(key.substring(prefix.length, key.length - suffix.length));
+    return keys.sort();
+  }
+
   toJSObject() {
     return Object.fromEntries(this.actions);
   }
 
   static fromJSObject(json: ReturnType<TableState["toJSObject"]>) {
-    return new TableState(new Map(Object.entries(json)));
+    const actions = new Map(Object.entries(json));
+    const rawActions = new Map(
+      [...actions].map(([key, action]) => [
+        key,
+        action.type === "goto" ? `${action.value}` : `${action.type === "shift" ? "s" : "r"}${action.value}`,
+      ]),
+    );
+    return new TableState(actions, rawActions);
   }
 }
 
-type ReducerType<TokenType, Metadata, Node> = (
+export type ReducerType<TokenType, Metadata, Node> = (
   newInput: { bag: any; name: string | null },
   oldInput: { input: LR1StackSymbol<TokenType, Metadata, Node>[]; index: number },
 ) => Node;
@@ -39,21 +76,50 @@ export type LR1StackSymbol<TokenType, Metadata, Node> =
   | { type: "token"; token: Token<TokenType, Metadata> }
   | { type: "node"; node: Node };
 
+export type SparseOptions<TokenType> = {
+  productions: Production[];
+  states: TableState[];
+  toStringifiedTokenType: (tokenType: TokenType) => string;
+  /**
+   * Cross-check the productions against the parsing table right away, so a stale table is reported
+   * here instead of as an opaque crash in the middle of a parse. Off by default.
+   */
+  validate?: boolean;
+  /** Name of the grammar file, used to make validation errors actionable. */
+  source?: string;
+};
+
+export type FromProductionsOptions<TokenType> = {
+  productions: Production[];
+  toStringifiedTokenType: (tokenType: TokenType) => string;
+  mode?: "lr1" | "lalr1";
+  /** Generating states can take a while for big grammars; set to true to silence the timing log. */
+  quiet?: boolean;
+  onWarning?: (warning: ProductionWarning) => void;
+  onProgress?: (statesGenerated: number) => void;
+};
+
 export class Sparse<TokenType, Metadata, Node> {
   public constructor(
-    public readonly options: {
-      productions: Production[];
-      states: TableState[];
-      toStringifiedTokenType: (tokenType: TokenType) => string;
-    },
-  ) {}
+    public readonly options: SparseOptions<TokenType>,
+  ) {
+    if (options.validate === true) {
+      const result = validateTable(options.productions, options.states, { source: options.source });
+      if (result.success === false)
+        throw new Error(`The parsing table does not match the grammar: ${result.reason}`);
+    }
+  }
 
-  public static tryFromProductions<TokenType, Metadata, Node>(options: {
-    productions: Production[];
-    toStringifiedTokenType: (tokenType: TokenType) => string;
-    mode?: "lr1" | "lalr1";
-  }): Result<Sparse<TokenType, Metadata, Node>> {
-    const statesResult = generateStates(options.productions, { mode: options.mode });
+  public static tryFromProductions<TokenType, Metadata, Node>(
+    options: FromProductionsOptions<TokenType>,
+  ): Result<Sparse<TokenType, Metadata, Node>> {
+    const startedAt = Date.now();
+
+    const statesResult = generateStates(options.productions, {
+      mode: options.mode,
+      onWarning: options.onWarning,
+      onProgress: options.onProgress,
+    });
     if (statesResult.success === false) return statesResult;
     const states = statesResult.value;
 
@@ -63,17 +129,63 @@ export class Sparse<TokenType, Metadata, Node> {
       toStringifiedTokenType: options.toStringifiedTokenType,
     });
 
+    if (options.quiet !== true) {
+      const elapsed = Date.now() - startedAt;
+      const label = options.mode === "lalr1" ? "LALR(1)" : "LR(1)";
+      console.error(`Sparse generated ${sparse.options.states.length} ${label} states in ${elapsed}ms`);
+    }
+
     return { success: true, value: sparse };
   }
 
-  public static fromProductions<TokenType, Metadata, Node>(options: {
-    productions: Production[];
-    toStringifiedTokenType: (tokenType: TokenType) => string;
-    mode?: "lr1" | "lalr1";
-  }) {
+  public static fromProductions<TokenType, Metadata, Node>(options: FromProductionsOptions<TokenType>) {
     const result = Sparse.tryFromProductions<TokenType, Metadata, Node>(options);
-    if (result.success === false)
-      throw new Error(`Encountered error "${result.reason}" at ${result.token.line}:${result.token.column}`);
+    if (result.success === false) throw new SparseGrammarError(result.reason, result.token);
+    return result.value;
+  }
+
+  /** Builds a parser generator from a grammar file and a prebuilt parsing table file. */
+  public static async tryFromGrammarFile<TokenType, Metadata, Node>(options: {
+    grammarPath: string;
+    tablePath: string;
+    toStringifiedTokenType: (tokenType: TokenType) => string;
+    onWarning?: (warning: ProductionWarning) => void;
+    /** Set to false to skip cross-checking the table against the grammar. */
+    validate?: boolean;
+  }): Promise<Result<Sparse<TokenType, Metadata, Node>>> {
+    const productionsResult = await loadGrammar(options.grammarPath, { onWarning: options.onWarning });
+    if (productionsResult.success === false) return productionsResult;
+
+    const statesResult = await loadTable(options.tablePath, { productions: productionsResult.value });
+    if (statesResult.success === false) return statesResult;
+
+    if (options.validate !== false) {
+      const tableResult = validateTable(productionsResult.value, statesResult.value, {
+        source: options.tablePath,
+      });
+      if (tableResult.success === false) return tableResult;
+    }
+
+    return {
+      success: true,
+      value: new Sparse<TokenType, Metadata, Node>({
+        productions: productionsResult.value,
+        states: statesResult.value,
+        toStringifiedTokenType: options.toStringifiedTokenType,
+        source: options.grammarPath,
+      }),
+    };
+  }
+
+  public static async fromGrammarFile<TokenType, Metadata, Node>(options: {
+    grammarPath: string;
+    tablePath: string;
+    toStringifiedTokenType: (tokenType: TokenType) => string;
+    onWarning?: (warning: ProductionWarning) => void;
+    validate?: boolean;
+  }): Promise<Sparse<TokenType, Metadata, Node>> {
+    const result = await Sparse.tryFromGrammarFile<TokenType, Metadata, Node>(options);
+    if (result.success === false) throw new SparseGrammarError(result.reason, result.token);
     return result.value;
   }
 
@@ -99,6 +211,12 @@ export type ParserRecoveryFunction<TokenType, Metadata, Node> = (options: {
     success: true;
     token?: Token<TokenType, Metadata>;
   };
+  /**
+   * Pushes a token that the input did not contain (a synthesized semicolon, a closing brace, ...)
+   * onto both stacks, exactly as if the lexer had produced it. Unlike `finish({ newToken })`, which
+   * only hands the token back to the parser as a lookahead, this one actually shifts it.
+   */
+  insertToken: (token: Token<TokenType, Metadata>) => { success: false; reason: string } | undefined;
   states: TableState[];
 }) => { success: true; token?: Token<TokenType, Metadata> } | { success: false; reason: string };
 
@@ -108,38 +226,52 @@ export type ParserResult<TokenType, Metadata, Node> = {
   errors: ParserError<TokenType, Metadata>[];
 };
 
-class LR1Parser<TokenType, Metadata, Node> {
+export class LR1Parser<TokenType, Metadata, Node> {
   statesStack: Stack<number> = new Stack([0]);
   symbolsStack: Stack<LR1StackSymbol<TokenType, Metadata, Node>> = new Stack();
   exceptions: ParserError<TokenType, Metadata>[] = [];
 
   public constructor(
-    public readonly options: {
-      productions: Production[];
-      states: TableState[];
-      toStringifiedTokenType: (tokenType: TokenType) => string;
-    },
+    public readonly options: SparseOptions<TokenType>,
     public readonly reducer: ReducerType<TokenType, Metadata, Node>,
     public readonly recover: ParserRecoveryFunction<TokenType, Metadata, Node> | null,
     public readonly lexer: ReturnType<Slex<TokenType, Metadata>["generate"]>,
   ) {}
 
+  /**
+   * Clears the stacks and the collected errors so the parser can be handed to a fresh lexer.
+   * A parser instance consumes its lexer, so the usual way to parse a second input is to call
+   * `generator.generate(newLexer, options)` again.
+   */
+  public reset() {
+    this.statesStack = new Stack([0]);
+    this.symbolsStack = new Stack();
+    this.exceptions = [];
+  }
+
   public parse(): ParserResult<TokenType, Metadata, Node> {
     const { productions, states, toStringifiedTokenType } = this.options;
 
+    if (states.length === 0)
+      throw new LR1ParserGraveError("The parsing table is empty, so no token can be parsed", null as any);
+
+    const currentStates = () => {
+      const state = states[this.statesStack.peek()];
+      if (state == null)
+        throw new LR1ParserGraveError(
+          `The parsing table is out of sync with the grammar: state ${this.statesStack.peek()} does not exist (the table has ${states.length} states)`,
+          null as any,
+        );
+      return state;
+    };
+
     while (true) {
-      let currentState = this.statesStack.peek();
+      let currentState = currentStates();
       let token = this.lexer.peekNextToken();
-      let action = states[currentState].getTerminalAction(toStringifiedTokenType(token.type));
+      let action = currentState.getTerminalAction(toStringifiedTokenType(token.type));
 
       if (action == null) {
-        if (this.recover === null)
-          throw new LR1ParserGraveError(
-            `Invalid syntax. Expected ${[...states[currentState].actions.keys()].join(
-              ",",
-            )} but got ${toStringifiedTokenType(token.type)}`,
-            token,
-          );
+        if (this.recover === null) throw new LR1ParserGraveError(this.syntaxErrorMessage(currentState, token), token);
 
         const recoveryResult = this.recover({
           lexer: this.lexer,
@@ -149,22 +281,25 @@ class LR1Parser<TokenType, Metadata, Node> {
           addError: (reason: string) => this.exceptions.push({ message: reason, token: this.lexer.peekNextToken() }),
           crash: (reason: string) => ({ success: false, reason }),
           finish: (options?: { newToken: Token<TokenType, Metadata> }) => ({ success: true, token: options?.newToken }),
+          insertToken: (newToken: Token<TokenType, Metadata>) => this.insertToken(newToken),
           isSafe: () =>
-            states[this.statesStack.peek()].actions.has(`[${toStringifiedTokenType(this.lexer.peekNextToken().type)}]`),
+            currentStates().actions.has(`[${toStringifiedTokenType(this.lexer.peekNextToken().type)}]`),
         });
 
         if (recoveryResult.success === false) throw new LR1ParserGraveError(recoveryResult.reason, token);
 
         token = recoveryResult.token ?? this.lexer.peekNextToken();
-        currentState = this.statesStack.peek();
-        action = states[currentState].getTerminalAction(toStringifiedTokenType(token.type));
+        currentState = currentStates();
+        action = currentState.getTerminalAction(toStringifiedTokenType(token.type));
+
+        if (action == null) throw new LR1ParserGraveError(this.syntaxErrorMessage(currentState, token), token);
       }
 
       const fixedAction = action as TableAction;
       if (fixedAction.type === "shift") {
         // add current token to the stack and push the next state
         token = this.lexer.getNextToken();
-        this.statesStack.push(fixedAction.value);
+        this.pushState(fixedAction.value);
         this.symbolsStack.push({ type: "token", token: token });
       }
 
@@ -173,6 +308,12 @@ class LR1Parser<TokenType, Metadata, Node> {
         if (fixedAction.value === 0) break;
         const production = productions[fixedAction.value];
 
+        if (production == null)
+          throw new LR1ParserGraveError(
+            `The parsing table is out of sync with the grammar: it reduces by production ${fixedAction.value}, but the grammar only has ${productions.length} productions`,
+            token,
+          );
+
         // pop the stack and reduce by this production
         const popped: LR1StackSymbol<TokenType, Metadata, Node>[] = [];
         for (let i = 0; i < production.rhs.length; i++) {
@@ -200,17 +341,18 @@ class LR1Parser<TokenType, Metadata, Node> {
           const node = this.reducer(newInput, oldInput);
           this.symbolsStack.push({ type: "node", node });
         } catch (err) {
-          const input = JSON.stringify(popped, null, 2);
-          console.log(err);
+          const cause = err instanceof Error ? err.message : String(err);
+          const input = truncate(JSON.stringify(popped), 500);
           throw new LR1ParserGraveError(
-            `Error while performing reduction for production: ${productionIndex}. Reduction input: ${input}`,
+            `Error while performing the reduction for production ${productionIndex}${describeProduction(production)}: ${cause}\nReduction input: ${input}`,
             token,
+            err instanceof Error ? err : undefined,
           );
         }
 
         // get the top node and figure out what state to add to state stack
         const topNode = this.statesStack.peek();
-        const gotoAction = states[topNode].getVariableAction(production.identifier);
+        const gotoAction = currentStates().getVariableAction(production.identifier);
 
         if (gotoAction == undefined)
           throw new LR1ParserGraveError(
@@ -223,21 +365,89 @@ class LR1Parser<TokenType, Metadata, Node> {
             token,
           );
 
-        this.statesStack.push(gotoAction.value);
+        this.pushState(gotoAction.value);
       }
     }
 
+    // A table can accept the empty input by reducing to production 0 straight away, in which case
+    // nothing was ever shifted and there is no tree to hand back.
+    if (this.symbolsStack.isEmpty) return { result: null, errors: this.exceptions };
+
     const top = this.symbolsStack.peek();
     return { result: top.type === "node" ? top.node : null, errors: this.exceptions };
   }
+
+  private pushState(state: number) {
+    if (state === undefined || this.options.states[state] == null)
+      throw new LR1ParserGraveError(
+        `The parsing table is out of sync with the grammar: it points at state ${state}, but the table only has ${this.options.states.length} states`,
+        null as any,
+      );
+    this.statesStack.push(state);
+  }
+
+  /** Shifts a token that was not in the input, used by error recovery. */
+  private insertToken(token: Token<TokenType, Metadata>): { success: false; reason: string } | undefined {
+    const state = this.options.states[this.statesStack.peek()];
+    const action = state?.getTerminalAction(this.options.toStringifiedTokenType(token.type));
+
+    if (action == null)
+      return {
+        success: false,
+        reason: `Cannot insert ${this.options.toStringifiedTokenType(token.type)} here: state ${
+          this.statesStack.peek()
+        } does not accept it. Expected ${state?.getTerminalKeys().join(", ") ?? "nothing"}`,
+      };
+
+    if (action.type !== "shift")
+      return {
+        success: false,
+        reason: `Cannot insert ${this.options.toStringifiedTokenType(
+          token.type,
+        )} here: state ${this.statesStack.peek()} reduces on it instead of shifting it`,
+      };
+
+    this.pushState(action.value);
+    this.symbolsStack.push({ type: "token", token });
+    return undefined;
+  }
+
+  private syntaxErrorMessage(state: TableState, token: Token<TokenType, Metadata>) {
+    const expected = state.getTerminalKeys();
+    const got = this.options.toStringifiedTokenType(token.type);
+    const lexeme = token.lexeme === "" ? "" : ` ("${token.lexeme}")`;
+    return (
+      `Invalid syntax at ${token.line}:${token.column}: got ${got}${lexeme}, but expected ${
+        expected.length === 0 ? "the end of the input" : `one of [${expected.join("], [")}]`
+      }`
+    );
+  }
 }
 
 export class LR1ParserGraveError<TokenType, Metadata> extends Error {
   constructor(
     public readonly reason: string,
-    public readonly currentToken: Token<TokenType, Metadata>,
+    public readonly currentToken: Token<TokenType, Metadata> | null,
     err?: Error,
   ) {
-    super(reason);
+    super(reason, err === undefined ? undefined : { cause: err });
+    this.name = "LR1ParserGraveError";
   }
 }
+
+/** A grammar (or table) problem found before parsing started, always tied to a place in the file. */
+export class SparseGrammarError extends Error {
+  constructor(
+    public readonly reason: string,
+    public readonly token: GrammarToken,
+  ) {
+    super(`Encountered error "${reason}" at ${token.line}:${token.column}`);
+    this.name = "SparseGrammarError";
+  }
+}
+
+const truncate = (value: string, max: number) =>
+  value.length <= max ? value : `${value.slice(0, max)}... (${value.length} characters)`;
+
+const describeProduction = (production: Production) =>
+  production.name == null ? ` (${production.identifier})` : ` ("${production.name}")`;