@scinorandex/sparse 0.1.1 → 0.1.2

package/example/kleene-test/example.ts

@@ -1,6 +1,13 @@
 import { Slex, Token as SlexToken } from "@scinorandex/slex";
-import { buildProductions, LR1StackSymbol, Sparse } from "../../src/index";
-import fs from "fs/promises";
+import {
+  LR1StackSymbol,
+  Sparse,
+  assertReducersCoverGrammar,
+  defineReducers,
+  enumToString,
+  loadGrammar,
+} from "../../src/index";
+import path from "path";
 
 // prettier-ignore
 enum TokenType {
@@ -52,32 +59,43 @@ class KleeneNode<T> extends Node {
   }
 }
 
-type Reducer = (bag: any, input: LR1StackSymbol<TokenType, Metadata, Node>[]) => Node;
-const reducers: { [key: string]: Reducer } = {
-  "autogenerated-kleene": (bag, input) => {
+// Every reducer receives (newInput, oldInput): newInput is { bag, name } (the bag holds the named
+// symbols of the right hand side), oldInput is { input, index } (the raw symbols in order).
+type Reducer = (
+  newInput: { bag: any; name: string | null },
+  oldInput: { input: LR1StackSymbol<TokenType, Metadata, Node>[]; index: number },
+) => Node;
+
+const reducers: Record<string, Reducer> = {
+  // Every "*" and "+" group produces a list production named "autogenerated-kleene", which has to
+  // flatten the list it is handed: the first reduction has no "rest", later ones do.
+  "autogenerated-kleene": ({ bag }, { input }) => {
     const { rest, ...otherNodes } = bag;
     if (rest == null) return new KleeneNode(input, otherNodes);
     else if (rest instanceof KleeneNode) return rest.add(input, otherNodes);
-    // You should not be able to reach this because this function always returns a KleeneNode
     else throw new Error("Invalid reducer");
   },
 
-  program: (bag: { first_number: Node; rest: KleeneNode<{ op: Token; extra_number: Token }> }, input) => {
-    return new Node(input);
-  },
+  program: (_, { input }) => new Node(input),
 };
 
+const root = path.join(__dirname, "..", "..");
+
 async function main() {
-  const toStringifiedTokenType = (type: TokenType) => TokenType[type];
-  const productions = buildProductions(await fs.readFile("./example/kleene-test/grammar.txt", "utf8"));
+  const productionsResult = await loadGrammar(path.join(root, "example/kleene-test/grammar.txt"));
+  if (productionsResult.success === false) throw new Error(productionsResult.reason);
+
+  // Fails loudly here if a production in the grammar has no reducer, instead of halfway through a parse.
+  assertReducersCoverGrammar(productionsResult.value, reducers);
+
+  // No table file for this example, so the states are generated from the grammar at startup.
+  const parserGenerator = Sparse.fromProductions<TokenType, Metadata, Node>({
+    productions: productionsResult.value,
+    toStringifiedTokenType: enumToString(TokenType),
+  });
 
-  const parserGenerator = Sparse.fromProductions<TokenType, Metadata, Node>({ productions, toStringifiedTokenType });
   const parser = parserGenerator.generate(lexer, {
-    reducer: ({ bag, name }, { input }) => {
-      const reducer = reducers[name ?? ""];
-      if (reducer == null) throw new Error(`No reducer for ${name}`);
-      return reducer(bag, input);
-    },
+    reducer: defineReducers(reducers),
   });
 
   console.log(parser.parse().result!);

package/example/math/example.ts

@@ -1,6 +1,6 @@
 import { Slex } from "@scinorandex/slex";
-import { buildProductions, buildStates, LR1StackSymbol, Sparse } from "../../src/index";
-import fs from "fs/promises";
+import { LR1StackSymbol, Sparse, enumToString } from "../../src/index";
+import path from "path";
 
 // prettier-ignore
 enum TokenType {
@@ -33,14 +33,19 @@ class Node {
   }
 }
 
+const root = path.join(__dirname, "..", "..");
+
 async function main() {
-  const toStringifiedTokenType = (type: TokenType) => TokenType[type];
-  const productions = buildProductions(await fs.readFile("./example/math/grammar.txt", "utf8"));
-  const states = buildStates(await fs.readFile("./example/math/table.txt", "utf8"));
-  const parserGenerator = new Sparse<TokenType, Metadata, Node>({ productions, states, toStringifiedTokenType });
+  // fromGrammarFile reads the grammar, reads the prebuilt table, and checks that the two match.
+  const parserGenerator = await Sparse.fromGrammarFile<TokenType, Metadata, Node>({
+    grammarPath: path.join(root, "example/math/grammar.txt"),
+    tablePath: path.join(root, "example/math/table.txt"),
+    toStringifiedTokenType: enumToString(TokenType),
+    onWarning: ({ reason, token }) => console.warn(`${token.line}:${token.column} ${reason}`),
+  });
 
   const parser = parserGenerator.generate(lexer, { reducer: (_, { input }) => new Node(input) });
   console.log(parser.parse().result!.toObject());
 }
 
-main();
+main();

package/example/selfhosted/example.ts

@@ -1,39 +1,45 @@
 import fs from "fs/promises";
-import { generateStates } from "../../src";
-import { buildProductions, grammarLexerGenerator } from "../../src/meta/selfhosted";
+import path from "path";
+import { buildErrorWindow, generateStates, tryBuildProductions } from "../../src";
+import { grammarLexerGenerator } from "../../src/meta/selfhosted";
 import { dehydateProduction } from "../../src/meta/common";
 
-async function codegenMetaGrammar() {
-  const grammar = await fs.readFile("./example/selfhosted/01-meta-grammar.txt", "utf8");
+const root = path.join(__dirname, "..", "..");
 
-  const lexer = grammarLexerGenerator.generate(grammar, () => ({}));
-  const productions = buildProductions(lexer);
-  const statesResult = generateStates(productions);
-  if (statesResult.success === false) throw new Error(statesResult.reason);
-
-  await fs.writeFile(
-    "./example/selfhosted/codegen-meta.ts",
-    `export const states = ${JSON.stringify(statesResult.value.toJSObject(), null, 2)};
-    export const productions = ${JSON.stringify(productions.map(dehydateProduction), null, 2)};
-    export const selfhosted = { states, productions };`,
-  );
-}
-
-async function codegenTableGrammar() {
-  const grammar = await fs.readFile("./example/selfhosted/02-table-grammar.txt", "utf8");
+/**
+ * Regenerates the LR(1) states that Sparse uses to read its own grammar and table file formats.
+ * `src/meta/states.ts` and `src/table/states.ts` are byte-identical copies of the two files this
+ * writes, so after running this you must copy them over.
+ */
+async function codegen(name: string, grammarName: string, outputPath: string) {
+  const grammarPath = path.join(root, "example/selfhosted", grammarName);
+  const grammar = await fs.readFile(grammarPath, "utf8");
 
   const lexer = grammarLexerGenerator.generate(grammar, () => ({}));
-  const productions = buildProductions(lexer);
+  const productionsResult = tryBuildProductions(lexer);
+  if (productionsResult.success === false) {
+    console.error(`${productionsResult.reason}\n${buildErrorWindow(grammar, productionsResult.token)}`);
+    process.exitCode = 1;
+    return;
+  }
+  const productions = productionsResult.value;
+
   const statesResult = generateStates(productions);
-  if (statesResult.success === false) throw new Error(statesResult.reason);
+  if (statesResult.success === false) {
+    console.error(`${statesResult.reason}\n${buildErrorWindow(grammar, statesResult.token)}`);
+    process.exitCode = 1;
+    return;
+  }
 
   await fs.writeFile(
-    "./example/selfhosted/codegen-table.ts",
+    path.join(root, "example/selfhosted", outputPath),
     `export const states = ${JSON.stringify(statesResult.value.toJSObject(), null, 2)};
     export const productions = ${JSON.stringify(productions.map(dehydateProduction), null, 2)};
     export const selfhosted = { states, productions };`,
   );
+
+  console.log(`${name}: ${productions.length} productions, ${statesResult.value.states.length} states -> ${outputPath}`);
 }
 
-codegenMetaGrammar();
-codegenTableGrammar();
+codegen("meta grammar", "01-meta-grammar.txt", "codegen-meta.ts");
+codegen("table grammar", "02-table-grammar.txt", "codegen-table.ts");

package/opencode.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://opencode.ai/config.json",
-  "enabled_providers": ["llama"],
+  "enabled_providers": ["llama", "openrouter"],
   "provider": {
     "llama": {
       "npm": "@ai-sdk/openai-compatible",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scinorandex/sparse",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Yet another parser generator",
   "main": "dist/index.js",
   "scripts": {
@@ -31,7 +31,7 @@
   "devDependencies": {
     "@types/minimist": "^1.2.5",
     "@types/node": "^26.0.0",
-    "@scinorandex/slex": "^0.0.4",
+    "@scinorandex/slex": "^0.1.2",
     "typescript": "^5.6.0",
     "vitest": "^3.0.0"
   },

package/src/cli.ts

@@ -1,49 +1,158 @@
 #! /usr/bin/env node
 import fs from "fs/promises";
-import { tryBuildProductions } from "./index";
-import { generateStates } from "./generator";
-import minimist from "minimist";
 import path from "path";
-import { buildErrorWindow } from "./utils/errorWindowBuilder";
+import minimist from "minimist";
+import { tryBuildProductions, tryBuildStates, generateStates, validateTable, buildErrorWindow } from "./index";
+import type { GeneratorResult, ProductionWarning } from "./index";
 
 const args = minimist(process.argv.slice(2));
 
-function checkFileExists(filepath: string) {
-  return new Promise<boolean>((resolve, reject) => {
-    fs.access(filepath, fs.constants.F_OK)
-      .then(() => resolve(true))
-      .catch(() => resolve(false));
-  });
+const BINARY_NAME = "@scinorandex/sparse";
+
+const USAGE = `Usage: npx @scinorandex/sparse --input=<grammar> --output=<table> [options]
+
+Generates an LR(1) parsing table from a grammar file.
+
+Required:
+  --input=<file>        Grammar file to read
+  --output=<file>       Where to write the parsing table (created if the directory is missing)
+
+Options:
+  --lalr                Generate LALR(1) states instead of LR(1) (smaller table, ~35% faster parsing)
+  --check               Do not write anything: verify that <output> already matches the grammar
+  --stdout              Print the table instead of writing it to a file
+  --quiet               Do not print warnings about suspicious grammar rules
+  --help                Show this message
+
+Run with npx ${BINARY_NAME} (the package is @scinorandex/sparse).`;
+
+function fail(message: string) {
+  console.error(message);
+  process.exitCode = 1;
+}
+
+/** Serializing can fail for grammars the table file format cannot express; report that, not a stack trace. */
+function toTableOrFail(result: GeneratorResult): string | null {
+  try {
+    return result.toTable();
+  } catch (err) {
+    fail(err instanceof Error ? err.message : String(err));
+    return null;
+  }
 }
 
 async function main() {
-  if (typeof args.input !== "string" || typeof args.output !== "string") {
-    console.log("Usage: npx @scinorandex/sparse --input=<input> --output=<output> [--lalr]");
+  if (args.help === true || args.h === true) {
+    console.log(USAGE);
+    return;
+  }
+
+  if (typeof args.input !== "string") {
+    fail(`Missing --input.\n\n${USAGE}`);
+    return;
+  }
+
+  const useStdout = args.stdout === true;
+  const check = args.check === true;
+  if (typeof args.output !== "string" && !useStdout) {
+    fail(`Missing --output.\n\n${USAGE}`);
     return;
   }
 
   const inputFile = path.resolve(args.input);
-  const outputFile = path.resolve(args.output);
-  console.log(`Reading grammar from ${inputFile} and outputting table to ${outputFile}\n`);
+  const outputFile = typeof args.output === "string" ? path.resolve(args.output) : undefined;
+
+  let grammar: string;
+  try {
+    grammar = await fs.readFile(inputFile, "utf8");
+  } catch {
+    fail(`Cannot read grammar file ${inputFile}: it does not exist.`);
+    return;
+  }
 
-  if (!(await checkFileExists(inputFile))) return console.log(`File ${inputFile} does not exist`);
+  const onWarning = (warning: ProductionWarning) => {
+    if (args.quiet === true) return;
+    console.error(`Warning in ${inputFile} at ${warning.token.line}:${warning.token.column}: ${warning.reason}`);
+  };
 
-  const grammar = await fs.readFile(args.input, "utf8");
-  const productionsResult = tryBuildProductions(grammar);
+  const productionsResult = tryBuildProductions(grammar, { onWarning });
   if (productionsResult.success === false) {
-    console.log(productionsResult.reason);
-    console.log(buildErrorWindow(grammar, productionsResult.token));
+    fail(`${productionsResult.reason}\n\n${buildErrorWindow(grammar, productionsResult.token)}`);
+    return;
+  }
+  const productions = productionsResult.value;
+
+  if (check) {
+    if (outputFile == null) {
+      fail(`--check needs --output to know which table to check.\n\n${USAGE}`);
+      return;
+    }
+
+    let existingTable: string;
+    try {
+      existingTable = await fs.readFile(outputFile, "utf8");
+    } catch {
+      fail(`Cannot check ${outputFile}: it does not exist.`);
+      return;
+    }
+
+    const statesResult = tryBuildStates(existingTable, { productions });
+    if (statesResult.success === false) {
+      fail(`The existing table is not a valid parsing table:\n${statesResult.reason}`);
+      return;
+    }
+
+    const tableCheckResult = validateTable(productions, statesResult.value, { source: outputFile });
+    if (tableCheckResult.success === false)
+      fail(`The existing table does not match the grammar:\n${tableCheckResult.reason}`);
+
+    const expected = generateStates(productions, { mode: args.lalr === true ? "lalr1" : "lr1" });
+    if (expected.success === false) {
+      fail(`${expected.reason}\n\n${buildErrorWindow(grammar, expected.token)}`);
+      return;
+    }
+
+    const expectedTable = toTableOrFail(expected.value);
+    if (expectedTable == null) return;
+    const actualTable = existingTable.trim();
+
+    if (expectedTable !== actualTable) {
+      fail(
+        `${outputFile} is out of date with respect to ${inputFile}.\n` +
+          `Re-run without --check to regenerate it${args.lalr === true ? " (with --lalr)" : ""}.`,
+      );
+      return;
+    }
+
+    console.log(
+      `${outputFile} is up to date with respect to ${inputFile} (${expectedTable.split("\n").length} states).`,
+    );
     return;
   }
 
-  const generatorResult = generateStates(productionsResult.value, { mode: args.lalr === true ? "lalr1" : "lr1" });
+  const generatorResult = generateStates(productions, { mode: args.lalr === true ? "lalr1" : "lr1" });
   if (generatorResult.success === false) {
-    console.log(generatorResult.reason);
-    console.log(buildErrorWindow(grammar, generatorResult.token));
+    fail(`${generatorResult.reason}\n\n${buildErrorWindow(grammar, generatorResult.token)}`);
     return;
   }
 
-  await fs.writeFile(outputFile, generatorResult.value.toTable(), { encoding: "utf-8" });
+  const table = toTableOrFail(generatorResult.value);
+  if (table == null) return;
+
+  if (useStdout) {
+    console.log(table);
+    return;
+  }
+
+  if (outputFile == null) return;
+
+  await fs.mkdir(path.dirname(outputFile), { recursive: true });
+  await fs.writeFile(outputFile, table, { encoding: "utf-8" });
+
+  const stateCount = table === "" ? 0 : table.split("\n").length;
+  console.log(
+    `Wrote ${stateCount} ${args.lalr === true ? "LALR(1)" : "LR(1)"} states from ${inputFile} to ${outputFile}.`,
+  );
 }
 
 main();

package/src/generator.ts

@@ -1,5 +1,5 @@
 import { TableState } from "./parser";
-import { GrammarToken, Production } from "./meta/common";
+import { GrammarToken, Production, ProductionWarning, validateProductions } from "./meta/common";
 import { Result } from "./utils/Result";
 
 const xContainsAllOfY = <T>(xs: Set<T>, ys: Set<T>) => [...ys].every((x) => xs.has(x));
@@ -20,6 +20,13 @@ function computeFirstSets(allProductions: Production[]): Result<Map<string, Set<
       const toBeModified = ret.get(production.identifier)!;
 
       const firstRhsToken = production.rhs[0];
+      if (firstRhsToken == null)
+        return {
+          success: false,
+          reason: `Production "${production.identifier}" has an empty right hand side, which Sparse does not support`,
+          token: production.lhs,
+        };
+
       if (firstRhsToken.type === "variable") {
         const add = ret.get(firstRhsToken.identifier);
 
@@ -127,8 +134,20 @@ const FIELD_SEP = "\x01";
 type Item = Production & { dot: number; lookahead: string[]; prodIndex: number; key: string };
 type State = { itemSet: Item[]; kernel: Item; count: number };
 
-export type GeneratorOptions = { mode?: "lr1" | "lalr1" };
-export const generateStates = (productions: Production[], options: GeneratorOptions = {}): Result<GeneratorResult> => {
+export type GeneratorOptions = {
+  mode?: "lr1" | "lalr1";
+  onWarning?: (warning: ProductionWarning) => void;
+  /** Called every time a new state is discovered, useful to show progress on big grammars. */
+  onProgress?: (statesGenerated: number) => void;
+};
+
+export const generateStates = (
+  productions: Production[],
+  options: GeneratorOptions = {},
+): Result<GeneratorResult> => {
+  const validationResult = validateProductions(productions, { onWarning: options.onWarning });
+  if (validationResult.success === false) return validationResult;
+
   const followSetsResult = computeFollowSets(productions);
   if (followSetsResult.success == false) return followSetsResult;
 
@@ -353,6 +372,7 @@ export const generateStates = (productions: Production[], options: GeneratorOpti
           existingStateIndex = newState.count;
         }
 
+        options.onProgress?.(states.length);
         GotoTable[currentState.count]!.set(gotoTransition, existingStateIndex);
       }
 
@@ -385,6 +405,7 @@ export const generateStates = (productions: Production[], options: GeneratorOpti
           existingStateIndex = newState.count;
         }
 
+        options.onProgress?.(states.length);
         ActionTable[currentState.count]!.set(shiftTransition, { action: "shift", value: existingStateIndex });
       }
     }
@@ -463,7 +484,9 @@ function mergeStatesForLALR(
     const gotos = new Map<string, number>();
 
     for (const member of members) {
-      for (const [key, entry] of ActionTable[member].entries()) {
+      // A state whose items only move to other states (no shifts, no completed items) has no
+      // entry in the action table at all.
+      for (const [key, entry] of (ActionTable[member] ?? new Map()).entries()) {
         const value: Action = {
           action: entry.action,
           value: entry.action === "shift" ? oldToNew[entry.value] : entry.value,
@@ -513,7 +536,15 @@ export class GeneratorResult {
   ) {}
 
   toTable(): string {
-    return this.ActionTable.map((actions, idx) => {
+    const actionless = this.ActionTable.findIndex((actions) => actions === undefined || actions.size === 0);
+    if (actionless !== -1)
+      throw new Error(
+        `State ${actionless} of this grammar has no actions, so it cannot be written in the parsing table format. ` +
+          `Use Sparse.fromProductions to keep the states in memory instead.`,
+      );
+
+    // Array.from, not map: the action table is sparse and map would keep the holes.
+    return Array.from(this.ActionTable, (actions, idx) => {
       return [
         ...[...actions.entries()].map(([k, { action, value }]) => `${k}=${action === "reduce" ? "r" : "s"}${value}`),
         ...(this.GotoTable[idx] === undefined ? [] : [...this.GotoTable[idx].entries()].map(([k, v]) => `${k}=${v}`)),
@@ -522,14 +553,24 @@ export class GeneratorResult {
   }
 
   toStates(): TableState[] {
-    return this.ActionTable.map((actions, idx) => {
+    // Array.from, not map: a state without actions leaves a hole in the action table, and map would
+    // hand back an array that is missing that state and shifts every later state number.
+    return Array.from(this.ActionTable, (actions, idx) => {
       const tableState = new TableState();
 
-      for (const [k, { action, value }] of actions.entries())
+      // See toTable(): a state can legitimately have no actions, it just cannot be serialized.
+      if (actions === undefined) return tableState;
+
+      for (const [k, { action, value }] of actions.entries()) {
         tableState.actions.set(k, { type: action === "reduce" ? "reduce" : "shift", value });
+        tableState.rawActions.set(k, `${action === "reduce" ? "r" : "s"}${value}`);
+      }
 
       if (this.GotoTable[idx] !== undefined)
-        for (const [k, value] of this.GotoTable[idx].entries()) tableState.actions.set(k, { type: "goto", value });
+        for (const [k, value] of this.GotoTable[idx].entries()) {
+          tableState.actions.set(k, { type: "goto", value });
+          tableState.rawActions.set(k, `${value}`);
+        }
 
       return tableState;
     });

package/src/index.ts

@@ -1,14 +1,66 @@
 export { GeneratorResult, generateStates } from "./generator";
 export type { GeneratorOptions } from "./generator";
-export { buildProductions, tryBuildProductions } from "./meta/selfhosted";
+
+export {
+  buildProductions,
+  tryBuildProductions,
+  grammarLexerGenerator,
+  getSelfHostedParserGenerator,
+} from "./meta/selfhosted";
+export type { LexerInterface } from "./meta/selfhosted";
+
 export { Result } from "./utils/Result";
 export {
   ParserRecoveryFunction,
   Sparse,
+  SparseGrammarError,
   LR1ParserGraveError,
   LR1StackSymbol,
+  LR1Parser,
   ParserResult,
   ParserError,
+  TableAction,
+  TableState,
 } from "./parser";
-export { buildStates } from "./table/selfhosted";
-export { Production } from "./meta/common";
+export type { ReducerType, SparseOptions, FromProductionsOptions } from "./parser";
+
+export { buildStates, tryBuildStates } from "./table/selfhosted";
+export type { BuildStatesOptions } from "./table/selfhosted";
+export { validateTable, validateTableStates } from "./table/validate";
+export type { ValidateTableOptions, ValidateTableStatesOptions, TableFormatWarning } from "./table/validate";
+
+export {
+  dehydateProduction,
+  hydrateProduction,
+  isVariableIdentifier,
+  syntheticGrammarToken,
+  terminalIdentifier,
+  validateProductions,
+  variableIdentifier,
+  GrammarTokenType,
+} from "./meta/common";
+export type {
+  GrammarToken,
+  GrammarTokenMetadata,
+  Production,
+  ProductionRhsItem,
+  ProductionWarning,
+  ProductionWarningKind,
+  ValidateProductionsOptions,
+} from "./meta/common";
+
+export {
+  assertReducersCoverGrammar,
+  defineReducers,
+  enumToString,
+  missingReducerMessage,
+  missingReducerNames,
+  namedProductions,
+  KLEENE_REDUCER_NAME,
+} from "./utils/reducers";
+export type { ReducerMap } from "./utils/reducers";
+
+export { loadGrammar, loadTable } from "./utils/loadFiles";
+export type { LoadGrammarOptions, LoadTableOptions } from "./utils/loadFiles";
+export { buildErrorWindow } from "./utils/errorWindowBuilder";
+export { Stack } from "./utils/Stack";