@scinorandex/sparse 0.0.8 → 0.1.2

package/example/kleene-test/example.ts

@@ -1,6 +1,13 @@
 import { Slex, Token as SlexToken } from "@scinorandex/slex";
-import { buildProductions, LR1StackSymbol, Sparse } from "../../src/index";
-import fs from "fs/promises";
+import {
+  LR1StackSymbol,
+  Sparse,
+  assertReducersCoverGrammar,
+  defineReducers,
+  enumToString,
+  loadGrammar,
+} from "../../src/index";
+import path from "path";
 
 // prettier-ignore
 enum TokenType {
@@ -52,32 +59,43 @@ class KleeneNode<T> extends Node {
   }
 }
 
-type Reducer = (bag: any, input: LR1StackSymbol<TokenType, Metadata, Node>[]) => Node;
-const reducers: { [key: string]: Reducer } = {
-  "autogenerated-kleene": (bag, input) => {
+// Every reducer receives (newInput, oldInput): newInput is { bag, name } (the bag holds the named
+// symbols of the right hand side), oldInput is { input, index } (the raw symbols in order).
+type Reducer = (
+  newInput: { bag: any; name: string | null },
+  oldInput: { input: LR1StackSymbol<TokenType, Metadata, Node>[]; index: number },
+) => Node;
+
+const reducers: Record<string, Reducer> = {
+  // Every "*" and "+" group produces a list production named "autogenerated-kleene", which has to
+  // flatten the list it is handed: the first reduction has no "rest", later ones do.
+  "autogenerated-kleene": ({ bag }, { input }) => {
     const { rest, ...otherNodes } = bag;
     if (rest == null) return new KleeneNode(input, otherNodes);
     else if (rest instanceof KleeneNode) return rest.add(input, otherNodes);
-    // You should not be able to reach this because this function always returns a KleeneNode
     else throw new Error("Invalid reducer");
   },
 
-  program: (bag: { first_number: Node; rest: KleeneNode<{ op: Token; extra_number: Token }> }, input) => {
-    return new Node(input);
-  },
+  program: (_, { input }) => new Node(input),
 };
 
+const root = path.join(__dirname, "..", "..");
+
 async function main() {
-  const toStringifiedTokenType = (type: TokenType) => TokenType[type];
-  const productions = buildProductions(await fs.readFile("./example/kleene-test/grammar.txt", "utf8"));
+  const productionsResult = await loadGrammar(path.join(root, "example/kleene-test/grammar.txt"));
+  if (productionsResult.success === false) throw new Error(productionsResult.reason);
+
+  // Fails loudly here if a production in the grammar has no reducer, instead of halfway through a parse.
+  assertReducersCoverGrammar(productionsResult.value, reducers);
+
+  // No table file for this example, so the states are generated from the grammar at startup.
+  const parserGenerator = Sparse.fromProductions<TokenType, Metadata, Node>({
+    productions: productionsResult.value,
+    toStringifiedTokenType: enumToString(TokenType),
+  });
 
-  const parserGenerator = Sparse.fromProductions<TokenType, Metadata, Node>({ productions, toStringifiedTokenType });
   const parser = parserGenerator.generate(lexer, {
-    reducer: ({ bag, name }, { input }) => {
-      const reducer = reducers[name ?? ""];
-      if (reducer == null) throw new Error(`No reducer for ${name}`);
-      return reducer(bag, input);
-    },
+    reducer: defineReducers(reducers),
   });
 
   console.log(parser.parse().result!);

package/example/math/example.ts

@@ -1,6 +1,6 @@
 import { Slex } from "@scinorandex/slex";
-import { buildProductions, buildStates, LR1StackSymbol, Sparse } from "../../src/index";
-import fs from "fs/promises";
+import { LR1StackSymbol, Sparse, enumToString } from "../../src/index";
+import path from "path";
 
 // prettier-ignore
 enum TokenType {
@@ -33,14 +33,19 @@ class Node {
   }
 }
 
+const root = path.join(__dirname, "..", "..");
+
 async function main() {
-  const toStringifiedTokenType = (type: TokenType) => TokenType[type];
-  const productions = buildProductions(await fs.readFile("./example/math/grammar.txt", "utf8"));
-  const states = buildStates(await fs.readFile("./example/math/table.txt", "utf8"));
-  const parserGenerator = new Sparse<TokenType, Metadata, Node>({ productions, states, toStringifiedTokenType });
+  // fromGrammarFile reads the grammar, reads the prebuilt table, and checks that the two match.
+  const parserGenerator = await Sparse.fromGrammarFile<TokenType, Metadata, Node>({
+    grammarPath: path.join(root, "example/math/grammar.txt"),
+    tablePath: path.join(root, "example/math/table.txt"),
+    toStringifiedTokenType: enumToString(TokenType),
+    onWarning: ({ reason, token }) => console.warn(`${token.line}:${token.column} ${reason}`),
+  });
 
   const parser = parserGenerator.generate(lexer, { reducer: (_, { input }) => new Node(input) });
   console.log(parser.parse().result!.toObject());
 }
 
-main();
+main();

package/example/selfhosted/example.ts

@@ -1,39 +1,45 @@
 import fs from "fs/promises";
-import { generateStates } from "../../src";
-import { buildProductions, grammarLexerGenerator } from "../../src/meta/selfhosted";
+import path from "path";
+import { buildErrorWindow, generateStates, tryBuildProductions } from "../../src";
+import { grammarLexerGenerator } from "../../src/meta/selfhosted";
 import { dehydateProduction } from "../../src/meta/common";
 
-async function codegenMetaGrammar() {
-  const grammar = await fs.readFile("./example/selfhosted/01-meta-grammar.txt", "utf8");
+const root = path.join(__dirname, "..", "..");
 
-  const lexer = grammarLexerGenerator.generate(grammar, () => ({}));
-  const productions = buildProductions(lexer);
-  const statesResult = generateStates(productions);
-  if (statesResult.success === false) throw new Error(statesResult.reason);
-
-  await fs.writeFile(
-    "./example/selfhosted/codegen-meta.ts",
-    `export const states = ${JSON.stringify(statesResult.value.toJSObject(), null, 2)};
-    export const productions = ${JSON.stringify(productions.map(dehydateProduction), null, 2)};
-    export const selfhosted = { states, productions };`,
-  );
-}
-
-async function codegenTableGrammar() {
-  const grammar = await fs.readFile("./example/selfhosted/02-table-grammar.txt", "utf8");
+/**
+ * Regenerates the LR(1) states that Sparse uses to read its own grammar and table file formats.
+ * `src/meta/states.ts` and `src/table/states.ts` are byte-identical copies of the two files this
+ * writes, so after running this you must copy them over.
+ */
+async function codegen(name: string, grammarName: string, outputPath: string) {
+  const grammarPath = path.join(root, "example/selfhosted", grammarName);
+  const grammar = await fs.readFile(grammarPath, "utf8");
 
   const lexer = grammarLexerGenerator.generate(grammar, () => ({}));
-  const productions = buildProductions(lexer);
+  const productionsResult = tryBuildProductions(lexer);
+  if (productionsResult.success === false) {
+    console.error(`${productionsResult.reason}\n${buildErrorWindow(grammar, productionsResult.token)}`);
+    process.exitCode = 1;
+    return;
+  }
+  const productions = productionsResult.value;
+
   const statesResult = generateStates(productions);
-  if (statesResult.success === false) throw new Error(statesResult.reason);
+  if (statesResult.success === false) {
+    console.error(`${statesResult.reason}\n${buildErrorWindow(grammar, statesResult.token)}`);
+    process.exitCode = 1;
+    return;
+  }
 
   await fs.writeFile(
-    "./example/selfhosted/codegen-table.ts",
+    path.join(root, "example/selfhosted", outputPath),
     `export const states = ${JSON.stringify(statesResult.value.toJSObject(), null, 2)};
     export const productions = ${JSON.stringify(productions.map(dehydateProduction), null, 2)};
     export const selfhosted = { states, productions };`,
   );
+
+  console.log(`${name}: ${productions.length} productions, ${statesResult.value.states.length} states -> ${outputPath}`);
 }
 
-codegenMetaGrammar();
-codegenTableGrammar();
+codegen("meta grammar", "01-meta-grammar.txt", "codegen-meta.ts");
+codegen("table grammar", "02-table-grammar.txt", "codegen-table.ts");

package/opencode.json

@@ -0,0 +1,18 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "enabled_providers": ["llama", "openrouter"],
+  "provider": {
+    "llama": {
+      "npm": "@ai-sdk/openai-compatible",
+      "name": "llama.cpp",
+      "options": { "baseURL": "http://192.168.0.78:8080/v1" },
+      "models": {
+        "your-model-id": {
+          "id": "Qwen3.8-27B-GGUF:UD-IQ3_XXS",
+          "name": "Qwen3.8-27B-GGUF:UD-IQ3_XXS",
+          "limit": { "context": 163840, "output": 8192 }
+        }
+      }
+    }
+  }
+}

package/package.json

@@ -1,11 +1,15 @@
 {
   "name": "@scinorandex/sparse",
-  "version": "0.0.8",
+  "version": "0.1.2",
   "description": "Yet another parser generator",
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
-    "watch": "tsc -w"
+    "watch": "tsc -w",
+    "tc": "tsc --noEmit -p tsconfig.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "bench": "vitest bench"
   },
   "bin": {
     "sparse": "dist/cli.js"
@@ -27,7 +31,9 @@
   "devDependencies": {
     "@types/minimist": "^1.2.5",
     "@types/node": "^26.0.0",
-    "@scinorandex/slex": "^0.0.4"
+    "@scinorandex/slex": "^0.1.2",
+    "typescript": "^5.6.0",
+    "vitest": "^3.0.0"
   },
   "peerDependencies": {
     "@scinorandex/slex": "*"

package/src/cli.ts

@@ -1,49 +1,158 @@
 #! /usr/bin/env node
 import fs from "fs/promises";
-import { tryBuildProductions } from "./index";
-import { generateStates } from "./generator";
-import minimist from "minimist";
 import path from "path";
-import { buildErrorWindow } from "./utils/errorWindowBuilder";
+import minimist from "minimist";
+import { tryBuildProductions, tryBuildStates, generateStates, validateTable, buildErrorWindow } from "./index";
+import type { GeneratorResult, ProductionWarning } from "./index";
 
 const args = minimist(process.argv.slice(2));
 
-function checkFileExists(filepath: string) {
-  return new Promise<boolean>((resolve, reject) => {
-    fs.access(filepath, fs.constants.F_OK)
-      .then(() => resolve(true))
-      .catch(() => resolve(false));
-  });
+const BINARY_NAME = "@scinorandex/sparse";
+
+const USAGE = `Usage: npx @scinorandex/sparse --input=<grammar> --output=<table> [options]
+
+Generates an LR(1) parsing table from a grammar file.
+
+Required:
+  --input=<file>        Grammar file to read
+  --output=<file>       Where to write the parsing table (created if the directory is missing)
+
+Options:
+  --lalr                Generate LALR(1) states instead of LR(1) (smaller table, ~35% faster parsing)
+  --check               Do not write anything: verify that <output> already matches the grammar
+  --stdout              Print the table instead of writing it to a file
+  --quiet               Do not print warnings about suspicious grammar rules
+  --help                Show this message
+
+Run with npx ${BINARY_NAME} (the package is @scinorandex/sparse).`;
+
+function fail(message: string) {
+  console.error(message);
+  process.exitCode = 1;
+}
+
+/** Serializing can fail for grammars the table file format cannot express; report that, not a stack trace. */
+function toTableOrFail(result: GeneratorResult): string | null {
+  try {
+    return result.toTable();
+  } catch (err) {
+    fail(err instanceof Error ? err.message : String(err));
+    return null;
+  }
 }
 
 async function main() {
-  if (typeof args.input !== "string" || typeof args.output !== "string") {
-    console.log("Usage: npx @scinorandex/sparse --input=<input> --output=<output>");
+  if (args.help === true || args.h === true) {
+    console.log(USAGE);
+    return;
+  }
+
+  if (typeof args.input !== "string") {
+    fail(`Missing --input.\n\n${USAGE}`);
+    return;
+  }
+
+  const useStdout = args.stdout === true;
+  const check = args.check === true;
+  if (typeof args.output !== "string" && !useStdout) {
+    fail(`Missing --output.\n\n${USAGE}`);
     return;
   }
 
   const inputFile = path.resolve(args.input);
-  const outputFile = path.resolve(args.output);
-  console.log(`Reading grammar from ${inputFile} and outputting table to ${outputFile}\n`);
+  const outputFile = typeof args.output === "string" ? path.resolve(args.output) : undefined;
+
+  let grammar: string;
+  try {
+    grammar = await fs.readFile(inputFile, "utf8");
+  } catch {
+    fail(`Cannot read grammar file ${inputFile}: it does not exist.`);
+    return;
+  }
 
-  if (!(await checkFileExists(inputFile))) return console.log(`File ${inputFile} does not exist`);
+  const onWarning = (warning: ProductionWarning) => {
+    if (args.quiet === true) return;
+    console.error(`Warning in ${inputFile} at ${warning.token.line}:${warning.token.column}: ${warning.reason}`);
+  };
 
-  const grammar = await fs.readFile(args.input, "utf8");
-  const productionsResult = tryBuildProductions(grammar);
+  const productionsResult = tryBuildProductions(grammar, { onWarning });
   if (productionsResult.success === false) {
-    console.log(productionsResult.reason);
-    console.log(buildErrorWindow(grammar, productionsResult.token));
+    fail(`${productionsResult.reason}\n\n${buildErrorWindow(grammar, productionsResult.token)}`);
+    return;
+  }
+  const productions = productionsResult.value;
+
+  if (check) {
+    if (outputFile == null) {
+      fail(`--check needs --output to know which table to check.\n\n${USAGE}`);
+      return;
+    }
+
+    let existingTable: string;
+    try {
+      existingTable = await fs.readFile(outputFile, "utf8");
+    } catch {
+      fail(`Cannot check ${outputFile}: it does not exist.`);
+      return;
+    }
+
+    const statesResult = tryBuildStates(existingTable, { productions });
+    if (statesResult.success === false) {
+      fail(`The existing table is not a valid parsing table:\n${statesResult.reason}`);
+      return;
+    }
+
+    const tableCheckResult = validateTable(productions, statesResult.value, { source: outputFile });
+    if (tableCheckResult.success === false)
+      fail(`The existing table does not match the grammar:\n${tableCheckResult.reason}`);
+
+    const expected = generateStates(productions, { mode: args.lalr === true ? "lalr1" : "lr1" });
+    if (expected.success === false) {
+      fail(`${expected.reason}\n\n${buildErrorWindow(grammar, expected.token)}`);
+      return;
+    }
+
+    const expectedTable = toTableOrFail(expected.value);
+    if (expectedTable == null) return;
+    const actualTable = existingTable.trim();
+
+    if (expectedTable !== actualTable) {
+      fail(
+        `${outputFile} is out of date with respect to ${inputFile}.\n` +
+          `Re-run without --check to regenerate it${args.lalr === true ? " (with --lalr)" : ""}.`,
+      );
+      return;
+    }
+
+    console.log(
+      `${outputFile} is up to date with respect to ${inputFile} (${expectedTable.split("\n").length} states).`,
+    );
     return;
   }
 
-  const generatorResult = generateStates(productionsResult.value);
+  const generatorResult = generateStates(productions, { mode: args.lalr === true ? "lalr1" : "lr1" });
   if (generatorResult.success === false) {
-    console.log(generatorResult.reason);
-    console.log(buildErrorWindow(grammar, generatorResult.token));
+    fail(`${generatorResult.reason}\n\n${buildErrorWindow(grammar, generatorResult.token)}`);
     return;
   }
 
-  await fs.writeFile(outputFile, generatorResult.value.toTable(), { encoding: "utf-8" });
+  const table = toTableOrFail(generatorResult.value);
+  if (table == null) return;
+
+  if (useStdout) {
+    console.log(table);
+    return;
+  }
+
+  if (outputFile == null) return;
+
+  await fs.mkdir(path.dirname(outputFile), { recursive: true });
+  await fs.writeFile(outputFile, table, { encoding: "utf-8" });
+
+  const stateCount = table === "" ? 0 : table.split("\n").length;
+  console.log(
+    `Wrote ${stateCount} ${args.lalr === true ? "LALR(1)" : "LR(1)"} states from ${inputFile} to ${outputFile}.`,
+  );
 }
 
 main();