grammatik 0.1.0

package/dist/types/index.d.ts

@@ -0,0 +1,214 @@
+/**
+ * Type declarations for grammatik.
+ *
+ * Hand-written rather than inferred: JSON Logic is `any` by construction, so
+ * generated declarations would say `any` everywhere the interesting shapes are.
+ */
+
+/** A JSON Logic node. Deliberately loose — grammars are open-ended data. */
+export type Logic = any
+
+/** A parser expression: one of the combinators, as JSON Logic. */
+export type Parser = Logic
+
+export interface TokenDef {
+  /** The token type produced. */
+  name: string
+  /** Regex source, matched sticky at the cursor. Mutually exclusive with `literal`. */
+  pattern?: string
+  /** A literal string. Cheaper than a pattern; prefer it. */
+  literal?: string
+  /** Consume but produce no token, e.g. whitespace and comments. */
+  skip?: boolean
+  ignoreCase?: boolean
+  /** Defer to this token when it matches more text, e.g. `if` yielding to an identifier. */
+  longerAlt?: string
+  /** The lexer mode this token belongs to. Defaults to `'default'`. */
+  mode?: string
+  /** Several modes, for a token that serves more than one. */
+  modes?: string[]
+  /** Enter this mode after the token matches. */
+  pushMode?: string
+  /** Leave the current mode after the token matches. */
+  popMode?: boolean
+}
+
+export interface Token {
+  type: string
+  image: string
+  start: number
+  end: number
+  /** 1-based; 0 under `positions: 'offset'`. */
+  line: number
+  /** 1-based; 0 under `positions: 'offset'`. */
+  col: number
+}
+
+export interface GrammarSpec {
+  tokens: TokenDef[]
+  rules: Record<string, Parser>
+  start: string
+  /** Recorded in a generated file's header; also the default `moduleName`. */
+  name?: string
+  /** A .gram file's `methods { ... }` block, as source. */
+  methodsBlock?: string
+}
+
+export type ExecutionMode = 'generated' | 'interpreted'
+export type PositionMode = 'full' | 'offset'
+
+export interface ParserOptions {
+  /** Extra JSON Logic methods available to semantic actions. */
+  methods?: Record<string, any>
+  /**
+   * `generated` builds one JavaScript function per rule and compiles semantic
+   * actions; `interpreted` uses closures and walks the actions, which needs no
+   * `eval` and so runs under a strict Content-Security-Policy.
+   */
+  execution?: ExecutionMode
+  /** `offset` skips scanning the input for newlines. */
+  positions?: PositionMode
+  /** Packrat memoization on (rule, position). */
+  memo?: boolean
+  /** Require the whole token stream to be consumed. */
+  strict?: boolean
+  /** FIRST-set dispatch for `alt`. Off forces ordered backtracking. */
+  ll1?: boolean
+  /** Throw `GrammarError` on left recursion and unknown rule references. */
+  validate?: boolean
+  /** Runaway guard; 0 disables. */
+  maxSteps?: number
+  /** @deprecated Legacy alias: `false` selects `execution: 'interpreted'`. */
+  compile?: boolean
+  /** @deprecated Legacy alias: `false` selects `execution: 'interpreted'`. */
+  unsafeEval?: boolean
+}
+
+export interface Analysis {
+  /** Rule name -> the token types that can begin it. */
+  firsts: Record<string, Set<string>>
+  /** Rule names that can match the empty string. */
+  nullable: Set<string>
+  /** Left recursion, unknown references and malformed nodes. */
+  errors: string[]
+}
+
+export interface Lexer {
+  tokenize (text: string): Token[]
+  tokenNames: string[]
+  positions: PositionMode
+  /** Every mode the token definitions mention, `'default'` first. */
+  modeNames: string[]
+}
+
+export interface JLParser {
+  parse (text: string): any
+  parseTokens (tokens: Token[], source?: string): any
+  tokenize (text: string): Token[]
+  lexer: Lexer
+  engine: any
+  analysis: Analysis
+  execution: ExecutionMode
+  compiled: boolean
+  readonly firsts: Record<string, Set<string>>
+}
+
+export declare function createParser (spec: GrammarSpec, options?: ParserOptions): JLParser
+
+export declare function createLexer (
+  tokens: TokenDef[],
+  options?: { positions?: PositionMode }
+): Lexer
+
+export declare function analyze (
+  rules: Record<string, Parser>,
+  options?: { tokenNames?: string[] }
+): Analysis
+
+export interface EmitOptions extends ParserOptions {
+  /** Module format of the generated file. */
+  format?: 'esm' | 'cjs'
+  /** Name recorded in the file header. Defaults to `spec.name`. */
+  moduleName?: string
+  /** What the file imports the runtime from. Defaults to `grammatik/runtime`. */
+  runtimeSpecifier?: string
+  /** What the file imports the engine from, when it still needs one. */
+  engineSpecifier?: string
+  /** Overrides the block carried on the grammar. */
+  methodsBlock?: string
+}
+
+/**
+ * Writes a grammar out as a JavaScript module: one function per rule, as source.
+ * Supplying `methods` lets semantic actions be compiled to source too, which can
+ * remove the `json-logic-engine` dependency from the output entirely.
+ */
+export declare function emitModule (spec: GrammarSpec, options?: EmitOptions): string
+
+/** Parses the grammar-definition language into a plain grammar object. */
+export declare function parseDefinition (source: string, options?: ParserOptions): GrammarSpec
+export declare function createParserFromSource (source: string, options?: ParserOptions): JLParser
+export declare function createDefinitionParser (options?: ParserOptions): JLParser
+
+export declare const definitionGrammar: GrammarSpec
+export declare const definitionTokens: TokenDef[]
+export declare const definitionRules: Record<string, Parser>
+export declare const CONSTRUCTOR_NAMES: string[]
+
+/** Separates a trailing `methods { ... }` block from the grammar text. */
+export declare function splitMethodsBlock (source: string): { grammar: string, block: string | null }
+
+export declare function analyzeMethodsBlock (body: string): {
+  names: string[]
+  hasDefault: boolean
+  imports: string[]
+}
+
+/** Evaluates a block to its methods table. Needs `new Function`; no imports. */
+export declare function evaluateMethodsBlock (body: string): {
+  methods: Record<string, any>
+  exports: Record<string, any>
+}
+
+/** Every combinator name, e.g. `seq`, `alt`, `infixLeft`. */
+export declare const OPERATORS: string[]
+export declare const PARSER_OPS: Set<string>
+export declare function createMethods (): Record<string, any>
+
+/**
+ * JSON Logic methods available inside semantic actions, beyond the engine's own.
+ * Currently `obj`, which builds an object from flat key/value pairs so a node's
+ * key can come from the input. Registered automatically by `createParser` and
+ * `emitModule`; exported for hosts that build their own engine.
+ */
+export declare function createValueMethods (): Record<string, any>
+export declare function buildDispatch (...args: any[]): any
+export declare function firstOf (...args: any[]): any
+
+/** The sentinel a parser returns when it did not match. */
+export declare const FAIL: symbol
+
+export declare function createState (
+  tokens: Token[],
+  grammar: Record<string, Parser>,
+  options?: { memo?: boolean, maxSteps?: number, track?: boolean }
+): any
+
+export declare class ParseError extends Error {
+  name: 'ParseError'
+  /** The token the parse stopped at, or null at end of input. */
+  token: Token | null
+  /** Token types that would have been accepted there. */
+  expected: string[]
+  /** The rule nesting at the point of failure, outermost first. */
+  ruleStack: string[]
+}
+
+export declare class GrammarError extends Error {
+  name: 'GrammarError'
+  problems: string[]
+}
+
+export declare class LexError extends Error {
+  name: 'LexError'
+}

package/dist/types/runtime.d.ts

@@ -0,0 +1,40 @@
+/**
+ * `grammatik/runtime` — the surface a generated parser imports.
+ *
+ * Deliberately small: a generated file needs the lexer, the parse state and
+ * error construction, and nothing else. The analyzer, planner and code
+ * generator were already run when the file was written.
+ */
+
+import type { Token, TokenDef, PositionMode, Lexer, Parser } from './index.js'
+
+export declare const FAIL: symbol
+
+export declare function createState (
+  tokens: Token[],
+  grammar: Record<string, Parser>,
+  options?: { memo?: boolean, maxSteps?: number, track?: boolean }
+): any
+
+/** Records that `expectation` would have been accepted at the cursor. */
+export declare function expect (state: any, expectation: string): void
+
+export declare function buildParseError (state: any): ParseError
+
+export declare function createLexer (
+  tokens: TokenDef[],
+  options?: { positions?: PositionMode }
+): Lexer
+
+export declare class ParseError extends Error {
+  name: 'ParseError'
+  token: Token | null
+  expected: string[]
+  ruleStack: string[]
+}
+
+export declare class LexError extends Error {
+  name: 'LexError'
+}
+
+export type { Token, TokenDef, PositionMode, Lexer }

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "grammatik",
+  "version": "0.1.0",
+  "description": "A parser toolkit whose grammars are data: tokens, rules and semantic actions all serializable, compiled ahead of time.",
+  "type": "module",
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/cjs/index.js"
+    },
+    "./runtime": {
+      "types": "./dist/types/runtime.d.ts",
+      "import": "./dist/esm/runtime.js",
+      "require": "./dist/cjs/runtime.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "src",
+    "bin",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "node scripts/build.js",
+    "prepublishOnly": "npm run build && npm test && npm run test:csp",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:csp": "node --disallow-code-generation-from-strings scripts/check-no-eval.mjs",
+    "generate": "node bin/grammatik.js generate",
+    "bench": "node bench/bench.js",
+    "bench:compare": "node bench/compare-codex.js",
+    "bench:formula": "node bench/peggy/build.js && node bench/formula.js",
+    "bench:generated": "node bench/peggy/build.js && node bench/generate-formula.js && node bench/generated.js",
+    "bench:json-parsers": "node bench/jsonParsers/run.js"
+  },
+  "keywords": [
+    "parser",
+    "grammar",
+    "lexer",
+    "parser-generator",
+    "json-logic",
+    "chevrotain",
+    "peg"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "json-logic-engine": "^5.0.7"
+  },
+  "devDependencies": {
+    "chevrotain": "^13.2.0",
+    "esbuild": "^0.28.2",
+    "peggy": "^5.1.0",
+    "vitest": "^3.2.4"
+  },
+  "bin": {
+    "grammatik": "./bin/grammatik.js"
+  }
+}

package/src/analyze.d.ts

@@ -0,0 +1,78 @@
+/** Returns the single operator key of a JSON Logic node, or null. */
+export function opOf(node: any): string;
+/**
+ * Rewrites a `seq` into a cheaper equivalent combinator when it matches a known
+ * idiom, in the spirit of json-logic-engine's own `checkIdioms`.
+ *
+ * Grammars are usually written with the general primitives, because that is
+ * what a grammar author reaches for. Two of those spellings are both very
+ * common and needlessly expensive, and both can be recognized mechanically:
+ *
+ *   seq(open, label(n, body), close, action({val: n}))  ->  between(open, body, close)
+ *   seq(label(n, parser), action(A))                    ->  as(parser, A')
+ *
+ * Together they were 40% of all `seq` invocations in the formula benchmark.
+ * Recognition means an existing grammar gets the faster path without being
+ * rewritten by hand.
+ *
+ * Returns the replacement node, or null when nothing matched.
+ *
+ * @param {any[]} children
+ */
+export function recognizeSeqIdiom(children: any[]): {
+    between: {
+        open: any;
+        body: any;
+        close: any;
+    };
+    as?: undefined;
+} | {
+    as: any[];
+    between?: undefined;
+};
+/**
+ * @param {Record<string, any>} grammar
+ * @param {{ tokenNames?: string[] }} [options]
+ * @returns {{ firsts: Map<string, First>, errors: string[], leftRecursive: string[] }}
+ */
+export function analyze(grammar: Record<string, any>, options?: {
+    tokenNames?: string[];
+}): {
+    firsts: Map<string, First>;
+    errors: string[];
+    leftRecursive: string[];
+};
+/**
+ * Builds an O(1) dispatch table for an `alt`, or null when the branches are not
+ * LL(1)-separable and we must fall back to ordered backtracking.
+ *
+ * @param {any[]} branches
+ * @param {Map<string, First>} firsts
+ * @returns {Map<string, any>|null}
+ */
+export function buildDispatch(branches: any[], firsts: Map<string, First>): Map<string, any> | null;
+/** The set of node shapes this module understands as *parsers*. */
+export const PARSER_OPS: Set<string>;
+export type First = {
+    /**
+     * Token types that can begin this node.
+     */
+    tokens: Set<string>;
+    /**
+     * Whether the node can match while consuming nothing.
+     */
+    nullable: boolean;
+    /**
+     * Whether analysis could not fully determine the set.
+     * Any `unknown` disables LL(1) dispatch for the enclosing
+     * `alt`, which is a correctness guard, not a bug.
+     */
+    unknown: boolean;
+};
+/**
+ * Computes the FIRST set of a node given current per-rule estimates.
+ * @param {*} node
+ * @param {Map<string, First>} ruleFirsts
+ * @returns {First}
+ */
+export function firstOf(node: any, ruleFirsts: Map<string, First>): First;