grammatik 0.1.0 → 0.1.2

package/dist/types/index.d.ts

@@ -70,6 +70,29 @@ export interface ParserOptions {
   positions?: PositionMode
   /** Packrat memoization on (rule, position). */
   memo?: boolean
+  /**
+   * Memoize only the rules that several colliding branches of an `alt` would
+   * otherwise each re-parse (default `true`; ignored when `memo` is on). Avoids
+   * the exponential blow-up of nested alternatives that share a leading rule,
+   * at no cost to a grammar with none. See `memoRules`.
+   */
+  autoMemo?: boolean
+  /**
+   * In generated code, write single-caller operator levels (`infixLeft`,
+   * `infixRight`, `prefix`, `postfix`) into their caller instead of calling them
+   * (default `true` for `createParser`, `false` for `emitModule`). Several levels of an
+   * expression grammar become one function: 2-3x faster on a bare precedence chain, about
+   * 1.02-1.04x on real grammars, at the cost of a bigger file, which is why an emitted
+   * parser leaves it off.
+   */
+  inlineRules?: boolean
+  /**
+   * Recognize common spellings of a grammar (`seq(open, label, close, action)`, a
+   * prefix plus one labelled result, a constant action, an `alt` of bare `consume`s,
+   * a shared leading run of `alt` branches) and run them as the cheaper combinator
+   * they are equivalent to. Default `true`; `false` runs the grammar exactly as written.
+   */
+  idioms?: boolean
   /** Require the whole token stream to be consumed. */
   strict?: boolean
   /** FIRST-set dispatch for `alt`. Off forces ordered backtracking. */
@@ -109,6 +132,8 @@ export interface JLParser {
   engine: any
   analysis: Analysis
   execution: ExecutionMode
+  /** The rules `autoMemo` chose to memoize, in declaration order. */
+  memoRules: string[]
   compiled: boolean
   readonly firsts: Record<string, Set<string>>
 }
@@ -126,6 +151,17 @@ export declare function analyze (
 ): Analysis
 
 export interface EmitOptions extends ParserOptions {
+  /**
+   * How the file lexes. `'generated'` (default) writes the lexer out as source, so the
+   * file imports only the parse state and error class from the runtime: a smaller
+   * bundle (JSON: 24 KB to 10 KB minified), a faster start, and faster lexing.
+   * `'runtime'` keeps the token table and builds the lexer from it when the file loads.
+   */
+  lexer?: 'generated' | 'runtime'
+  /** Scanner substitution for the lexer, as for `createLexer`. Default `true`. */
+  scanners?: boolean
+  /** Fusing runs of regex tokens in the lexer, as for `createLexer`. Default `true`. */
+  fuse?: boolean
   /** Module format of the generated file. */
   format?: 'esm' | 'cjs'
   /** Name recorded in the file header. Defaults to `spec.name`. */

package/dist/types/runtime.d.ts

@@ -1,9 +1,10 @@
 /**
  * `grammatik/runtime` — the surface a generated parser imports.
  *
- * Deliberately small: a generated file needs the lexer, the parse state and
- * error construction, and nothing else. The analyzer, planner and code
- * generator were already run when the file was written.
+ * Deliberately small: a generated file needs the parse state and error
+ * construction, and nothing else. The analyzer, planner, code generator and
+ * (by default) the lexer were already run when the file was written. `createLexer`
+ * is still here for files emitted with `lexer: 'runtime'`.
  */
 
 import type { Token, TokenDef, PositionMode, Lexer, Parser } from './index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grammatik",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "A parser toolkit whose grammars are data: tokens, rules and semantic actions all serializable, compiled ahead of time.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -38,7 +38,8 @@
     "bench:compare": "node bench/compare-codex.js",
     "bench:formula": "node bench/peggy/build.js && node bench/formula.js",
     "bench:generated": "node bench/peggy/build.js && node bench/generate-formula.js && node bench/generated.js",
-    "bench:json-parsers": "node bench/jsonParsers/run.js"
+    "bench:json-parsers": "node bench/jsonParsers/run.js",
+    "build:site": "node site/build.js"
   },
   "keywords": [
     "parser",

package/src/analyze.js

@@ -60,9 +60,9 @@ export function opOf (node) {
 
 /** The set of node shapes this module understands as *parsers*. */
 export const PARSER_OPS = new Set([
-  'consume', 'seq', 'alt', 'many', 'many1', 'option',
+  'consume', 'seq', 'alt', 'many', 'many1', 'option', 'flag',
   'manySep', 'many1Sep', 'infixLeft', 'infixRight', 'postfix', 'prefix',
-  'between', 'as', 'oneOf', 'eof',
+  'between', 'pick', 'yields', 'as', 'oneOf', 'eof',
   'subrule', 'label', 'action',
   'lookahead', 'negLookahead', 'text', 'epsilon'
 ])
@@ -122,7 +122,8 @@ function firstOf (node, ruleFirsts) {
     }
 
     case 'many':
-    case 'option': {
+    case 'option':
+    case 'flag': {
       const inner = firstOf(arg, ruleFirsts)
       return { tokens: inner.tokens, nullable: true, unknown: inner.unknown }
     }
@@ -153,6 +154,11 @@ function firstOf (node, ruleFirsts) {
       // Exactly a sequence of open, body, close.
       return firstOf({ seq: [arg && arg.open, arg && arg.body, arg && arg.close] }, ruleFirsts)
 
+    case 'pick':
+    case 'yields':
+      // Exactly a sequence of its parsers.
+      return firstOf({ seq: arg && Array.isArray(arg.parsers) ? arg.parsers : [] }, ruleFirsts)
+
     case 'prefix': {
       // Zero operators are allowed, so the operand's first set counts too.
       const ops = firstOf(arg && arg.operator, ruleFirsts)
@@ -254,15 +260,169 @@ export function recognizeSeqIdiom (children) {
     }
   }
 
+  // The two idioms below drop the `seq`'s bindings object, so they decline when a
+  // child could reach into an enclosing scope: without that object an outward hop
+  // would land one scope further out than it did.
+  if (children.length >= 2) {
+    const last = children[children.length - 1]
+    const head = children.slice(0, -1)
+
+    // yields: parsers, then an action that is a constant — `seq(consume(True), action(true))`
+    if (opOf(last) === 'action' && isConstant(last.action) && head.every(isPlainParser) && !head.some(usesOuterScope)) {
+      return { yields: { parsers: head, value: last.action } }
+    }
+
+    // pick: any number of unlabelled parsers around exactly one labelled one, then
+    // an action that returns just that label — `seq(consume(Where), label(e, P), action({val: e}))`.
+    // `between` is the special case with one on each side.
+    if (opOf(last) === 'action' && opOf(last.action) === 'val' && typeof last.action.val === 'string') {
+      const name = last.action.val
+      const labelled = head.map((child, i) => (opOf(child) === 'label' ? i : -1)).filter((i) => i !== -1)
+      if (labelled.length === 1 && head[labelled[0]].label[0] === name &&
+          head.every((child, i) => i === labelled[0] || isPlainParser(child)) &&
+          !head.some(usesOuterScope)) {
+        return { pick: { parsers: head.map((child, i) => (i === labelled[0] ? child.label[1] : child)), index: labelled[0] } }
+      }
+    }
+  }
+
   return null
 }
 
+/** A value that cannot be mutated or aliased: safe to hand out from every parse. */
+const isConstant = (value) =>
+  value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
+
+/**
+ * True when the logic reads from an outer scope (`{val: [[-1], ...]}` or
+ * `{var: "../x"}`). The engine's compiler does not thread `above` into built
+ * functions, so such actions must stay interpreted — and a rewrite that removes
+ * a `seq`'s bindings object would change what such a hop reaches.
+ */
+export function usesOuterScope (logic) {
+  if (Array.isArray(logic)) return logic.some(usesOuterScope)
+  if (!logic || typeof logic !== 'object') return false
+
+  for (const key of Object.keys(logic)) {
+    const arg = logic[key]
+    if (key === 'val' && Array.isArray(arg) && Array.isArray(arg[0])) return true
+    if (key === 'var') {
+      if (typeof arg === 'string' && arg.includes('../')) return true
+      if (Array.isArray(arg) && typeof arg[0] === 'string' && arg[0].includes('../')) return true
+    }
+    if (usesOuterScope(arg)) return true
+  }
+  return false
+}
+
 /** A node that is a parser and not one of `seq`'s own interpreted shapes. */
 function isPlainParser (node) {
   const op = opOf(node)
   return op !== null && op !== 'label' && op !== 'action'
 }
 
+/** Structural equality over plain JSON — every shape a grammar node can be. */
+function deepEqual (a, b) {
+  if (a === b) return true
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false
+    for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false
+    return true
+  }
+  if (a === null || b === null || typeof a !== 'object' || typeof b !== 'object') return false
+  const keysA = Object.keys(a)
+  const keysB = Object.keys(b)
+  if (keysA.length !== keysB.length) return false
+  for (const key of keysA) {
+    if (!Object.prototype.hasOwnProperty.call(b, key) || !deepEqual(a[key], b[key])) return false
+  }
+  return true
+}
+
+/**
+ * Left-factors an `alt` whose branches are all `seq` and share a literal
+ * leading run of nodes — `alt(seq(P, X, actionX), seq(P, Y, actionY))`
+ * becomes `seq(P, alt(seq(X, actionX), seq(Y, actionY)))`, so a shared,
+ * possibly multi-token prefix is matched once per attempt instead of once per
+ * colliding branch.
+ *
+ * Two things have to hold for the rewrite to be safe, not just structurally
+ * possible:
+ *
+ *  - The shared prefix cannot contain a `label` or an `action`. A `seq`'s
+ *    bindings are scoped to itself; hoisting a labelled node out from under
+ *    the branch that reads it would strand the binding in a different scope
+ *    than the action that expects to see it.
+ *  - Every branch must already end its own decision with an `action`
+ *    somewhere in its remainder (`seq` yields that action's value instead of
+ *    the flat array of its children). That is what lets the rewritten form
+ *    reproduce the original value exactly, by wrapping the factored `alt` in
+ *    one more `label`/`action` pair rather than redesigning what the branch
+ *    returns.
+ *
+ * A branch that is a strict prefix of another (nothing left after the shared
+ * run) fails the second check on its own — its remainder is empty, so it has
+ * no action left to find — and the whole rewrite backs off rather than guess
+ * what an author meant by writing one alternative as a prefix of another.
+ *
+ * @param {any[]} branches
+ * @returns {any|null}
+ */
+export function recognizeAltPrefixIdiom (branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null
+  if (!branches.every((b) => opOf(b) === 'seq' && Array.isArray(b.seq))) return null
+
+  const seqs = branches.map((b) => b.seq)
+  const minLen = Math.min(...seqs.map((s) => s.length))
+
+  let n = 0
+  while (n < minLen) {
+    const candidate = seqs[0][n]
+    const op = opOf(candidate)
+    if (op === 'label' || op === 'action') break
+    if (!seqs.every((s) => deepEqual(s[n], candidate))) break
+    n++
+  }
+  if (n === 0) return null
+  if (!seqs.every((s) => s.slice(n).some((child) => opOf(child) === 'action'))) return null
+
+  const prefix = seqs[0].slice(0, n)
+  const remainders = seqs.map((s) => ({ seq: s.slice(n) }))
+  return {
+    seq: [
+      ...prefix,
+      { label: ['__grammatik_rest', { alt: remainders }] },
+      { action: { val: '__grammatik_rest' } }
+    ]
+  }
+}
+
+/**
+ * Rewrites an `alt` whose every branch is a bare `consume` into `oneOf`:
+ * `alt(consume(Plus), consume(Minus))` is `oneOf(Plus, Minus)`.
+ *
+ * Same language, same value (the matched token), same expected-token list on
+ * failure — a token has one type, so "first branch that matches" and "any of
+ * these" cannot differ. What differs is cost: `alt` builds a branch per token,
+ * `oneOf` is a single test. Measured at 1.3–1.8x for the 2–7 token operator sets
+ * grammars actually have, and still 1.2x at 24.
+ *
+ * @param {any[]} branches
+ * @returns {{ oneOf: string[] } | null}
+ */
+export function recognizeAltConsumeIdiom (branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null
+  const names = []
+  for (const branch of branches) {
+    if (opOf(branch) !== 'consume') return null
+    const arg = branch.consume
+    const name = typeof arg === 'string' ? arg : (Array.isArray(arg) && arg.length === 1 ? arg[0] : null)
+    if (typeof name !== 'string') return null
+    names.push(name)
+  }
+  return { oneOf: names }
+}
+
 const BAIL = Symbol('grammatik.bail')
 
 /**
@@ -346,7 +506,7 @@ function computeFirsts (grammar) {
  * @param {Map<string, First>} ruleFirsts
  * @param {Set<string>} out
  */
-function leftmostCalls (node, ruleFirsts, out) {
+export function leftmostCalls (node, ruleFirsts, out) {
   const op = opOf(node)
   if (op === null) return
   const arg = node[op]
@@ -364,6 +524,7 @@ function leftmostCalls (node, ruleFirsts, out) {
     case 'many':
     case 'many1':
     case 'option':
+    case 'flag':
     case 'lookahead':
     case 'negLookahead':
       leftmostCalls(arg, ruleFirsts, out)
@@ -374,6 +535,10 @@ function leftmostCalls (node, ruleFirsts, out) {
     case 'between':
       if (arg) leftmostCalls({ seq: [arg.open, arg.body, arg.close] }, ruleFirsts, out)
       break
+    case 'pick':
+    case 'yields':
+      if (arg && Array.isArray(arg.parsers)) leftmostCalls({ seq: arg.parsers }, ruleFirsts, out)
+      break
     case 'prefix':
       if (arg) {
         leftmostCalls(arg.operator, ruleFirsts, out)
@@ -451,6 +616,7 @@ function validateNode (node, grammar, tokenNames, errors, path) {
     case 'many':
     case 'many1':
     case 'option':
+    case 'flag':
     case 'text':
     case 'lookahead':
     case 'negLookahead':
@@ -472,6 +638,26 @@ function validateNode (node, grammar, tokenNames, errors, path) {
       }
       break
     }
+    case 'pick': {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0) {
+        errors.push(`${path}: 'pick' expects { parsers: [parser, ...], index }`)
+      } else if (!Number.isInteger(arg.index) || arg.index < 0 || arg.index >= arg.parsers.length) {
+        errors.push(`${path}: 'pick' index ${JSON.stringify(arg.index)} is out of range for ${arg.parsers.length} parser(s)`)
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.pick[${i}]`))
+      }
+      break
+    }
+    case 'yields': {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0 || !('value' in arg)) {
+        errors.push(`${path}: 'yields' expects { parsers: [parser, ...], value }`)
+      } else if (!isConstant(arg.value)) {
+        errors.push(`${path}: 'yields' value must be a string, number, boolean or null, got ${JSON.stringify(arg.value)}`)
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.yields[${i}]`))
+      }
+      break
+    }
     case 'prefix': {
       if (!arg || typeof arg !== 'object' || !('operator' in arg) || !('operand' in arg) || !('combine' in arg)) {
         errors.push(`${path}: 'prefix' expects { operator, operand, combine }`)
@@ -573,30 +759,51 @@ export function analyze (grammar, options = {}) {
 }
 
 /**
- * Builds an O(1) dispatch table for an `alt`, or null when the branches are not
- * LL(1)-separable and we must fall back to ordered backtracking.
+ * Builds an O(1) dispatch table for an `alt`, or null when no branch has a
+ * known, non-nullable FIRST set (a nullable/opaque branch could match
+ * anything, so no table is safe at all).
+ *
+ * A token whose FIRST set membership is unique to one branch maps straight to
+ * that branch. A token shared by more than one branch — the choice needs more
+ * than one token of lookahead there — maps to an array of the colliding
+ * branches, in their original `alt` order; a caller resolves that case with
+ * ordered backtracking over just that subset, rather than over every branch.
+ * Equal-content collision groups are the same array instance, so a caller
+ * that dedupes on the table's values (e.g. to share generated code between
+ * case labels) gets that for free.
  *
  * @param {any[]} branches
  * @param {Map<string, First>} firsts
  * @returns {Map<string, any>|null}
  */
 export function buildDispatch (branches, firsts) {
-  /** @type {Map<string, any>} */
-  const table = new Map()
-
-  for (const branch of branches) {
-    const first = firstOf(branch, firsts)
-    // A nullable or opaque branch could match anything, so no table is safe.
+  const perBranch = branches.map((branch) => firstOf(branch, firsts))
+  for (const first of perBranch) {
     if (first.nullable || first.unknown || first.tokens.size === 0) return null
-    for (const token of first.tokens) {
-      // Overlapping FIRST sets mean the choice needs more than one token of
-      // lookahead; ordered backtracking still handles it correctly.
-      if (table.has(token)) return null
-      table.set(token, branch)
+  }
+
+  /** @type {Map<string, any[]>} */
+  const byToken = new Map()
+  for (let i = 0; i < branches.length; i++) {
+    for (const token of perBranch[i].tokens) {
+      let list = byToken.get(token)
+      if (list === undefined) { list = []; byToken.set(token, list) }
+      list.push(branches[i])
     }
   }
+  if (byToken.size === 0) return null
+
+  const canonical = new Map()
+  const table = new Map()
+  for (const [token, list] of byToken) {
+    if (list.length === 1) { table.set(token, list[0]); continue }
+    const key = list.map((branch) => branches.indexOf(branch)).join(',')
+    let group = canonical.get(key)
+    if (group === undefined) { group = list; canonical.set(key, group) }
+    table.set(token, group)
+  }
 
-  return table.size > 0 ? table : null
+  return table
 }
 
 export { firstOf }