grammatik 0.1.0 → 0.1.2

package/src/codegen.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom, recognizeAltConsumeIdiom } from './analyze.js'
 
 /**
  * A real source generator for grammars.
@@ -23,10 +23,24 @@ import { buildDispatch, opOf, recognizeSeqIdiom } from './analyze.js'
 
 /**
  * @param {{ planner: any, rules: Record<string, any>, firsts: Map<string, any>|null,
- *           memo: boolean, maxSteps: number }} config
+ *           memo: boolean, maxSteps: number, tokenIds?: Map<string, number>|null,
+ *           memoSlots?: Map<string, number>, inlineRules?: boolean, idioms?: boolean }} config
  */
 export function createCodegen (config) {
-  const { planner, rules, firsts, memo, maxSteps } = config
+  const { planner, rules, firsts, memo, maxSteps, tokenIds = null, memoSlots = new Map(), inlineRules = true, idioms = true } = config
+
+  /**
+   * `token.id === <n>` is an integer compare instead of a string compare, and
+   * is what generated code prefers wherever a token's declared type matters.
+   * Falls back to `token.type === <name>` only for a hand-built token that
+   * bypassed this parser's own lexer (`parseTokens` accepts anything shaped
+   * like a Token) — every token the bundled lexer produces has an id.
+   */
+  const typeCheck = (tokenVar, name) => {
+    const id = tokenIds ? tokenIds.get(name) : undefined
+    if (id === undefined) return `${tokenVar}.type === ${JSON.stringify(name)}`
+    return `(${tokenVar}.id === ${id} || (${tokenVar}.id === undefined && ${tokenVar}.type === ${JSON.stringify(name)}))`
+  }
 
   /** Runtime values the generated code closes over. */
   const HELPERS = []
@@ -61,6 +75,56 @@ export function createCodegen (config) {
   /** A literal value sitting in parser position. */
   const constantHelper = (node) => helper(planner.plan(node), { kind: 'constant', node })
 
+  /**
+   * Operator-precedence levels (`Additive = infixLeft(subrule(Multiplicative), ...)`)
+   * are written as a chain of rules, one per level, and each level costs a call, a
+   * loop and a token check even when no operator of that level follows: measured at
+   * ~10 ns per level per atom, so an 8-level chain costs several times the atom
+   * itself. A level with exactly one caller can be written into that caller's body
+   * instead, which keeps the loop and drops the call.
+   *
+   * Only these four shapes, and only a single call site: nothing is duplicated, and
+   * the function that results stays small enough for V8 to optimize (a rule like
+   * `Primary`, with an `alt` of many branches, is not a level and is left a call).
+   */
+  // How many levels go into one function. Unbounded, a long chain makes a function
+  // larger than V8 will optimize at all (its `max-optimized-bytecode-size`), and the
+  // gain turns back into a loss: measured on a synthetic chain, 8 levels in one
+  // function ran at 66 ns per atom against 33 ns for 7. Six per function was the best
+  // of the caps tried (2, 3, 4, 5, 6, 8) across chains of 3 to 16 levels.
+  const MAX_INLINE_DEPTH = 6
+  /** `oneOf` with more names than this is generated as a `switch` on the token id. */
+  const ONE_OF_SWITCH_AT = 12
+  const INLINE_OPS = new Set(['infixLeft', 'infixRight', 'prefix', 'postfix'])
+  const callSites = new Map()
+  ;(function count (node) {
+    if (Array.isArray(node)) { for (const child of node) count(child); return }
+    if (!node || typeof node !== 'object') return
+    if ('subrule' in node) {
+      const target = Array.isArray(node.subrule) ? node.subrule[0] : node.subrule
+      if (typeof target === 'string') callSites.set(target, (callSites.get(target) || 0) + 1)
+    }
+    for (const key of Object.keys(node)) count(node[key])
+  })(rules)
+  /** Rules being written into a caller right now, so a cycle cannot recurse forever. */
+  const inlining = new Set()
+  /**
+   * Some constructs write a child twice: `infixLeft` and `infixRight` their operand (once for the
+   * first, once inside the loop), `many1` and `many1Sep` their item. Inline a rule into *both*
+   * copies and a chain of such rules multiplies out — one copy per level at the first, two at the
+   * next, four at the next: measured, the formula grammar's module nearly doubled and a six-level
+   * chain grew fivefold. The second copy is the loop body, which only runs when more input follows,
+   * so it stays a call; only the first copy, the one every parse goes through, is inlined.
+   */
+  let coldDepth = 0
+  const cold = (write) => {
+    coldDepth++
+    try { return write() } finally { coldDepth-- }
+  }
+  const canInline = (name) =>
+    inlineRules && coldDepth === 0 && !memo && !memoSlots.has(name) && !inlining.has(name) && inlining.size < MAX_INLINE_DEPTH &&
+    callSites.get(name) === 1 && INLINE_OPS.has(opOf(rules[name]))
+
   function indexOfRule (name) {
     let index = ruleIndex.get(name)
     if (index !== undefined) return index
@@ -99,7 +163,7 @@ export function createCodegen (config) {
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
-          `if (${token} !== undefined && ${token}.type === ${JSON.stringify(name)}) { c.idx++; ${result} = ${token}; }`,
+          `if (${token} !== undefined && ${typeCheck(token, name)}) { c.idx++; ${result} = ${token}; }`,
           `else { E(c, ${JSON.stringify(name)}); ${result} = F; }`
         )
         return result
@@ -140,6 +204,17 @@ export function createCodegen (config) {
         return result
       }
 
+      case 'flag': {
+        const start = nextId()
+        out.push(`let ${result};`, `const ${start} = c.idx;`, '{')
+        const value = gen(arg, out, bindingsVar)
+        out.push(
+          `if (${value} === F) { c.idx = ${start}; ${result} = false; } else ${result} = true;`,
+          '}'
+        )
+        return result
+      }
+
       case 'many':
       case 'many1': {
         const atLeastOne = op === 'many1'
@@ -159,7 +234,7 @@ export function createCodegen (config) {
         }
 
         out.push('for (;;) {', `const ${before} = c.idx;`)
-        const value = gen(arg, out, bindingsVar)
+        const value = atLeastOne ? cold(() => gen(arg, out, bindingsVar)) : gen(arg, out, bindingsVar)
         out.push(
           `if (${value} === F) { c.idx = ${before}; break; }`,
           `if (c.idx === ${before}) P(${JSON.stringify(op)}, ${before});`,
@@ -196,7 +271,7 @@ export function createCodegen (config) {
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`)
         }
 
-        const right = gen(arg.operand, out, bindingsVar)
+        const right = cold(() => gen(arg.operand, out, bindingsVar))
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${bindings} === null) ${bindings} = { left: null, op: null, right: null };`,
@@ -212,12 +287,34 @@ export function createCodegen (config) {
       case 'oneOf': {
         const names = Array.isArray(arg) ? arg : [arg]
         const token = nextId()
-        const test = names.map((n) => `${token}.type === ${JSON.stringify(n)}`).join(' || ')
+        const ids = tokenIds ? names.map((n) => tokenIds.get(n)) : []
+        const failed = `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(' ')} ${result} = F; }`
+
+        if (names.length > ONE_OF_SWITCH_AT && ids.every((id) => id !== undefined)) {
+          // Past a handful of names a chain of comparisons costs more than a jump
+          // table: measured, the chain matched `alt`'s switch at 12 names and lost
+          // beyond. Integer cases only, as `alt` does it — a `case undefined` for a
+          // hand-built token would stop V8 building the table — so the id is
+          // normalised first, from the type name when the token has none.
+          const hit = nextId()
+          const cases = [...new Set(ids)].map((id) => `case ${id}:`).join(' ')
+          out.push(
+            `let ${result};`,
+            `const ${token} = c.tokens[c.idx];`,
+            `let ${hit} = false;`,
+            `if (${token} !== undefined) { switch (${token}.id !== undefined ? ${token}.id : I[${token}.type]) { ${cases} ${hit} = true; } }`,
+            `if (${hit}) { c.idx++; ${result} = ${token}; }`,
+            failed
+          )
+          return result
+        }
+
+        const test = names.map((n) => typeCheck(token, n)).join(' || ')
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
           `if (${token} !== undefined && (${test})) { c.idx++; ${result} = ${token}; }`,
-          `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(' ')} ${result} = F; }`
+          failed
         )
         return result
       }
@@ -253,7 +350,7 @@ export function createCodegen (config) {
           operatorVar = gen(arg.operator, out, bindingsVar)
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`)
         }
-        const right = gen(arg.operand, out, bindingsVar)
+        const right = cold(() => gen(arg.operand, out, bindingsVar))
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${items} === null) ${items} = [];`,
@@ -334,6 +431,22 @@ export function createCodegen (config) {
         return result
       }
 
+      case 'pick':
+      case 'yields': {
+        // The parsers in order, then one of their results or a constant; no
+        // bindings object, which is the point (see `recognizeSeqIdiom`).
+        const label = 'L' + nextId()
+        const start = nextId()
+        out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
+        const values = arg.parsers.map((parser) => {
+          const value = gen(parser, out, bindingsVar)
+          out.push(`if (${value} === F) { c.idx = ${start}; ${result} = F; break ${label}; }`)
+          return value
+        })
+        out.push(`${result} = ${op === 'pick' ? values[arg.index] : JSON.stringify(arg.value)};`, '}')
+        return result
+      }
+
       case 'prefix': {
         const label = 'L' + nextId()
         const start = nextId()
@@ -415,7 +528,7 @@ export function createCodegen (config) {
         )
         const sep = gen(sepNode, out, bindingsVar)
         out.push(`if (${sep} === F) { c.idx = ${beforeSep}; break; }`, `const ${beforeItem} = c.idx;`)
-        const item = gen(arg.rule, out, bindingsVar)
+        const item = cold(() => gen(arg.rule, out, bindingsVar))
         out.push(
           `if (${item} === F) { c.idx = ${trailing ? beforeItem : beforeSep}; break; }`,
           `${result}.push(${item});`,
@@ -432,8 +545,20 @@ export function createCodegen (config) {
           out.push(`const ${result} = ${helper(planner.plan(node), { kind: 'unknownRule', name })}(c);`)
           return result
         }
-        const index = indexOfRule(name)
         const parent = nextId()
+        if (canInline(name)) {
+          out.push(tickSrc)
+          out.push(
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`
+          )
+          inlining.add(name)
+          const inner = gen(rules[name], out, null)
+          inlining.delete(name)
+          out.push(`if (c.track) c.ruleDepth = ${parent};`)
+          return inner
+        }
+        const index = indexOfRule(name)
         out.push(tickSrc)
         if (memo) {
           const key = nextId()
@@ -454,6 +579,27 @@ export function createCodegen (config) {
             `c.memo.set(${key}, { value: ${result}, end: c.idx });`,
             '}'
           )
+        } else if (memoSlots.has(name)) {
+          // Chosen by `selectMemoRules`: remember the result per position.
+          const table = nextId()
+          const at = nextId()
+          const end = nextId()
+          out.push(
+            `let ${result};`,
+            `const ${table} = c.rm[${memoSlots.get(name)}];`,
+            `const ${at} = c.idx;`,
+            `const ${end} = ${table}.ends[${at}];`,
+            `if (${end} !== 0) { c.idx = ${end} - 1; ${result} = ${table}.vals[${at}]; }`,
+            'else {',
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`,
+            `${result} = R[${index}](c);`,
+            `if (c.track) c.ruleDepth = ${parent};`,
+            `if (${result} === F) c.idx = ${at};`,
+            `${table}.ends[${at}] = c.idx + 1;`,
+            `${table}.vals[${at}] = ${result};`,
+            '}'
+          )
         } else {
           out.push(
             `const ${parent} = c.ruleDepth;`,
@@ -476,7 +622,7 @@ export function createCodegen (config) {
     if (!Array.isArray(children)) throw new Error("'seq' expects an array of parsers")
 
     // Same idiom recognition as the planner, so both paths agree.
-    const idiom = recognizeSeqIdiom(children)
+    const idiom = idioms ? recognizeSeqIdiom(children) : null
     if (idiom !== null) return gen(idiom, out, null)
 
     let hasLabels = false
@@ -520,29 +666,78 @@ export function createCodegen (config) {
     return result
   }
 
+  /**
+   * Emits an inline ordered-backtracking block over `list`, into a fresh
+   * variable. Used both for a whole `alt` with no dispatch table, and for the
+   * residual subset of branches a dispatch table cannot separate by one
+   * token.
+   */
+  function genOrderedBacktrack (list, out, bindingsVar) {
+    const result = nextId()
+    const label = 'L' + nextId()
+    const start = nextId()
+    out.push(`let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
+    for (const branch of list) {
+      out.push('{')
+      const value = gen(branch, out, bindingsVar)
+      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label}; }`, `c.idx = ${start};`, '}')
+    }
+    out.push(`${result} = F;`, '}')
+    return result
+  }
+
   function genAlt (branches, out, result, bindingsVar) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches")
 
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null
+    if (oneOf !== null) return gen(oneOf, out, bindingsVar)
+
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null
+    if (factored !== null) return gen(factored, out, bindingsVar)
+
     const table = firsts ? buildDispatch(branches, firsts) : null
     const label = 'L' + nextId()
 
     if (table !== null) {
       // A switch on the token type: V8 turns a dense string switch into a
       // hash lookup with inlined bodies, and every branch is inlined here.
+      // A table value that is an array is a residual group — tokens shared by
+      // more than one branch — resolved by ordered backtracking over just
+      // that subset instead of a single inlined branch.
+      //
+      // Switching on token.id (an integer) instead of token.type (a string)
+      // when every expected token resolves to one — a numeric switch is a jump
+      // table without even the string-hashing step a dense string switch still
+      // does. `I[token.type]` is the same fallback typeCheck uses: a hand-built
+      // token from parseTokens has no id, and I is a name -> id table built
+      // once from the same tokenIds this parser resolved every other id from.
       const expected = [...table.keys()]
+      const expectedIds = tokenIds ? expected.map((t) => tokenIds.get(t)) : null
+      const numeric = expectedIds !== null && expectedIds.every((id) => id !== undefined)
       const token = nextId()
       out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${token} = c.tokens[c.idx];`)
       out.push(`if (${token} === undefined) { ${expected.map((t) => `E(c, ${JSON.stringify(t)});`).join(' ')} ${result} = F; break ${label}; }`)
-      out.push(`switch (${token}.type) {`)
+
+      let discriminant
+      if (numeric) {
+        discriminant = nextId()
+        out.push(`const ${discriminant} = ${token}.id !== undefined ? ${token}.id : I[${token}.type];`)
+        out.push(`switch (${discriminant}) {`)
+      } else {
+        discriminant = `${token}.type`
+        out.push(`switch (${discriminant}) {`)
+      }
 
       const byBranch = new Map()
-      for (const [tokenName, branch] of table) {
+      for (const tokenName of expected) {
+        const branch = table.get(tokenName)
         if (!byBranch.has(branch)) byBranch.set(branch, [])
         byBranch.get(branch).push(tokenName)
       }
+      const caseLabel = (t) => numeric ? tokenIds.get(t) : JSON.stringify(t)
       for (const [branch, tokenNames] of byBranch) {
-        out.push(tokenNames.map((t) => `case ${JSON.stringify(t)}:`).join(' ') + ' {')
-        const value = gen(branch, out, bindingsVar)
+        out.push(tokenNames.map((t) => `case ${caseLabel(t)}:`).join(' ') + ' {')
+        const value = Array.isArray(branch) ? genOrderedBacktrack(branch, out, bindingsVar) : gen(branch, out, bindingsVar)
         out.push(`${result} = ${value}; break ${label};`, '}')
       }
       out.push(
@@ -554,17 +749,13 @@ export function createCodegen (config) {
     }
 
     // Ordered backtracking, still fully inlined.
-    const start = nextId()
-    out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
-    for (const branch of branches) {
-      out.push('{')
-      const value = gen(branch, out, bindingsVar)
-      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label}; }`, `c.idx = ${start};`, '}')
-    }
-    out.push(`${result} = F;`, '}')
-    return result
+    out.push(tickSrc)
+    return genOrderedBacktrack(branches, out, bindingsVar)
   }
 
+  /** Name -> id, for a generated switch's runtime fallback when token.id is missing. */
+  const idByName = tokenIds ? Object.fromEntries(tokenIds) : null
+
   /** Generates and installs the function for one rule. */
   function buildRule (name) {
     const out = []
@@ -576,8 +767,8 @@ export function createCodegen (config) {
     SOURCES[ruleIndex.get(name)] = body
     const source = `return ${body}`
     // eslint-disable-next-line no-new-func
-    const factory = new Function('F', 'E', 'P', 'T', 'R', 'H', source)
-    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS)
+    const factory = new Function('F', 'E', 'P', 'T', 'R', 'H', 'I', source)
+    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS, idByName)
   }
 
   function progressError (op, at) {

package/src/definition.js

@@ -227,6 +227,7 @@ const CONSTRUCTORS = {
   many: { arity: [1, 1], of: ([a]) => ({ many: a }) },
   many1: { arity: [1, 1], of: ([a]) => ({ many1: a }) },
   option: { arity: [1, 1], of: ([a]) => ({ option: a }) },
+  flag: { arity: [1, 1], of: ([a]) => ({ flag: a }) },
   action: { arity: [1, 1], of: ([a]) => ({ action: a }) },
   lookahead: { arity: [1, 1], of: ([a]) => ({ lookahead: a }) },
   negLookahead: { arity: [1, 1], of: ([a]) => ({ negLookahead: a }) },
@@ -241,6 +242,10 @@ const CONSTRUCTORS = {
   many1Sep: { arity: [2, 2], of: ([a, b]) => ({ many1Sep: { rule: a, sep: b } }) },
 
   between: { arity: [3, 3], of: ([a, b, c]) => ({ between: { open: a, body: b, close: c } }) },
+  // pick(1, consume(Open), subrule(Body), consume(Close)) — the result at that index.
+  pick: { arity: [2, Infinity], of: ([index, ...parsers]) => ({ pick: { parsers, index } }) },
+  // yields(consume(True), true) — run the parsers, give back the constant.
+  yields: { arity: [2, Infinity], of: (args) => ({ yields: { parsers: args.slice(0, -1), value: args[args.length - 1] } }) },
   infixLeft: { arity: [3, 3], of: ([a, b, c]) => ({ infixLeft: { operand: a, operator: b, combine: logicOf(c) } }) },
   infixRight: { arity: [3, 3], of: ([a, b, c]) => ({ infixRight: { operand: a, operator: b, combine: logicOf(c) } }) },
   prefix: { arity: [3, 3], of: ([a, b, c]) => ({ prefix: { operator: a, operand: b, combine: logicOf(c) } }) },

package/src/emit.js

@@ -2,9 +2,11 @@
 'use strict'
 
 import { createLexer } from './lexer.js'
+import { generateLexer } from './lexgen.js'
 import { analyze } from './analyze.js'
 import { createPlanner } from './plan.js'
 import { createCodegen } from './codegen.js'
+import { selectMemoRules } from './memoplan.js'
 import { usesOuterScope } from './plan.js'
 import { emitMethodsBlock, evaluateMethodsBlock } from './methodsblock.js'
 import { createValueMethods } from './values.js'
@@ -100,8 +102,9 @@ const literal = (value, indent = 0) => JSON.stringify(value, null, 2)
 
 /**
  * @param {import('./parser.js').GrammarSpec} spec
- * @param {{ memo?: boolean, strict?: boolean, ll1?: boolean, maxSteps?: number,
+ * @param {{ memo?: boolean, autoMemo?: boolean, inlineRules?: boolean, idioms?: boolean, strict?: boolean, ll1?: boolean, maxSteps?: number,
  *           positions?: 'full'|'offset', execution?: 'generated'|'interpreted',
+ *           lexer?: 'generated'|'runtime', scanners?: boolean, fuse?: boolean,
  *           format?: 'esm'|'cjs', validate?: boolean, moduleName?: string,
  *           runtimeSpecifier?: string, engineSpecifier?: string,
  *           methods?: Record<string, any> }} [options]
@@ -111,11 +114,21 @@ export function emitModule (spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec
   const {
     memo = false,
+    autoMemo = true,
+    // Off here, on in `createParser`. Inlining a precedence chain is worth 2-3x on a bare chain but only
+    // 1.02-1.04x on the formula and SQL examples, and it makes the file bigger: the formula module grows
+    // 32% minified (38 to 50 KB) even after the second copy of each operand is kept as a call. A parser
+    // built at run time pays nothing for that; a file someone ships does.
+    inlineRules = false,
+    idioms = true,
     strict = true,
     ll1 = true,
     maxSteps = 0,
     positions = 'full',
     execution = 'generated',
+    lexer: lexerMode = 'generated',
+    scanners = true,
+    fuse = true,
     format = 'esm',
     validate = true,
     moduleName = spec.name || 'parser',
@@ -125,6 +138,7 @@ export function emitModule (spec, options = {}) {
   } = options
 
   if (format !== 'esm' && format !== 'cjs') throw new Error(`emitModule: format must be 'esm' or 'cjs', got ${quote(format)}`)
+  if (lexerMode !== 'generated' && lexerMode !== 'runtime') throw new Error(`emitModule: lexer must be 'generated' or 'runtime', got ${quote(lexerMode)}`)
 
   // A `methods { ... }` block travels with the grammar and is written into the
   // output as source. Nothing evaluates it here, so this path stays eval-free.
@@ -132,6 +146,8 @@ export function emitModule (spec, options = {}) {
   const block = methodsBlock ? emitMethodsBlock(methodsBlock, format) : null
   if (!start || !(start in rules)) throw new Error(`emitModule: start rule '${start}' is not defined in rules`)
 
+  // Built here even when the file will carry its own lexer: constructing it is what validates the token
+  // definitions, and it fixes the token ids the rules are compiled against.
   const lexer = createLexer(tokenDefs, { positions })
   const analysis = analyze(rules, { tokenNames: lexer.tokenNames })
   if (validate && analysis.errors.length > 0) throw new GrammarError(analysis.errors)
@@ -139,9 +155,17 @@ export function emitModule (spec, options = {}) {
   // The planner is needed only so codegen can record what each helper is; the
   // closures it makes are thrown away.
   const firsts = ll1 ? analysis.firsts : null
+  // The generated file's own `createLexer(tokenDefs, ...)` call assigns the same
+  // ids, from the same `tokenDefs` array in the same order — see the matching
+  // comment in parser.js.
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]))
   const engine = { build: () => { throw new Error('not compiled during emit') }, run: () => {} }
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false })
-  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps })
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0
+    ? selectMemoRules(rules, analysis.firsts, { ll1 })
+    : []
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]))
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds, memoSlots, idioms })
+  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms })
 
   codegen.compile(start)
 
@@ -210,7 +234,9 @@ export function emitModule (spec, options = {}) {
   // actions may use the value methods, so it has to register them the way
   // `createParser` does. They come from the runtime rather than being written
   // out here, so there is one implementation rather than two.
-  const runtimeImports = ['FAIL as F', 'expect as E', 'createState', 'buildParseError', 'createLexer']
+  // A lexer written out as source needs only the error class from the runtime; the engine stays out of the file.
+  const generatedLexer = lexerMode === 'generated'
+  const runtimeImports = ['FAIL as F', 'expect as E', 'createState', 'buildParseError', generatedLexer ? 'LexError' : 'createLexer']
   if (needsEngine) runtimeImports.push('createValueMethods')
 
   const importRuntime = cjs
@@ -240,7 +266,7 @@ export function emitModule (spec, options = {}) {
 //
 // The grammar was analyzed when this file was written: FIRST sets, left-recursion
 // checks and rule specialization all happened then. Importing it does no grammar
-// work and generates no code.
+// work and generates no code.${generatedLexer ? '\n// The lexer was written out as source too, so there is no lexer engine to load.' : ''}
 ${engineFree
   ? '// Every semantic action was compiled to source too, so this file has no\n// dependency on json-logic-engine at all.'
   : interpretedActions
@@ -254,7 +280,7 @@ const tokens = ${literal(tokenDefs)}
 
 const start = ${quote(start)}
 
-const lexer = createLexer(tokens, { positions: ${quote(positions)} })
+const lexer = ${generatedLexer ? generateLexer(tokenDefs, { positions, scanners, fuse }) : `createLexer(tokens, { positions: ${quote(positions)}, scanners: ${scanners}, fuse: ${fuse} })`}
 const tokenize = lexer.tokenize
 
 ${engineFree
@@ -335,6 +361,14 @@ const H = [
 ${helpers.map((source, index) => `  /* ${index} */ ${source}`).join(',\n')}
 ]
 
+/**
+ * Name -> id, the same numbering \`lexer\` assigns. A generated \`alt\`'s switch
+ * dispatches on a token's id when it has one; this is only read as a fallback,
+ * for a hand-built token from \`parseTokens\` that bypassed this file's own
+ * lexer and so has no id of its own.
+ */
+const I = ${literal(Object.fromEntries(tokenIds))}
+
 ${ruleSources}
 
 const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name, index])))}
@@ -345,7 +379,7 @@ const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name,
  * @param {string} [source]
  */
 function parseTokens (tokenList, source, track = false) {
-  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track })
+  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track, memoSlots: ${memoRules.length} })
   state.source = source
   state.positions = ${quote(positions)}
   state.ruleNames[0] = start

package/src/lexer.d.ts

@@ -5,19 +5,60 @@
  * large win for punctuation-heavy grammars (JSON is ~60% punctuation by token count).
  *
  * @param {TokenDef[]} defs
- * @param {{ positions?: 'full' | 'offset' }} [options]
+ * @param {{ positions?: 'full' | 'offset', scanners?: boolean, fuse?: boolean, incremental?: boolean, recover?: boolean, errorToken?: string }} [options]
  *   `full` (default) records line and column on every token. `offset` records
  *   only `start`/`end`, which skips a scan of the entire input for newlines —
  *   worth taking when errors do not need to name a line, or when positions are
  *   resolved lazily from the offset afterwards.
- * @returns {{ tokenize: (text: string) => Token[], tokenNames: string[], positions: string }}
+ *
+ *   `incremental` (default false) is opt-in: when set, the returned lexer also
+ *   exposes `tokenizeIncremental` and `relex`, for editors that want to re-lex
+ *   only the region touched by a single edit instead of the whole document.
+ *   A lexer created without it is bit-for-bit the plain lexer — no overhead.
+ *
+ *   `scanners` (default true) runs patterns of a few common shapes as
+ *   table-driven loops instead of regex calls. The tokens are identical;
+ *   `false` forces every pattern through its regex.
+ *
+ *   `fuse` (default true) joins runs of three or more consecutive regex candidates
+ *   into one alternation, so a position where several tokens could start costs one
+ *   regex call rather than one per rejected token. Same tokens; `false` tries them
+ *   one at a time.
+ *
+ *   `recover` (default false, only meaningful with `incremental`) makes an
+ *   unmatched character produce a single-character token of type `errorToken`
+ *   (default `'Error'`) instead of throwing, so highlighting survives invalid
+ *   or half-typed input.
+ * @returns {{ tokenize: (text: string) => Token[], tokenizeIncremental?: (text: string) => IncrementalResult, relex?: (text: string, prev: IncrementalResult, edit: { start: number, oldEnd: number, newEnd: number }) => IncrementalResult, tokenNames: string[], positions: string, modeNames: string[] }}
  */
 export function createLexer(defs: TokenDef[], options?: {
     positions?: "full" | "offset";
+    scanners?: boolean;
+    fuse?: boolean;
+    incremental?: boolean;
+    recover?: boolean;
+    errorToken?: string;
 }): {
     tokenize: (text: string) => Token[];
+    tokenizeIncremental?: (text: string) => IncrementalResult;
+    relex?: (text: string, prev: IncrementalResult, edit: {
+        start: number;
+        oldEnd: number;
+        newEnd: number;
+    }) => IncrementalResult;
     tokenNames: string[];
     positions: string;
+    modeNames: string[];
+};
+/**
+ * Opaque state returned by `tokenizeIncremental`/`relex` — pass it back into
+ * `relex` unmodified; only `.tokens` is meant to be read directly.
+ */
+export type IncrementalResult = {
+    text: string;
+    tokens: Token[];
+    checkpoints: object[];
+    eof: object;
 };
 /**
  * A Chevrotain-style lexer. Token definitions are plain JSON, so a whole