grammatik 0.1.1 → 0.1.2

package/dist/types/index.d.ts

@@ -70,6 +70,29 @@ export interface ParserOptions {
   positions?: PositionMode
   /** Packrat memoization on (rule, position). */
   memo?: boolean
+  /**
+   * Memoize only the rules that several colliding branches of an `alt` would
+   * otherwise each re-parse (default `true`; ignored when `memo` is on). Avoids
+   * the exponential blow-up of nested alternatives that share a leading rule,
+   * at no cost to a grammar with none. See `memoRules`.
+   */
+  autoMemo?: boolean
+  /**
+   * In generated code, write single-caller operator levels (`infixLeft`,
+   * `infixRight`, `prefix`, `postfix`) into their caller instead of calling them
+   * (default `true` for `createParser`, `false` for `emitModule`). Several levels of an
+   * expression grammar become one function: 2-3x faster on a bare precedence chain, about
+   * 1.02-1.04x on real grammars, at the cost of a bigger file, which is why an emitted
+   * parser leaves it off.
+   */
+  inlineRules?: boolean
+  /**
+   * Recognize common spellings of a grammar (`seq(open, label, close, action)`, a
+   * prefix plus one labelled result, a constant action, an `alt` of bare `consume`s,
+   * a shared leading run of `alt` branches) and run them as the cheaper combinator
+   * they are equivalent to. Default `true`; `false` runs the grammar exactly as written.
+   */
+  idioms?: boolean
   /** Require the whole token stream to be consumed. */
   strict?: boolean
   /** FIRST-set dispatch for `alt`. Off forces ordered backtracking. */
@@ -109,6 +132,8 @@ export interface JLParser {
   engine: any
   analysis: Analysis
   execution: ExecutionMode
+  /** The rules `autoMemo` chose to memoize, in declaration order. */
+  memoRules: string[]
   compiled: boolean
   readonly firsts: Record<string, Set<string>>
 }
@@ -126,6 +151,17 @@ export declare function analyze (
 ): Analysis
 
 export interface EmitOptions extends ParserOptions {
+  /**
+   * How the file lexes. `'generated'` (default) writes the lexer out as source, so the
+   * file imports only the parse state and error class from the runtime: a smaller
+   * bundle (JSON: 24 KB to 10 KB minified), a faster start, and faster lexing.
+   * `'runtime'` keeps the token table and builds the lexer from it when the file loads.
+   */
+  lexer?: 'generated' | 'runtime'
+  /** Scanner substitution for the lexer, as for `createLexer`. Default `true`. */
+  scanners?: boolean
+  /** Fusing runs of regex tokens in the lexer, as for `createLexer`. Default `true`. */
+  fuse?: boolean
   /** Module format of the generated file. */
   format?: 'esm' | 'cjs'
   /** Name recorded in the file header. Defaults to `spec.name`. */

package/dist/types/runtime.d.ts

@@ -1,9 +1,10 @@
 /**
  * `grammatik/runtime` — the surface a generated parser imports.
  *
- * Deliberately small: a generated file needs the lexer, the parse state and
- * error construction, and nothing else. The analyzer, planner and code
- * generator were already run when the file was written.
+ * Deliberately small: a generated file needs the parse state and error
+ * construction, and nothing else. The analyzer, planner, code generator and
+ * (by default) the lexer were already run when the file was written. `createLexer`
+ * is still here for files emitted with `lexer: 'runtime'`.
  */
 
 import type { Token, TokenDef, PositionMode, Lexer, Parser } from './index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grammatik",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "A parser toolkit whose grammars are data: tokens, rules and semantic actions all serializable, compiled ahead of time.",
   "type": "module",
   "main": "./dist/cjs/index.js",

package/src/analyze.js

@@ -60,9 +60,9 @@ export function opOf (node) {
 
 /** The set of node shapes this module understands as *parsers*. */
 export const PARSER_OPS = new Set([
-  'consume', 'seq', 'alt', 'many', 'many1', 'option',
+  'consume', 'seq', 'alt', 'many', 'many1', 'option', 'flag',
   'manySep', 'many1Sep', 'infixLeft', 'infixRight', 'postfix', 'prefix',
-  'between', 'as', 'oneOf', 'eof',
+  'between', 'pick', 'yields', 'as', 'oneOf', 'eof',
   'subrule', 'label', 'action',
   'lookahead', 'negLookahead', 'text', 'epsilon'
 ])
@@ -122,7 +122,8 @@ function firstOf (node, ruleFirsts) {
     }
 
     case 'many':
-    case 'option': {
+    case 'option':
+    case 'flag': {
       const inner = firstOf(arg, ruleFirsts)
       return { tokens: inner.tokens, nullable: true, unknown: inner.unknown }
     }
@@ -153,6 +154,11 @@ function firstOf (node, ruleFirsts) {
       // Exactly a sequence of open, body, close.
       return firstOf({ seq: [arg && arg.open, arg && arg.body, arg && arg.close] }, ruleFirsts)
 
+    case 'pick':
+    case 'yields':
+      // Exactly a sequence of its parsers.
+      return firstOf({ seq: arg && Array.isArray(arg.parsers) ? arg.parsers : [] }, ruleFirsts)
+
     case 'prefix': {
       // Zero operators are allowed, so the operand's first set counts too.
       const ops = firstOf(arg && arg.operator, ruleFirsts)
@@ -254,9 +260,61 @@ export function recognizeSeqIdiom (children) {
     }
   }
 
+  // The two idioms below drop the `seq`'s bindings object, so they decline when a
+  // child could reach into an enclosing scope: without that object an outward hop
+  // would land one scope further out than it did.
+  if (children.length >= 2) {
+    const last = children[children.length - 1]
+    const head = children.slice(0, -1)
+
+    // yields: parsers, then an action that is a constant — `seq(consume(True), action(true))`
+    if (opOf(last) === 'action' && isConstant(last.action) && head.every(isPlainParser) && !head.some(usesOuterScope)) {
+      return { yields: { parsers: head, value: last.action } }
+    }
+
+    // pick: any number of unlabelled parsers around exactly one labelled one, then
+    // an action that returns just that label — `seq(consume(Where), label(e, P), action({val: e}))`.
+    // `between` is the special case with one on each side.
+    if (opOf(last) === 'action' && opOf(last.action) === 'val' && typeof last.action.val === 'string') {
+      const name = last.action.val
+      const labelled = head.map((child, i) => (opOf(child) === 'label' ? i : -1)).filter((i) => i !== -1)
+      if (labelled.length === 1 && head[labelled[0]].label[0] === name &&
+          head.every((child, i) => i === labelled[0] || isPlainParser(child)) &&
+          !head.some(usesOuterScope)) {
+        return { pick: { parsers: head.map((child, i) => (i === labelled[0] ? child.label[1] : child)), index: labelled[0] } }
+      }
+    }
+  }
+
   return null
 }
 
+/** A value that cannot be mutated or aliased: safe to hand out from every parse. */
+const isConstant = (value) =>
+  value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
+
+/**
+ * True when the logic reads from an outer scope (`{val: [[-1], ...]}` or
+ * `{var: "../x"}`). The engine's compiler does not thread `above` into built
+ * functions, so such actions must stay interpreted — and a rewrite that removes
+ * a `seq`'s bindings object would change what such a hop reaches.
+ */
+export function usesOuterScope (logic) {
+  if (Array.isArray(logic)) return logic.some(usesOuterScope)
+  if (!logic || typeof logic !== 'object') return false
+
+  for (const key of Object.keys(logic)) {
+    const arg = logic[key]
+    if (key === 'val' && Array.isArray(arg) && Array.isArray(arg[0])) return true
+    if (key === 'var') {
+      if (typeof arg === 'string' && arg.includes('../')) return true
+      if (Array.isArray(arg) && typeof arg[0] === 'string' && arg[0].includes('../')) return true
+    }
+    if (usesOuterScope(arg)) return true
+  }
+  return false
+}
+
 /** A node that is a parser and not one of `seq`'s own interpreted shapes. */
 function isPlainParser (node) {
   const op = opOf(node)
@@ -339,6 +397,32 @@ export function recognizeAltPrefixIdiom (branches) {
   }
 }
 
+/**
+ * Rewrites an `alt` whose every branch is a bare `consume` into `oneOf`:
+ * `alt(consume(Plus), consume(Minus))` is `oneOf(Plus, Minus)`.
+ *
+ * Same language, same value (the matched token), same expected-token list on
+ * failure — a token has one type, so "first branch that matches" and "any of
+ * these" cannot differ. What differs is cost: `alt` builds a branch per token,
+ * `oneOf` is a single test. Measured at 1.3–1.8x for the 2–7 token operator sets
+ * grammars actually have, and still 1.2x at 24.
+ *
+ * @param {any[]} branches
+ * @returns {{ oneOf: string[] } | null}
+ */
+export function recognizeAltConsumeIdiom (branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null
+  const names = []
+  for (const branch of branches) {
+    if (opOf(branch) !== 'consume') return null
+    const arg = branch.consume
+    const name = typeof arg === 'string' ? arg : (Array.isArray(arg) && arg.length === 1 ? arg[0] : null)
+    if (typeof name !== 'string') return null
+    names.push(name)
+  }
+  return { oneOf: names }
+}
+
 const BAIL = Symbol('grammatik.bail')
 
 /**
@@ -422,7 +506,7 @@ function computeFirsts (grammar) {
  * @param {Map<string, First>} ruleFirsts
  * @param {Set<string>} out
  */
-function leftmostCalls (node, ruleFirsts, out) {
+export function leftmostCalls (node, ruleFirsts, out) {
   const op = opOf(node)
   if (op === null) return
   const arg = node[op]
@@ -440,6 +524,7 @@ function leftmostCalls (node, ruleFirsts, out) {
     case 'many':
     case 'many1':
     case 'option':
+    case 'flag':
     case 'lookahead':
     case 'negLookahead':
       leftmostCalls(arg, ruleFirsts, out)
@@ -450,6 +535,10 @@ function leftmostCalls (node, ruleFirsts, out) {
     case 'between':
       if (arg) leftmostCalls({ seq: [arg.open, arg.body, arg.close] }, ruleFirsts, out)
       break
+    case 'pick':
+    case 'yields':
+      if (arg && Array.isArray(arg.parsers)) leftmostCalls({ seq: arg.parsers }, ruleFirsts, out)
+      break
     case 'prefix':
       if (arg) {
         leftmostCalls(arg.operator, ruleFirsts, out)
@@ -527,6 +616,7 @@ function validateNode (node, grammar, tokenNames, errors, path) {
     case 'many':
     case 'many1':
     case 'option':
+    case 'flag':
     case 'text':
     case 'lookahead':
     case 'negLookahead':
@@ -548,6 +638,26 @@ function validateNode (node, grammar, tokenNames, errors, path) {
       }
       break
     }
+    case 'pick': {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0) {
+        errors.push(`${path}: 'pick' expects { parsers: [parser, ...], index }`)
+      } else if (!Number.isInteger(arg.index) || arg.index < 0 || arg.index >= arg.parsers.length) {
+        errors.push(`${path}: 'pick' index ${JSON.stringify(arg.index)} is out of range for ${arg.parsers.length} parser(s)`)
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.pick[${i}]`))
+      }
+      break
+    }
+    case 'yields': {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0 || !('value' in arg)) {
+        errors.push(`${path}: 'yields' expects { parsers: [parser, ...], value }`)
+      } else if (!isConstant(arg.value)) {
+        errors.push(`${path}: 'yields' value must be a string, number, boolean or null, got ${JSON.stringify(arg.value)}`)
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.yields[${i}]`))
+      }
+      break
+    }
     case 'prefix': {
       if (!arg || typeof arg !== 'object' || !('operator' in arg) || !('operand' in arg) || !('combine' in arg)) {
         errors.push(`${path}: 'prefix' expects { operator, operand, combine }`)

package/src/codegen.js

@@ -2,7 +2,7 @@
 'use strict'
 
 import { FAIL, expect } from './state.js'
-import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom } from './analyze.js'
+import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom, recognizeAltConsumeIdiom } from './analyze.js'
 
 /**
  * A real source generator for grammars.
@@ -23,10 +23,11 @@ import { buildDispatch, opOf, recognizeSeqIdiom, recognizeAltPrefixIdiom } from
 
 /**
  * @param {{ planner: any, rules: Record<string, any>, firsts: Map<string, any>|null,
- *           memo: boolean, maxSteps: number, tokenIds?: Map<string, number>|null }} config
+ *           memo: boolean, maxSteps: number, tokenIds?: Map<string, number>|null,
+ *           memoSlots?: Map<string, number>, inlineRules?: boolean, idioms?: boolean }} config
  */
 export function createCodegen (config) {
-  const { planner, rules, firsts, memo, maxSteps, tokenIds = null } = config
+  const { planner, rules, firsts, memo, maxSteps, tokenIds = null, memoSlots = new Map(), inlineRules = true, idioms = true } = config
 
   /**
    * `token.id === <n>` is an integer compare instead of a string compare, and
@@ -74,6 +75,56 @@ export function createCodegen (config) {
   /** A literal value sitting in parser position. */
   const constantHelper = (node) => helper(planner.plan(node), { kind: 'constant', node })
 
+  /**
+   * Operator-precedence levels (`Additive = infixLeft(subrule(Multiplicative), ...)`)
+   * are written as a chain of rules, one per level, and each level costs a call, a
+   * loop and a token check even when no operator of that level follows: measured at
+   * ~10 ns per level per atom, so an 8-level chain costs several times the atom
+   * itself. A level with exactly one caller can be written into that caller's body
+   * instead, which keeps the loop and drops the call.
+   *
+   * Only these four shapes, and only a single call site: nothing is duplicated, and
+   * the function that results stays small enough for V8 to optimize (a rule like
+   * `Primary`, with an `alt` of many branches, is not a level and is left a call).
+   */
+  // How many levels go into one function. Unbounded, a long chain makes a function
+  // larger than V8 will optimize at all (its `max-optimized-bytecode-size`), and the
+  // gain turns back into a loss: measured on a synthetic chain, 8 levels in one
+  // function ran at 66 ns per atom against 33 ns for 7. Six per function was the best
+  // of the caps tried (2, 3, 4, 5, 6, 8) across chains of 3 to 16 levels.
+  const MAX_INLINE_DEPTH = 6
+  /** `oneOf` with more names than this is generated as a `switch` on the token id. */
+  const ONE_OF_SWITCH_AT = 12
+  const INLINE_OPS = new Set(['infixLeft', 'infixRight', 'prefix', 'postfix'])
+  const callSites = new Map()
+  ;(function count (node) {
+    if (Array.isArray(node)) { for (const child of node) count(child); return }
+    if (!node || typeof node !== 'object') return
+    if ('subrule' in node) {
+      const target = Array.isArray(node.subrule) ? node.subrule[0] : node.subrule
+      if (typeof target === 'string') callSites.set(target, (callSites.get(target) || 0) + 1)
+    }
+    for (const key of Object.keys(node)) count(node[key])
+  })(rules)
+  /** Rules being written into a caller right now, so a cycle cannot recurse forever. */
+  const inlining = new Set()
+  /**
+   * Some constructs write a child twice: `infixLeft` and `infixRight` their operand (once for the
+   * first, once inside the loop), `many1` and `many1Sep` their item. Inline a rule into *both*
+   * copies and a chain of such rules multiplies out — one copy per level at the first, two at the
+   * next, four at the next: measured, the formula grammar's module nearly doubled and a six-level
+   * chain grew fivefold. The second copy is the loop body, which only runs when more input follows,
+   * so it stays a call; only the first copy, the one every parse goes through, is inlined.
+   */
+  let coldDepth = 0
+  const cold = (write) => {
+    coldDepth++
+    try { return write() } finally { coldDepth-- }
+  }
+  const canInline = (name) =>
+    inlineRules && coldDepth === 0 && !memo && !memoSlots.has(name) && !inlining.has(name) && inlining.size < MAX_INLINE_DEPTH &&
+    callSites.get(name) === 1 && INLINE_OPS.has(opOf(rules[name]))
+
   function indexOfRule (name) {
     let index = ruleIndex.get(name)
     if (index !== undefined) return index
@@ -153,6 +204,17 @@ export function createCodegen (config) {
         return result
       }
 
+      case 'flag': {
+        const start = nextId()
+        out.push(`let ${result};`, `const ${start} = c.idx;`, '{')
+        const value = gen(arg, out, bindingsVar)
+        out.push(
+          `if (${value} === F) { c.idx = ${start}; ${result} = false; } else ${result} = true;`,
+          '}'
+        )
+        return result
+      }
+
       case 'many':
       case 'many1': {
         const atLeastOne = op === 'many1'
@@ -172,7 +234,7 @@ export function createCodegen (config) {
         }
 
         out.push('for (;;) {', `const ${before} = c.idx;`)
-        const value = gen(arg, out, bindingsVar)
+        const value = atLeastOne ? cold(() => gen(arg, out, bindingsVar)) : gen(arg, out, bindingsVar)
         out.push(
           `if (${value} === F) { c.idx = ${before}; break; }`,
           `if (c.idx === ${before}) P(${JSON.stringify(op)}, ${before});`,
@@ -209,7 +271,7 @@ export function createCodegen (config) {
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`)
         }
 
-        const right = gen(arg.operand, out, bindingsVar)
+        const right = cold(() => gen(arg.operand, out, bindingsVar))
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${bindings} === null) ${bindings} = { left: null, op: null, right: null };`,
@@ -225,12 +287,34 @@ export function createCodegen (config) {
       case 'oneOf': {
         const names = Array.isArray(arg) ? arg : [arg]
         const token = nextId()
+        const ids = tokenIds ? names.map((n) => tokenIds.get(n)) : []
+        const failed = `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(' ')} ${result} = F; }`
+
+        if (names.length > ONE_OF_SWITCH_AT && ids.every((id) => id !== undefined)) {
+          // Past a handful of names a chain of comparisons costs more than a jump
+          // table: measured, the chain matched `alt`'s switch at 12 names and lost
+          // beyond. Integer cases only, as `alt` does it — a `case undefined` for a
+          // hand-built token would stop V8 building the table — so the id is
+          // normalised first, from the type name when the token has none.
+          const hit = nextId()
+          const cases = [...new Set(ids)].map((id) => `case ${id}:`).join(' ')
+          out.push(
+            `let ${result};`,
+            `const ${token} = c.tokens[c.idx];`,
+            `let ${hit} = false;`,
+            `if (${token} !== undefined) { switch (${token}.id !== undefined ? ${token}.id : I[${token}.type]) { ${cases} ${hit} = true; } }`,
+            `if (${hit}) { c.idx++; ${result} = ${token}; }`,
+            failed
+          )
+          return result
+        }
+
         const test = names.map((n) => typeCheck(token, n)).join(' || ')
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
           `if (${token} !== undefined && (${test})) { c.idx++; ${result} = ${token}; }`,
-          `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(' ')} ${result} = F; }`
+          failed
         )
         return result
       }
@@ -266,7 +350,7 @@ export function createCodegen (config) {
           operatorVar = gen(arg.operator, out, bindingsVar)
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`)
         }
-        const right = gen(arg.operand, out, bindingsVar)
+        const right = cold(() => gen(arg.operand, out, bindingsVar))
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${items} === null) ${items} = [];`,
@@ -347,6 +431,22 @@ export function createCodegen (config) {
         return result
       }
 
+      case 'pick':
+      case 'yields': {
+        // The parsers in order, then one of their results or a constant; no
+        // bindings object, which is the point (see `recognizeSeqIdiom`).
+        const label = 'L' + nextId()
+        const start = nextId()
+        out.push(tickSrc, `let ${result};`, `${label}: {`, `const ${start} = c.idx;`)
+        const values = arg.parsers.map((parser) => {
+          const value = gen(parser, out, bindingsVar)
+          out.push(`if (${value} === F) { c.idx = ${start}; ${result} = F; break ${label}; }`)
+          return value
+        })
+        out.push(`${result} = ${op === 'pick' ? values[arg.index] : JSON.stringify(arg.value)};`, '}')
+        return result
+      }
+
       case 'prefix': {
         const label = 'L' + nextId()
         const start = nextId()
@@ -428,7 +528,7 @@ export function createCodegen (config) {
         )
         const sep = gen(sepNode, out, bindingsVar)
         out.push(`if (${sep} === F) { c.idx = ${beforeSep}; break; }`, `const ${beforeItem} = c.idx;`)
-        const item = gen(arg.rule, out, bindingsVar)
+        const item = cold(() => gen(arg.rule, out, bindingsVar))
         out.push(
           `if (${item} === F) { c.idx = ${trailing ? beforeItem : beforeSep}; break; }`,
           `${result}.push(${item});`,
@@ -445,8 +545,20 @@ export function createCodegen (config) {
           out.push(`const ${result} = ${helper(planner.plan(node), { kind: 'unknownRule', name })}(c);`)
           return result
         }
-        const index = indexOfRule(name)
         const parent = nextId()
+        if (canInline(name)) {
+          out.push(tickSrc)
+          out.push(
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`
+          )
+          inlining.add(name)
+          const inner = gen(rules[name], out, null)
+          inlining.delete(name)
+          out.push(`if (c.track) c.ruleDepth = ${parent};`)
+          return inner
+        }
+        const index = indexOfRule(name)
         out.push(tickSrc)
         if (memo) {
           const key = nextId()
@@ -467,6 +579,27 @@ export function createCodegen (config) {
             `c.memo.set(${key}, { value: ${result}, end: c.idx });`,
             '}'
           )
+        } else if (memoSlots.has(name)) {
+          // Chosen by `selectMemoRules`: remember the result per position.
+          const table = nextId()
+          const at = nextId()
+          const end = nextId()
+          out.push(
+            `let ${result};`,
+            `const ${table} = c.rm[${memoSlots.get(name)}];`,
+            `const ${at} = c.idx;`,
+            `const ${end} = ${table}.ends[${at}];`,
+            `if (${end} !== 0) { c.idx = ${end} - 1; ${result} = ${table}.vals[${at}]; }`,
+            'else {',
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`,
+            `${result} = R[${index}](c);`,
+            `if (c.track) c.ruleDepth = ${parent};`,
+            `if (${result} === F) c.idx = ${at};`,
+            `${table}.ends[${at}] = c.idx + 1;`,
+            `${table}.vals[${at}] = ${result};`,
+            '}'
+          )
         } else {
           out.push(
             `const ${parent} = c.ruleDepth;`,
@@ -489,7 +622,7 @@ export function createCodegen (config) {
     if (!Array.isArray(children)) throw new Error("'seq' expects an array of parsers")
 
     // Same idiom recognition as the planner, so both paths agree.
-    const idiom = recognizeSeqIdiom(children)
+    const idiom = idioms ? recognizeSeqIdiom(children) : null
     if (idiom !== null) return gen(idiom, out, null)
 
     let hasLabels = false
@@ -556,7 +689,10 @@ export function createCodegen (config) {
   function genAlt (branches, out, result, bindingsVar) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches")
 
-    const factored = recognizeAltPrefixIdiom(branches)
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null
+    if (oneOf !== null) return gen(oneOf, out, bindingsVar)
+
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null
     if (factored !== null) return gen(factored, out, bindingsVar)
 
     const table = firsts ? buildDispatch(branches, firsts) : null

package/src/definition.js

@@ -227,6 +227,7 @@ const CONSTRUCTORS = {
   many: { arity: [1, 1], of: ([a]) => ({ many: a }) },
   many1: { arity: [1, 1], of: ([a]) => ({ many1: a }) },
   option: { arity: [1, 1], of: ([a]) => ({ option: a }) },
+  flag: { arity: [1, 1], of: ([a]) => ({ flag: a }) },
   action: { arity: [1, 1], of: ([a]) => ({ action: a }) },
   lookahead: { arity: [1, 1], of: ([a]) => ({ lookahead: a }) },
   negLookahead: { arity: [1, 1], of: ([a]) => ({ negLookahead: a }) },
@@ -241,6 +242,10 @@ const CONSTRUCTORS = {
   many1Sep: { arity: [2, 2], of: ([a, b]) => ({ many1Sep: { rule: a, sep: b } }) },
 
   between: { arity: [3, 3], of: ([a, b, c]) => ({ between: { open: a, body: b, close: c } }) },
+  // pick(1, consume(Open), subrule(Body), consume(Close)) — the result at that index.
+  pick: { arity: [2, Infinity], of: ([index, ...parsers]) => ({ pick: { parsers, index } }) },
+  // yields(consume(True), true) — run the parsers, give back the constant.
+  yields: { arity: [2, Infinity], of: (args) => ({ yields: { parsers: args.slice(0, -1), value: args[args.length - 1] } }) },
   infixLeft: { arity: [3, 3], of: ([a, b, c]) => ({ infixLeft: { operand: a, operator: b, combine: logicOf(c) } }) },
   infixRight: { arity: [3, 3], of: ([a, b, c]) => ({ infixRight: { operand: a, operator: b, combine: logicOf(c) } }) },
   prefix: { arity: [3, 3], of: ([a, b, c]) => ({ prefix: { operator: a, operand: b, combine: logicOf(c) } }) },

package/src/emit.js

@@ -2,9 +2,11 @@
 'use strict'
 
 import { createLexer } from './lexer.js'
+import { generateLexer } from './lexgen.js'
 import { analyze } from './analyze.js'
 import { createPlanner } from './plan.js'
 import { createCodegen } from './codegen.js'
+import { selectMemoRules } from './memoplan.js'
 import { usesOuterScope } from './plan.js'
 import { emitMethodsBlock, evaluateMethodsBlock } from './methodsblock.js'
 import { createValueMethods } from './values.js'
@@ -100,8 +102,9 @@ const literal = (value, indent = 0) => JSON.stringify(value, null, 2)
 
 /**
  * @param {import('./parser.js').GrammarSpec} spec
- * @param {{ memo?: boolean, strict?: boolean, ll1?: boolean, maxSteps?: number,
+ * @param {{ memo?: boolean, autoMemo?: boolean, inlineRules?: boolean, idioms?: boolean, strict?: boolean, ll1?: boolean, maxSteps?: number,
  *           positions?: 'full'|'offset', execution?: 'generated'|'interpreted',
+ *           lexer?: 'generated'|'runtime', scanners?: boolean, fuse?: boolean,
  *           format?: 'esm'|'cjs', validate?: boolean, moduleName?: string,
  *           runtimeSpecifier?: string, engineSpecifier?: string,
  *           methods?: Record<string, any> }} [options]
@@ -111,11 +114,21 @@ export function emitModule (spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec
   const {
     memo = false,
+    autoMemo = true,
+    // Off here, on in `createParser`. Inlining a precedence chain is worth 2-3x on a bare chain but only
+    // 1.02-1.04x on the formula and SQL examples, and it makes the file bigger: the formula module grows
+    // 32% minified (38 to 50 KB) even after the second copy of each operand is kept as a call. A parser
+    // built at run time pays nothing for that; a file someone ships does.
+    inlineRules = false,
+    idioms = true,
     strict = true,
     ll1 = true,
     maxSteps = 0,
     positions = 'full',
     execution = 'generated',
+    lexer: lexerMode = 'generated',
+    scanners = true,
+    fuse = true,
     format = 'esm',
     validate = true,
     moduleName = spec.name || 'parser',
@@ -125,6 +138,7 @@ export function emitModule (spec, options = {}) {
   } = options
 
   if (format !== 'esm' && format !== 'cjs') throw new Error(`emitModule: format must be 'esm' or 'cjs', got ${quote(format)}`)
+  if (lexerMode !== 'generated' && lexerMode !== 'runtime') throw new Error(`emitModule: lexer must be 'generated' or 'runtime', got ${quote(lexerMode)}`)
 
   // A `methods { ... }` block travels with the grammar and is written into the
   // output as source. Nothing evaluates it here, so this path stays eval-free.
@@ -132,6 +146,8 @@ export function emitModule (spec, options = {}) {
   const block = methodsBlock ? emitMethodsBlock(methodsBlock, format) : null
   if (!start || !(start in rules)) throw new Error(`emitModule: start rule '${start}' is not defined in rules`)
 
+  // Built here even when the file will carry its own lexer: constructing it is what validates the token
+  // definitions, and it fixes the token ids the rules are compiled against.
   const lexer = createLexer(tokenDefs, { positions })
   const analysis = analyze(rules, { tokenNames: lexer.tokenNames })
   if (validate && analysis.errors.length > 0) throw new GrammarError(analysis.errors)
@@ -144,8 +160,12 @@ export function emitModule (spec, options = {}) {
   // comment in parser.js.
   const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]))
   const engine = { build: () => { throw new Error('not compiled during emit') }, run: () => {} }
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds })
-  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds })
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0
+    ? selectMemoRules(rules, analysis.firsts, { ll1 })
+    : []
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]))
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds, memoSlots, idioms })
+  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms })
 
   codegen.compile(start)
 
@@ -214,7 +234,9 @@ export function emitModule (spec, options = {}) {
   // actions may use the value methods, so it has to register them the way
   // `createParser` does. They come from the runtime rather than being written
   // out here, so there is one implementation rather than two.
-  const runtimeImports = ['FAIL as F', 'expect as E', 'createState', 'buildParseError', 'createLexer']
+  // A lexer written out as source needs only the error class from the runtime; the engine stays out of the file.
+  const generatedLexer = lexerMode === 'generated'
+  const runtimeImports = ['FAIL as F', 'expect as E', 'createState', 'buildParseError', generatedLexer ? 'LexError' : 'createLexer']
   if (needsEngine) runtimeImports.push('createValueMethods')
 
   const importRuntime = cjs
@@ -244,7 +266,7 @@ export function emitModule (spec, options = {}) {
 //
 // The grammar was analyzed when this file was written: FIRST sets, left-recursion
 // checks and rule specialization all happened then. Importing it does no grammar
-// work and generates no code.
+// work and generates no code.${generatedLexer ? '\n// The lexer was written out as source too, so there is no lexer engine to load.' : ''}
 ${engineFree
   ? '// Every semantic action was compiled to source too, so this file has no\n// dependency on json-logic-engine at all.'
   : interpretedActions
@@ -258,7 +280,7 @@ const tokens = ${literal(tokenDefs)}
 
 const start = ${quote(start)}
 
-const lexer = createLexer(tokens, { positions: ${quote(positions)} })
+const lexer = ${generatedLexer ? generateLexer(tokenDefs, { positions, scanners, fuse }) : `createLexer(tokens, { positions: ${quote(positions)}, scanners: ${scanners}, fuse: ${fuse} })`}
 const tokenize = lexer.tokenize
 
 ${engineFree
@@ -357,7 +379,7 @@ const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name,
  * @param {string} [source]
  */
 function parseTokens (tokenList, source, track = false) {
-  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track })
+  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track, memoSlots: ${memoRules.length} })
   state.source = source
   state.positions = ${quote(positions)}
   state.ruleNames[0] = start