grammatik 0.1.0 → 0.1.2

package/dist/esm/index.js

@@ -315,7 +315,338 @@ function firstCharSet(source, ignoreCase) {
   return found;
 }
 
-// src/lexer.js
+// src/scan.js
+var DEFER = -2;
+var CLASS = 0;
+var SEQ2 = 1;
+var KEYWORD = 2;
+var LINE = 3;
+var QUOTED = 4;
+var NUMBER = 5;
+var STRING = 6;
+var SCANNER_KINDS = Object.freeze({ CLASS, SEQ2, KEYWORD, LINE, QUOTED, NUMBER, STRING });
+var NONE = new Uint8Array(128);
+function make(kind, fields) {
+  return {
+    kind,
+    wide: false,
+    first: NONE,
+    rest: NONE,
+    second: NONE,
+    look: NONE,
+    lower: [],
+    upper: [],
+    prefix: "",
+    quote: -1,
+    sign: false,
+    jsonInt: false,
+    frac: false,
+    exp: false,
+    ctrl: false,
+    uni: false,
+    escAny: false,
+    esc: NONE,
+    ...fields
+  };
+}
+var SHORTHAND = {
+  d: (t) => {
+    for (let k = 48; k < 58; k++) t[k] = 1;
+  },
+  w: (t) => {
+    for (let k = 0; k < 128; k++) if (/\w/.test(String.fromCharCode(k))) t[k] = 1;
+  },
+  s: (t) => {
+    for (const k of [9, 10, 11, 12, 13, 32]) t[k] = 1;
+  }
+};
+var SIMPLE_ESCAPES2 = { t: "	", r: "\r", n: "\n", f: "\f", v: "\v" };
+var ALNUM = /[a-zA-Z0-9]/;
+function parseClass(body) {
+  if (body.length === 0 || body[0] === "^") return null;
+  const table = new Uint8Array(128);
+  for (let i = 0; i < body.length; i++) {
+    let c = body[i];
+    if (c === "\\") {
+      const n = body[++i];
+      if (n === "d" || n === "w") {
+        SHORTHAND[n](table);
+        continue;
+      }
+      if (n === void 0) return null;
+      if (ALNUM.test(n) && !(n in SIMPLE_ESCAPES2)) return null;
+      c = SIMPLE_ESCAPES2[n] ?? n;
+    }
+    if (body[i + 1] === "-" && i + 2 < body.length) {
+      let hi = body[i + 2];
+      let advance = 2;
+      if (hi === "\\") {
+        hi = body[i + 3];
+        advance = 3;
+        if (hi === void 0 || ALNUM.test(hi)) return null;
+      }
+      const from = c.charCodeAt(0);
+      const to = hi.charCodeAt(0);
+      if (from > to || to >= 128) return null;
+      for (let k = from; k <= to; k++) table[k] = 1;
+      i += advance;
+      continue;
+    }
+    const code = c.charCodeAt(0);
+    if (code >= 128) return null;
+    table[code] = 1;
+  }
+  return table;
+}
+var BODY = String.raw`((?:[^\]\\]|\\.)*)`;
+var RE_CLASS_RUN = new RegExp(String.raw`^\[${BODY}\]([+*])$`);
+var RE_CLASS_PAIR = new RegExp(String.raw`^\[${BODY}\]\[${BODY}\]\*$`);
+var RE_CLASS_SEQ = new RegExp(String.raw`^\[${BODY}\]\+\[${BODY}\]\+$`);
+var RE_SHORTHAND = /^\\([sdw])[+*]$/;
+var RE_KEYWORD = new RegExp(String.raw`^([A-Za-z0-9_]+)\(\?!\[${BODY}\]\)$`);
+var RE_LINE = new RegExp(String.raw`^([^\\\[\]()*+?.|^$]+)\[\^${BODY}\]\*$`);
+var RE_QUOTED = /^(['"])\(\?:\1\1\|\[\^\1\]\)\*\1$/;
+var RE_NUMBER = /^(-\?)?(?:(\(\?:0\|\[1-9\]\\d\*\))|\\d\+)(\(\?:\\\.\\d\+\)\?)?(\(\?:\[eE\]\[\+-\]\?\\d\+\)\?)?$/;
+var RE_STRING_ESC = new RegExp(String.raw`^(["'])\(\?:\[\^\1\\\\(\\u0000-\\u001f|\\x00-\\x1f)?\]\|\\\\\(\?:\[${BODY}\](\|u\[0-9a-fA-F\]\{4\})?\)\)\*\1$`);
+var RE_STRING_ANY = new RegExp(String.raw`^(["'])\(\?:\[\^\1\\\\\]\|\\\\\.\)\*\1$`);
+function buildScanner(source, ignoreCase) {
+  let m = RE_KEYWORD.exec(source);
+  if (m) {
+    const look = parseClass(m[2]);
+    if (look === null) return null;
+    const word = m[1];
+    const lower = [];
+    const upper = [];
+    for (const ch of word) {
+      lower.push((ignoreCase ? ch.toLowerCase() : ch).charCodeAt(0));
+      upper.push((ignoreCase ? ch.toUpperCase() : ch).charCodeAt(0));
+    }
+    return make(KEYWORD, { look, lower, upper });
+  }
+  if (ignoreCase) return null;
+  m = RE_STRING_ESC.exec(source);
+  if (m) {
+    const esc = parseClass(m[3]);
+    if (esc === null) return null;
+    return make(STRING, { quote: m[1].charCodeAt(0), ctrl: m[2] !== void 0, uni: m[4] !== void 0, esc });
+  }
+  m = RE_STRING_ANY.exec(source);
+  if (m) return make(STRING, { quote: m[1].charCodeAt(0), escAny: true });
+  m = RE_NUMBER.exec(source.replace(/\[0-9\]/g, "\\d"));
+  if (m && (m[1] || m[2] || m[3] || m[4])) {
+    return make(NUMBER, { sign: m[1] !== void 0, jsonInt: m[2] !== void 0, frac: m[3] !== void 0, exp: m[4] !== void 0 });
+  }
+  m = RE_CLASS_RUN.exec(source);
+  if (m) {
+    const table = parseClass(m[1]);
+    return table && make(CLASS, { first: table, rest: table });
+  }
+  m = RE_CLASS_PAIR.exec(source);
+  if (m) {
+    const first = parseClass(m[1]);
+    const rest = parseClass(m[2]);
+    return first && rest && make(CLASS, { first, rest });
+  }
+  m = RE_CLASS_SEQ.exec(source);
+  if (m) {
+    const first = parseClass(m[1]);
+    const second = parseClass(m[2]);
+    if (first && second && first.every((v, i) => !(v && second[i]))) {
+      return make(SEQ2, { first, rest: first, second });
+    }
+    return null;
+  }
+  m = RE_SHORTHAND.exec(source);
+  if (m) {
+    const table = new Uint8Array(128);
+    SHORTHAND[m[1]](table);
+    return make(CLASS, { first: table, rest: table, wide: m[1] === "s" });
+  }
+  m = RE_QUOTED.exec(source);
+  if (m) return make(QUOTED, { quote: m[1].charCodeAt(0) });
+  m = RE_LINE.exec(source);
+  if (m) {
+    const stop = parseClass(m[2]);
+    return stop && make(LINE, { prefix: m[1], look: stop });
+  }
+  return null;
+}
+var skipDigits = (text, i, length) => {
+  while (i < length) {
+    const c = text.charCodeAt(i);
+    if (c < 48 || c > 57) break;
+    i++;
+  }
+  return i;
+};
+var isHex = (c) => c >= 48 && c <= 57 || c >= 65 && c <= 70 || c >= 97 && c <= 102;
+function scanEnd(sc, text, offset, length) {
+  switch (sc.kind) {
+    case CLASS: {
+      const code = text.charCodeAt(offset);
+      if (code >= 128) return sc.wide ? DEFER : -1;
+      if (sc.first[code] !== 1) return -1;
+      const rest = sc.rest;
+      let j = offset + 1;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c >= 128) return sc.wide ? DEFER : j;
+        if (rest[c] !== 1) break;
+        j++;
+      }
+      return j;
+    }
+    case SEQ2: {
+      const c0 = text.charCodeAt(offset);
+      if (c0 >= 128 || sc.first[c0] !== 1) return -1;
+      const { first, second } = sc;
+      let j = offset + 1;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && first[c] === 1) j++;
+        else break;
+      }
+      const mid = j;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && second[c] === 1) j++;
+        else break;
+      }
+      return j > mid ? j : -1;
+    }
+    case KEYWORD: {
+      const { lower, upper } = sc;
+      const n = lower.length;
+      if (offset + n > length) return -1;
+      for (let i = 0; i < n; i++) {
+        const c = text.charCodeAt(offset + i);
+        if (c !== lower[i] && c !== upper[i]) return -1;
+      }
+      const end = offset + n;
+      if (end < length) {
+        const c = text.charCodeAt(end);
+        if (c < 128 && sc.look[c] === 1) return -1;
+      }
+      return end;
+    }
+    case LINE: {
+      if (!text.startsWith(sc.prefix, offset)) return -1;
+      let j = offset + sc.prefix.length;
+      const stop = sc.look;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && stop[c] === 1) break;
+        j++;
+      }
+      return j;
+    }
+    case NUMBER: {
+      let j = offset;
+      if (sc.sign && text.charCodeAt(j) === 45) j++;
+      const c = text.charCodeAt(j);
+      if (sc.jsonInt) {
+        if (c === 48) j++;
+        else if (c >= 49 && c <= 57) j = skipDigits(text, j + 1, length);
+        else return -1;
+      } else {
+        if (!(c >= 48 && c <= 57)) return -1;
+        j = skipDigits(text, j + 1, length);
+      }
+      if (sc.frac && text.charCodeAt(j) === 46) {
+        const k = skipDigits(text, j + 1, length);
+        if (k > j + 1) j = k;
+      }
+      if (sc.exp) {
+        const e = text.charCodeAt(j);
+        if (e === 101 || e === 69) {
+          let k = j + 1;
+          const s = text.charCodeAt(k);
+          if (s === 43 || s === 45) k++;
+          const end = skipDigits(text, k, length);
+          if (end > k) j = end;
+        }
+      }
+      return j;
+    }
+    case STRING: {
+      const q2 = sc.quote;
+      if (text.charCodeAt(offset) !== q2) return -1;
+      let j = offset + 1;
+      for (; ; ) {
+        if (j >= length) return -1;
+        const c = text.charCodeAt(j);
+        if (c === q2) return j + 1;
+        if (c === 92) {
+          const d = text.charCodeAt(j + 1);
+          if (sc.escAny) {
+            if (!(d >= 0) || d === 10 || d === 13 || d === 8232 || d === 8233) return -1;
+            j += 2;
+          } else if (d < 128 && sc.esc[d] === 1) {
+            j += 2;
+          } else if (sc.uni && d === 117 && isHex(text.charCodeAt(j + 2)) && isHex(text.charCodeAt(j + 3)) && isHex(text.charCodeAt(j + 4)) && isHex(text.charCodeAt(j + 5))) {
+            j += 6;
+          } else {
+            return -1;
+          }
+          continue;
+        }
+        if (sc.ctrl && c < 32) return -1;
+        j++;
+      }
+    }
+    default: {
+      const q2 = sc.quote;
+      if (text.charCodeAt(offset) !== q2) return -1;
+      const quote2 = String.fromCharCode(q2);
+      let fallback = -1;
+      let j = offset + 1;
+      for (; ; ) {
+        j = text.indexOf(quote2, j);
+        if (j === -1) return fallback;
+        if (text.charCodeAt(j + 1) !== q2) return j + 1;
+        fallback = j + 1;
+        j += 2;
+      }
+    }
+  }
+}
+
+// src/state.js
+var FAIL = /* @__PURE__ */ Symbol("grammatik.fail");
+var memoTables = (slots, length) => {
+  if (!slots) return null;
+  const tables = new Array(slots);
+  for (let i = 0; i < slots; i++) tables[i] = { ends: new Int32Array(length + 1), vals: new Array(length + 1) };
+  return tables;
+};
+var createState = (tokens, grammar, options = {}) => ({
+  tokens,
+  idx: 0,
+  maxIdx: 0,
+  expected: [],
+  grammar,
+  memo: options.memo ? /* @__PURE__ */ new Map() : null,
+  rm: memoTables(options.memoSlots, tokens.length),
+  // Entering a rule is one array store rather than an object allocation, and
+  // the stack is snapshotted only on a furthest failure — by the time an error
+  // is built it would otherwise have unwound to just the start rule.
+  ruleNames: [],
+  ruleDepth: 0,
+  failureStack: null,
+  track: options.track === true,
+  steps: 0,
+  maxSteps: options.maxSteps || 0
+});
+function expect(state, expectation) {
+  if (state.idx > state.maxIdx) {
+    state.maxIdx = state.idx;
+    if (!state.track) return;
+    state.expected = [];
+    state.failureStack = state.ruleNames.slice(0, state.ruleDepth);
+  }
+  if (!state.track || state.idx !== state.maxIdx) return;
+  if (!state.expected.includes(expectation)) state.expected.push(expectation);
+}
 var LexError = class extends Error {
   /**
    * @param {string} message
@@ -329,6 +660,33 @@ var LexError = class extends Error {
     this.col = position.col;
   }
 };
+var ParseError = class extends Error {
+  /**
+   * @param {string} message
+   * @param {{ token: import('./lexer.js').Token | null, expected: string[], ruleStack: string[] }} details
+   */
+  constructor(message, details) {
+    super(message);
+    this.name = "ParseError";
+    this.token = details.token;
+    this.expected = details.expected;
+    this.ruleStack = details.ruleStack;
+  }
+};
+var describeExpected = (expected) => expected.length === 0 ? "nothing" : expected.length === 1 ? expected[0] : `one of [${expected.join(", ")}]`;
+function buildParseError(state) {
+  const token = state.tokens[state.maxIdx] || null;
+  const expected = [...state.expected].sort();
+  const ruleStack = state.failureStack || state.ruleNames.slice(0, state.ruleDepth);
+  const at = (t) => state.positions === "offset" ? `offset ${t.start}` : `line ${t.line} column ${t.col}`;
+  const found = token ? `--> ${token.image} <-- (${token.type}) at ${at(token)}` : "end of input";
+  return new ParseError(
+    `Expecting ${describeExpected(expected)} but found ${found}`,
+    { token, expected, ruleStack }
+  );
+}
+
+// src/lexer.js
 var DEFAULT_MODE = "default";
 function modesOf(def) {
   if (Array.isArray(def.modes) && def.modes.length > 0) return def.modes;
@@ -338,6 +696,8 @@ function modesOf(def) {
 function createLexer(defs, options = {}) {
   const positions = options.positions === "offset" ? "offset" : "full";
   const trackLines = positions === "full";
+  const useScanners = options.scanners !== false;
+  const useFuse = options.fuse !== false;
   if (!Array.isArray(defs) || defs.length === 0) throw new Error("createLexer: expected a non-empty array of token definitions");
   const tokenNames = defs.map((d) => d.name);
   const seen = /* @__PURE__ */ new Set();
@@ -357,7 +717,7 @@ function createLexer(defs, options = {}) {
     return id;
   };
   for (const def of defs) for (const mode of modesOf(def)) idOf(mode);
-  const compiled = defs.map((def) => {
+  const compiled = defs.map((def, id) => {
     if (!def.name) throw new Error("createLexer: every token definition needs a name");
     if (!def.pattern && !def.literal) throw new Error(`createLexer: token '${def.name}' needs a pattern or a literal`);
     if (def.pattern && def.literal) throw new Error(`createLexer: token '${def.name}' has both a pattern and a literal; pick one`);
@@ -372,13 +732,24 @@ function createLexer(defs, options = {}) {
     }
     return {
       name: def.name,
+      // The declaration index. `createParser` derives the identical mapping
+      // from `tokenNames` independently (same array, same order), so a
+      // combinator that resolves a `consume` target's id at build time agrees
+      // with what a token actually carries without the two ever comparing notes.
+      id,
       skip: !!def.skip,
       literal: def.literal,
       longerAlt: def.longerAlt || null,
       regex: def.pattern ? new RegExp(def.pattern, def.ignoreCase ? "iy" : "y") : null,
+      // A table-driven stand-in for the regex where its shape is recognised;
+      // null otherwise (and always, with `scanners: false`). It is a field on
+      // every definition so they all keep one shape. See scan.js.
+      scanner: useScanners && def.pattern ? buildScanner(def.pattern, !!def.ignoreCase) : null,
       modes: modesOf(def),
       pushMode: def.pushMode ? idOf(def.pushMode) : -1,
-      popMode: !!def.popMode
+      popMode: !!def.popMode,
+      // Set only on the stand-in `fuse` builds for a run of regexes; see there.
+      members: null
     };
   });
   const byName = new Map(compiled.map((c) => [c.name, c]));
@@ -395,14 +766,47 @@ function createLexer(defs, options = {}) {
   for (const c of compiled) {
     c.firstChars = c.literal !== void 0 ? c.literal.length > 0 ? /* @__PURE__ */ new Set([c.literal.charCodeAt(0)]) : null : firstCharSet(c.regex.source, !!c.regex.ignoreCase);
   }
-  const modes = modeNames.map((name) => {
+  const FUSE_MIN = 3;
+  const fusible = (def) => def.literal === void 0 && def.scanner === null && def.regex !== null && !/\\[1-9]|\\k</.test(def.regex.source) && // `exec` of an always-matching `|` alternative returns one slot per capture group.
+  new RegExp(def.regex.source + "|").exec("").length === 1;
+  function fuse(list) {
+    if (!useFuse) return list;
+    const out = [];
+    for (let i = 0; i < list.length; ) {
+      let j = i;
+      while (j < list.length && fusible(list[j]) && list[j].regex.flags === list[i].regex.flags) j++;
+      if (j - i < FUSE_MIN) {
+        out.push(list[i++]);
+        continue;
+      }
+      const run = list.slice(i, j);
+      out.push({
+        name: run.map((d) => d.name).join("|"),
+        id: -1,
+        skip: false,
+        literal: void 0,
+        longerAlt: null,
+        longerAltDef: void 0,
+        regex: new RegExp(run.map((d) => `(${d.regex.source})`).join("|"), run[0].regex.flags),
+        scanner: null,
+        modes: run[0].modes,
+        pushMode: -1,
+        popMode: false,
+        members: run
+      });
+      i = j;
+    }
+    return out;
+  }
+  const modes = modeNames.map((name, id) => {
     const members = compiled.filter((c) => c.modes.includes(name));
     const alwaysTry = members.filter((c) => c.firstChars === null);
-    const candidatesFor = (code) => members.filter((c) => c.firstChars === null || c.firstChars.has(code));
+    const candidatesFor = (code) => fuse(members.filter((c) => c.firstChars === null || c.firstChars.has(code)));
     const ascii = new Array(128);
     for (let code = 0; code < 128; code++) ascii[code] = candidatesFor(code);
     return {
       name,
+      id,
       members,
       ascii,
       wide: /* @__PURE__ */ new Map(),
@@ -426,6 +830,30 @@ function createLexer(defs, options = {}) {
   const multiMode = modes.length > 1 || compiled.some((c) => c.popMode || c.pushMode !== -1);
   const namesModes = modes.length > 1;
   const initial = modes[0];
+  let fusedMember = null;
+  let fusedText = "";
+  function fusedEnd(run, text, offset) {
+    run.regex.lastIndex = offset;
+    const match = run.regex.exec(text);
+    if (match === null) return -1;
+    if (match[0].length === 0) {
+      for (const member of run.members) {
+        member.regex.lastIndex = offset;
+        const one = member.regex.exec(text);
+        if (one !== null && one[0].length > 0) {
+          fusedMember = member;
+          fusedText = one[0];
+          return offset + fusedText.length;
+        }
+      }
+      return -1;
+    }
+    let group = 1;
+    while (match[group] === void 0) group++;
+    fusedMember = run.members[group - 1];
+    fusedText = match[0];
+    return offset + fusedText.length;
+  }
   function longerAltEnd(def, text, offset) {
     const alt2 = def.longerAltDef;
     if (alt2.literal !== void 0) return text.startsWith(alt2.literal, offset) ? offset + alt2.literal.length : -1;
@@ -445,19 +873,30 @@ function createLexer(defs, options = {}) {
     outer: while (offset < length) {
       const applicable = mode.dispatch ? candidates(mode, text.charCodeAt(offset)) : mode.members;
       for (let i = 0; i < applicable.length; i++) {
-        const def = applicable[i];
+        let def = applicable[i];
         let end = -1;
         let matched = null;
-        if (def.literal !== void 0) {
+        if (def.members !== null) {
+          end = fusedEnd(def, text, offset);
+          if (end === -1) continue;
+          def = fusedMember;
+          matched = fusedText;
+        } else if (def.literal !== void 0) {
           if (text.startsWith(def.literal, offset)) end = offset + def.literal.length;
-        } else if (def.skip) {
-          def.regex.lastIndex = offset;
-          if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
         } else {
-          def.regex.lastIndex = offset;
-          const match = def.regex.exec(text);
-          if (match !== null && match[0].length > 0) matched = match[0];
-          if (matched !== null) end = offset + matched.length;
+          end = def.scanner === null ? DEFER : scanEnd(def.scanner, text, offset, length);
+          if (end === DEFER) {
+            end = -1;
+            if (def.skip) {
+              def.regex.lastIndex = offset;
+              if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
+            } else {
+              def.regex.lastIndex = offset;
+              const match = def.regex.exec(text);
+              if (match !== null && match[0].length > 0) matched = match[0];
+              if (matched !== null) end = offset + matched.length;
+            }
+          }
         }
         if (end === -1) continue;
         let winner = def;
@@ -470,14 +909,7 @@ function createLexer(defs, options = {}) {
         }
         if (!winner.skip) {
           const image = winner.literal !== void 0 ? winner.literal : winner === def && matched !== null ? matched : text.slice(offset, end);
-          tokens.push({
-            type: winner.name,
-            image,
-            start: offset,
-            end,
-            line: trackLines ? line : 0,
-            col: trackLines ? offset - lineStart + 1 : 0
-          });
+          tokens.push(trackLines ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 } : { type: winner.name, id: winner.id, image, start: offset, end });
         }
         if (trackLines) {
           for (let j = offset; j < end; j++) {
@@ -513,62 +945,238 @@ function createLexer(defs, options = {}) {
   }
   const inMode = (mode) => namesModes ? ` (in lexer mode '${mode.name}')` : "";
   const describe = (offset, line, lineStart) => trackLines ? `line ${line} column ${offset - lineStart + 1}` : `offset ${offset}`;
-  return { tokenize, tokenNames, positions, modeNames };
-}
-
-// src/state.js
-var FAIL = /* @__PURE__ */ Symbol("grammatik.fail");
-var createState = (tokens, grammar, options = {}) => ({
-  tokens,
-  idx: 0,
-  maxIdx: 0,
-  expected: [],
-  grammar,
-  memo: options.memo ? /* @__PURE__ */ new Map() : null,
-  // Entering a rule is one array store rather than an object allocation, and
-  // the stack is snapshotted only on a furthest failure — by the time an error
-  // is built it would otherwise have unwound to just the start rule.
-  ruleNames: [],
-  ruleDepth: 0,
-  failureStack: null,
-  track: options.track === true,
-  steps: 0,
-  maxSteps: options.maxSteps || 0
-});
-function expect(state, expectation) {
-  if (state.idx > state.maxIdx) {
-    state.maxIdx = state.idx;
-    if (!state.track) return;
-    state.expected = [];
-    state.failureStack = state.ruleNames.slice(0, state.ruleDepth);
+  if (!options.incremental) return { tokenize, tokenNames, positions, modeNames };
+  const recover = !!options.recover;
+  const errorTokenName = options.errorToken || "Error";
+  function snapshotStack(stack, depth) {
+    const ids = new Array(depth);
+    for (let i = 0; i < depth; i++) ids[i] = stack[i].id;
+    return ids;
   }
-  if (!state.track || state.idx !== state.maxIdx) return;
-  if (!state.expected.includes(expectation)) state.expected.push(expectation);
-}
-var ParseError = class extends Error {
-  /**
-   * @param {string} message
-   * @param {{ token: import('./lexer.js').Token | null, expected: string[], ruleStack: string[] }} details
-   */
-  constructor(message, details) {
-    super(message);
-    this.name = "ParseError";
-    this.token = details.token;
-    this.expected = details.expected;
-    this.ruleStack = details.ruleStack;
+  function sameStack(a, b) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
   }
-};
-var describeExpected = (expected) => expected.length === 0 ? "nothing" : expected.length === 1 ? expected[0] : `one of [${expected.join(", ")}]`;
-function buildParseError(state) {
-  const token = state.tokens[state.maxIdx] || null;
-  const expected = [...state.expected].sort();
-  const ruleStack = state.failureStack || state.ruleNames.slice(0, state.ruleDepth);
-  const at = (t) => state.positions === "offset" ? `offset ${t.start}` : `line ${t.line} column ${t.col}`;
-  const found = token ? `--> ${token.image} <-- (${token.type}) at ${at(token)}` : "end of input";
-  return new ParseError(
-    `Expecting ${describeExpected(expected)} but found ${found}`,
-    { token, expected, ruleStack }
-  );
+  function scan(text, offset, mode, stack, depth, line, lineStart) {
+    const tokens = [];
+    const checkpoints = [];
+    const length = text.length;
+    outer: while (offset < length) {
+      checkpoints.push({ modeId: mode.id, stack: snapshotStack(stack, depth), line, lineStart, offset });
+      const applicable = mode.dispatch ? candidates(mode, text.charCodeAt(offset)) : mode.members;
+      for (let i = 0; i < applicable.length; i++) {
+        let def = applicable[i];
+        let end = -1;
+        let matched = null;
+        if (def.members !== null) {
+          end = fusedEnd(def, text, offset);
+          if (end === -1) continue;
+          def = fusedMember;
+          matched = fusedText;
+        } else if (def.literal !== void 0) {
+          if (text.startsWith(def.literal, offset)) end = offset + def.literal.length;
+        } else {
+          end = def.scanner === null ? DEFER : scanEnd(def.scanner, text, offset, length);
+          if (end === DEFER) {
+            end = -1;
+            if (def.skip) {
+              def.regex.lastIndex = offset;
+              if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
+            } else {
+              def.regex.lastIndex = offset;
+              const match = def.regex.exec(text);
+              if (match !== null && match[0].length > 0) matched = match[0];
+              if (matched !== null) end = offset + matched.length;
+            }
+          }
+        }
+        if (end === -1) continue;
+        let winner = def;
+        if (def.longerAltDef) {
+          const altEnd = longerAltEnd(def, text, offset);
+          if (altEnd > end) {
+            end = altEnd;
+            winner = def.longerAltDef;
+          }
+        }
+        if (!winner.skip) {
+          const image = winner.literal !== void 0 ? winner.literal : winner === def && matched !== null ? matched : text.slice(offset, end);
+          tokens.push(trackLines ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 } : { type: winner.name, id: winner.id, image, start: offset, end });
+        }
+        if (trackLines) {
+          for (let j = offset; j < end; j++) {
+            if (text.charCodeAt(j) !== 10) continue;
+            line++;
+            lineStart = j + 1;
+          }
+        }
+        if (multiMode) {
+          if (winner.popMode) {
+            if (depth === 0) {
+              if (!recover) {
+                throw new LexError(
+                  `Token '${winner.name}' popped the initial lexer mode at ${describe(offset, line, lineStart)}`,
+                  { offset, line: trackLines ? line : 0, col: trackLines ? offset - lineStart + 1 : 0 }
+                );
+              }
+            } else {
+              mode = stack[--depth];
+            }
+          } else if (winner.pushMode !== -1) {
+            stack[depth++] = mode;
+            mode = modes[winner.pushMode];
+          }
+        }
+        offset = end;
+        continue outer;
+      }
+      if (recover) {
+        tokens.push(trackLines ? { type: errorTokenName, image: text[offset], start: offset, end: offset + 1, line, col: offset - lineStart + 1 } : { type: errorTokenName, id: -1, image: text[offset], start: offset, end: offset + 1 });
+        if (trackLines && text.charCodeAt(offset) === 10) {
+          line++;
+          lineStart = offset + 1;
+        }
+        offset++;
+        continue outer;
+      }
+      const column = offset - lineStart + 1;
+      throw new LexError(
+        trackLines ? `Unexpected character '${text[offset]}' at line ${line} column ${column}${inMode(mode)}` : `Unexpected character '${text[offset]}' at offset ${offset}${inMode(mode)}`,
+        { offset, line: trackLines ? line : 0, col: trackLines ? column : 0 }
+      );
+    }
+    return { tokens, checkpoints, mode, stack, depth, line, lineStart };
+  }
+  function tokenizeIncremental(text) {
+    if (typeof text !== "string") throw new TypeError("tokenizeIncremental: expected a string");
+    const result = scan(text, 0, initial, multiMode ? [] : [], 0, 1, 0);
+    return { text, tokens: result.tokens, checkpoints: result.checkpoints, eof: { modeId: result.mode.id, stack: snapshotStack(result.stack, result.depth), line: result.line, lineStart: result.lineStart, offset: text.length } };
+  }
+  function tokenFloor(tokens, offset) {
+    let lo = 0;
+    let hi = tokens.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (tokens[mid].start < offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+  function checkpointFloor(checkpoints, eof, offset) {
+    let lo = 0;
+    let hi = checkpoints.length + 1;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      const at = mid < checkpoints.length ? checkpoints[mid].offset : eof.offset;
+      if (at <= offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo - 1;
+  }
+  function checkpointAt(checkpoints, eof, offset) {
+    if (eof.offset === offset) return eof;
+    let lo = 0;
+    let hi = checkpoints.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (checkpoints[mid].offset < offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo < checkpoints.length && checkpoints[lo].offset === offset ? checkpoints[lo] : null;
+  }
+  function relex(newText, prev, edit) {
+    if (typeof newText !== "string") throw new TypeError("relex: expected a string");
+    const tokens = prev.tokens;
+    const checkpoints = prev.checkpoints;
+    const delta = edit.newEnd - edit.oldEnd;
+    let resumeIndex = checkpointFloor(checkpoints, prev.eof, edit.start);
+    if (resumeIndex > 0) resumeIndex--;
+    let boundary = resumeIndex < checkpoints.length ? checkpoints[resumeIndex].offset : prev.eof.offset;
+    let ti = tokenFloor(tokens, boundary);
+    while (ti > 0 && tokens[ti - 1].end === boundary) {
+      ti--;
+      boundary = tokens[ti].start;
+    }
+    if (boundary < (resumeIndex < checkpoints.length ? checkpoints[resumeIndex].offset : prev.eof.offset)) {
+      resumeIndex = checkpointFloor(checkpoints, prev.eof, boundary);
+    }
+    const resume = resumeIndex < checkpoints.length ? checkpoints[resumeIndex] : prev.eof;
+    const resumeMode = modes[resume.modeId];
+    const resumeStack = resume.stack.map((id) => modes[id]);
+    const scanned = scan(newText, resume.offset, resumeMode, resumeStack, resumeStack.length, resume.line, resume.lineStart);
+    const scannedEof = { modeId: scanned.mode.id, stack: snapshotStack(scanned.stack, scanned.depth), line: scanned.line, lineStart: scanned.lineStart, offset: newText.length };
+    const VERIFY_WINDOW = 4;
+    function convergenceVerified(oldOffset, newOffset) {
+      const oldStart = tokenFloor(tokens, oldOffset);
+      const freshStart = tokenFloor(scanned.tokens, newOffset);
+      for (let k = 0; k < VERIFY_WINDOW; k++) {
+        const oldTok = tokens[oldStart + k];
+        const freshTok = scanned.tokens[freshStart + k];
+        if (oldTok === void 0 && freshTok === void 0) return true;
+        if (oldTok === void 0 || freshTok === void 0) return false;
+        if (oldTok.type !== freshTok.type || oldTok.image !== freshTok.image) return false;
+      }
+      return true;
+    }
+    let convergeNewOffset = -1;
+    let convergeOldOffset = -1;
+    let convergeOldCheckpoint = null;
+    let freshCheckpointCount = 0;
+    for (let s = 0; s <= scanned.checkpoints.length; s++) {
+      const cp = s < scanned.checkpoints.length ? scanned.checkpoints[s] : scannedEof;
+      if (cp.offset < edit.newEnd) continue;
+      const oldOffset = cp.offset - delta;
+      const oldCp = checkpointAt(checkpoints, prev.eof, oldOffset);
+      if (oldCp && oldCp.modeId === cp.modeId && sameStack(oldCp.stack, cp.stack) && convergenceVerified(oldOffset, cp.offset)) {
+        convergeNewOffset = cp.offset;
+        convergeOldOffset = oldOffset;
+        convergeOldCheckpoint = oldCp;
+        freshCheckpointCount = s;
+        break;
+      }
+    }
+    const tokenFrom = tokenFloor(tokens, resume.offset);
+    const checkpointFrom = resumeIndex;
+    if (convergeNewOffset === -1) {
+      tokens.splice(tokenFrom, tokens.length - tokenFrom, ...scanned.tokens);
+      checkpoints.splice(checkpointFrom, checkpoints.length - checkpointFrom, ...scanned.checkpoints);
+      prev.text = newText;
+      prev.eof = scannedEof;
+      return prev;
+    }
+    const freshTokenCount = tokenFloor(scanned.tokens, convergeNewOffset);
+    const lineDelta = trackLines ? scanned.checkpoints.length > freshCheckpointCount ? scanned.checkpoints[freshCheckpointCount].line - convergeOldCheckpoint.line : scannedEof.line - convergeOldCheckpoint.line : 0;
+    const boundaryOldLineStart = convergeOldCheckpoint.lineStart;
+    const boundaryNewLineStart = (freshCheckpointCount < scanned.checkpoints.length ? scanned.checkpoints[freshCheckpointCount] : scannedEof).lineStart;
+    const shiftedLineStart = (oldLineStart) => oldLineStart === boundaryOldLineStart ? boundaryNewLineStart : oldLineStart + delta;
+    const tailTokenFrom = tokenFloor(tokens, convergeOldOffset);
+    for (let i = tailTokenFrom; i < tokens.length; i++) {
+      const t = tokens[i];
+      if (trackLines) {
+        const oldLineStart = t.start - (t.col - 1);
+        t.col = t.start + delta - shiftedLineStart(oldLineStart) + 1;
+        t.line += lineDelta;
+      }
+      t.start += delta;
+      t.end += delta;
+    }
+    tokens.splice(tokenFrom, tailTokenFrom - tokenFrom, ...scanned.tokens.slice(0, freshTokenCount));
+    const tailCheckpointFrom = checkpointFloor(checkpoints, prev.eof, convergeOldOffset - 1) + 1;
+    for (let i = tailCheckpointFrom; i < checkpoints.length; i++) {
+      const cp = checkpoints[i];
+      cp.lineStart = shiftedLineStart(cp.lineStart);
+      cp.line += lineDelta;
+      cp.offset += delta;
+    }
+    checkpoints.splice(checkpointFrom, tailCheckpointFrom - checkpointFrom, ...scanned.checkpoints.slice(0, freshCheckpointCount));
+    prev.text = newText;
+    prev.eof.lineStart = shiftedLineStart(prev.eof.lineStart);
+    prev.eof.line += lineDelta;
+    prev.eof.offset += delta;
+    return prev;
+  }
+  return { tokenize, tokenizeIncremental, relex, tokenNames, positions, modeNames };
 }
 
 // src/analyze.js
@@ -601,6 +1209,7 @@ var PARSER_OPS = /* @__PURE__ */ new Set([
   "many",
   "many1",
   "option",
+  "flag",
   "manySep",
   "many1Sep",
   "infixLeft",
@@ -608,6 +1217,8 @@ var PARSER_OPS = /* @__PURE__ */ new Set([
   "postfix",
   "prefix",
   "between",
+  "pick",
+  "yields",
   "as",
   "oneOf",
   "eof",
@@ -661,7 +1272,8 @@ function firstOf(node, ruleFirsts) {
       return result;
     }
     case "many":
-    case "option": {
+    case "option":
+    case "flag": {
       const inner = firstOf(arg, ruleFirsts);
       return { tokens: inner.tokens, nullable: true, unknown: inner.unknown };
     }
@@ -684,6 +1296,9 @@ function firstOf(node, ruleFirsts) {
       return firstOf(Array.isArray(arg) ? arg[0] : arg && arg.parser, ruleFirsts);
     case "between":
       return firstOf({ seq: [arg && arg.open, arg && arg.body, arg && arg.close] }, ruleFirsts);
+    case "pick":
+    case "yields":
+      return firstOf({ seq: arg && Array.isArray(arg.parsers) ? arg.parsers : [] }, ruleFirsts);
     case "prefix": {
       const ops = firstOf(arg && arg.operator, ruleFirsts);
       const inner = firstOf(arg && arg.operand, ruleFirsts);
@@ -747,12 +1362,94 @@ function recognizeSeqIdiom(children) {
       if (rewritten !== BAIL) return { as: [parser, rewritten] };
     }
   }
+  if (children.length >= 2) {
+    const last = children[children.length - 1];
+    const head = children.slice(0, -1);
+    if (opOf(last) === "action" && isConstant(last.action) && head.every(isPlainParser) && !head.some(usesOuterScope)) {
+      return { yields: { parsers: head, value: last.action } };
+    }
+    if (opOf(last) === "action" && opOf(last.action) === "val" && typeof last.action.val === "string") {
+      const name = last.action.val;
+      const labelled = head.map((child, i) => opOf(child) === "label" ? i : -1).filter((i) => i !== -1);
+      if (labelled.length === 1 && head[labelled[0]].label[0] === name && head.every((child, i) => i === labelled[0] || isPlainParser(child)) && !head.some(usesOuterScope)) {
+        return { pick: { parsers: head.map((child, i) => i === labelled[0] ? child.label[1] : child), index: labelled[0] } };
+      }
+    }
+  }
   return null;
 }
+var isConstant = (value) => value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+function usesOuterScope(logic) {
+  if (Array.isArray(logic)) return logic.some(usesOuterScope);
+  if (!logic || typeof logic !== "object") return false;
+  for (const key of Object.keys(logic)) {
+    const arg = logic[key];
+    if (key === "val" && Array.isArray(arg) && Array.isArray(arg[0])) return true;
+    if (key === "var") {
+      if (typeof arg === "string" && arg.includes("../")) return true;
+      if (Array.isArray(arg) && typeof arg[0] === "string" && arg[0].includes("../")) return true;
+    }
+    if (usesOuterScope(arg)) return true;
+  }
+  return false;
+}
 function isPlainParser(node) {
   const op = opOf(node);
   return op !== null && op !== "label" && op !== "action";
 }
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+    return true;
+  }
+  if (a === null || b === null || typeof a !== "object" || typeof b !== "object") return false;
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+  for (const key of keysA) {
+    if (!Object.prototype.hasOwnProperty.call(b, key) || !deepEqual(a[key], b[key])) return false;
+  }
+  return true;
+}
+function recognizeAltPrefixIdiom(branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null;
+  if (!branches.every((b) => opOf(b) === "seq" && Array.isArray(b.seq))) return null;
+  const seqs = branches.map((b) => b.seq);
+  const minLen = Math.min(...seqs.map((s) => s.length));
+  let n = 0;
+  while (n < minLen) {
+    const candidate = seqs[0][n];
+    const op = opOf(candidate);
+    if (op === "label" || op === "action") break;
+    if (!seqs.every((s) => deepEqual(s[n], candidate))) break;
+    n++;
+  }
+  if (n === 0) return null;
+  if (!seqs.every((s) => s.slice(n).some((child) => opOf(child) === "action"))) return null;
+  const prefix = seqs[0].slice(0, n);
+  const remainders = seqs.map((s) => ({ seq: s.slice(n) }));
+  return {
+    seq: [
+      ...prefix,
+      { label: ["__grammatik_rest", { alt: remainders }] },
+      { action: { val: "__grammatik_rest" } }
+    ]
+  };
+}
+function recognizeAltConsumeIdiom(branches) {
+  if (!Array.isArray(branches) || branches.length < 2) return null;
+  const names = [];
+  for (const branch of branches) {
+    if (opOf(branch) !== "consume") return null;
+    const arg = branch.consume;
+    const name = typeof arg === "string" ? arg : Array.isArray(arg) && arg.length === 1 ? arg[0] : null;
+    if (typeof name !== "string") return null;
+    names.push(name);
+  }
+  return { oneOf: names };
+}
 var BAIL = /* @__PURE__ */ Symbol("grammatik.bail");
 function rebaseOnLabel(logic, name) {
   if (Array.isArray(logic)) {
@@ -819,6 +1516,7 @@ function leftmostCalls(node, ruleFirsts, out) {
     case "many":
     case "many1":
     case "option":
+    case "flag":
     case "lookahead":
     case "negLookahead":
       leftmostCalls(arg, ruleFirsts, out);
@@ -829,6 +1527,10 @@ function leftmostCalls(node, ruleFirsts, out) {
     case "between":
       if (arg) leftmostCalls({ seq: [arg.open, arg.body, arg.close] }, ruleFirsts, out);
       break;
+    case "pick":
+    case "yields":
+      if (arg && Array.isArray(arg.parsers)) leftmostCalls({ seq: arg.parsers }, ruleFirsts, out);
+      break;
     case "prefix":
       if (arg) {
         leftmostCalls(arg.operator, ruleFirsts, out);
@@ -895,6 +1597,7 @@ function validateNode(node, grammar, tokenNames, errors, path) {
     case "many":
     case "many1":
     case "option":
+    case "flag":
     case "text":
     case "lookahead":
     case "negLookahead":
@@ -916,6 +1619,26 @@ function validateNode(node, grammar, tokenNames, errors, path) {
       }
       break;
     }
+    case "pick": {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0) {
+        errors.push(`${path}: 'pick' expects { parsers: [parser, ...], index }`);
+      } else if (!Number.isInteger(arg.index) || arg.index < 0 || arg.index >= arg.parsers.length) {
+        errors.push(`${path}: 'pick' index ${JSON.stringify(arg.index)} is out of range for ${arg.parsers.length} parser(s)`);
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.pick[${i}]`));
+      }
+      break;
+    }
+    case "yields": {
+      if (!arg || !Array.isArray(arg.parsers) || arg.parsers.length === 0 || !("value" in arg)) {
+        errors.push(`${path}: 'yields' expects { parsers: [parser, ...], value }`);
+      } else if (!isConstant(arg.value)) {
+        errors.push(`${path}: 'yields' value must be a string, number, boolean or null, got ${JSON.stringify(arg.value)}`);
+      } else {
+        arg.parsers.forEach((c, i) => validateNode(c, grammar, tokenNames, errors, `${path}.yields[${i}]`));
+      }
+      break;
+    }
     case "prefix": {
       if (!arg || typeof arg !== "object" || !("operator" in arg) || !("operand" in arg) || !("combine" in arg)) {
         errors.push(`${path}: 'prefix' expects { operator, operand, combine }`);
@@ -1002,16 +1725,38 @@ function analyze(grammar, options = {}) {
   return { firsts, errors, leftRecursive };
 }
 function buildDispatch(branches, firsts) {
-  const table = /* @__PURE__ */ new Map();
-  for (const branch of branches) {
-    const first = firstOf(branch, firsts);
+  const perBranch = branches.map((branch) => firstOf(branch, firsts));
+  for (const first of perBranch) {
     if (first.nullable || first.unknown || first.tokens.size === 0) return null;
-    for (const token of first.tokens) {
-      if (table.has(token)) return null;
-      table.set(token, branch);
+  }
+  const byToken = /* @__PURE__ */ new Map();
+  for (let i = 0; i < branches.length; i++) {
+    for (const token of perBranch[i].tokens) {
+      let list = byToken.get(token);
+      if (list === void 0) {
+        list = [];
+        byToken.set(token, list);
+      }
+      list.push(branches[i]);
+    }
+  }
+  if (byToken.size === 0) return null;
+  const canonical = /* @__PURE__ */ new Map();
+  const table = /* @__PURE__ */ new Map();
+  for (const [token, list] of byToken) {
+    if (list.length === 1) {
+      table.set(token, list[0]);
+      continue;
+    }
+    const key = list.map((branch) => branches.indexOf(branch)).join(",");
+    let group = canonical.get(key);
+    if (group === void 0) {
+      group = list;
+      canonical.set(key, group);
     }
+    table.set(token, group);
   }
-  return table.size > 0 ? table : null;
+  return table;
 }
 
 // src/plan.js
@@ -1020,20 +1765,6 @@ function declareSync(fn) {
   fn[SYNC] = true;
   return fn;
 }
-function usesOuterScope(logic) {
-  if (Array.isArray(logic)) return logic.some(usesOuterScope);
-  if (!logic || typeof logic !== "object") return false;
-  for (const key of Object.keys(logic)) {
-    const arg = logic[key];
-    if (key === "val" && Array.isArray(arg) && Array.isArray(arg[0])) return true;
-    if (key === "var") {
-      if (typeof arg === "string" && arg.includes("../")) return true;
-      if (Array.isArray(arg) && typeof arg[0] === "string" && arg[0].includes("../")) return true;
-    }
-    if (usesOuterScope(arg)) return true;
-  }
-  return false;
-}
 function mayReturnContext(logic) {
   if (Array.isArray(logic)) return logic.some(mayReturnContext);
   if (!logic || typeof logic !== "object") return false;
@@ -1046,7 +1777,7 @@ function mayReturnContext(logic) {
   return false;
 }
 function createPlanner(config) {
-  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true } = config;
+  const { engine, rules, firsts, memo, maxSteps, unsafeEval = true, tokenIds = null, memoSlots = /* @__PURE__ */ new Map(), idioms = true } = config;
   const slots = /* @__PURE__ */ new Map();
   const planCache = /* @__PURE__ */ new WeakMap();
   const opCaches = /* @__PURE__ */ new Map();
@@ -1129,9 +1860,10 @@ function createPlanner(config) {
   }
   const buildConsume = (arg) => {
     const name = typeof arg === "string" ? arg : arg[0];
+    const id = tokenIds ? tokenIds.get(name) : void 0;
     return (state) => {
       const token = state.tokens[state.idx];
-      if (token !== void 0 && token.type === name) {
+      if (token !== void 0 && (token.id === id || token.id === void 0 && token.type === name)) {
         state.idx++;
         return token;
       }
@@ -1192,6 +1924,17 @@ function createPlanner(config) {
       return value;
     };
   };
+  const buildFlag = (arg) => {
+    const child = plan(arg);
+    return (state) => {
+      const start = state.idx;
+      if (child(state) === FAIL) {
+        state.idx = start;
+        return false;
+      }
+      return true;
+    };
+  };
   const buildInfixLeft = (arg) => {
     const operand = plan(arg.operand);
     const operator = arg.operator === void 0 ? null : plan(arg.operator);
@@ -1227,10 +1970,11 @@ function createPlanner(config) {
   const buildOneOf = (arg) => {
     const names = Array.isArray(arg) ? arg : [arg];
     const set = new Set(names);
+    const idSet = tokenIds ? new Set(names.map((n) => tokenIds.get(n))) : null;
     const count = names.length;
     return (state) => {
       const token = state.tokens[state.idx];
-      if (token !== void 0 && set.has(token.type)) {
+      if (token !== void 0 && (idSet ? token.id !== void 0 ? idSet.has(token.id) : set.has(token.type) : set.has(token.type))) {
         state.idx++;
         return token;
       }
@@ -1316,6 +2060,26 @@ function createPlanner(config) {
       return value;
     };
   };
+  const buildSequence = (arg, pickIndex, constant) => {
+    const fns = arg.parsers.map(plan);
+    const count = fns.length;
+    return (state) => {
+      tick(state);
+      const start = state.idx;
+      let picked;
+      for (let i = 0; i < count; i++) {
+        const value = fns[i](state);
+        if (value === FAIL) {
+          state.idx = start;
+          return FAIL;
+        }
+        if (i === pickIndex) picked = value;
+      }
+      return pickIndex === -1 ? constant : picked;
+    };
+  };
+  const buildPick = (arg) => buildSequence(arg, arg.index, void 0);
+  const buildYields = (arg) => buildSequence(arg, -1, arg.value);
   const buildPrefix = (arg) => {
     const operator = plan(arg.operator);
     const operand = plan(arg.operand);
@@ -1399,6 +2163,31 @@ function createPlanner(config) {
         return value;
       };
     }
+    const memoSlot = memoSlots.get(name);
+    if (memoSlot !== void 0) {
+      return (state) => {
+        tick(state);
+        const table = state.rm[memoSlot];
+        const at = state.idx;
+        const end = table.ends[at];
+        if (end !== 0) {
+          state.idx = end - 1;
+          return table.vals[at];
+        }
+        let depth = 0;
+        if (state.track) {
+          depth = state.ruleDepth;
+          state.ruleNames[depth] = name;
+          state.ruleDepth = depth + 1;
+        }
+        const value = slot.fn(state);
+        if (state.track) state.ruleDepth = depth;
+        if (value === FAIL) state.idx = at;
+        table.ends[at] = state.idx + 1;
+        table.vals[at] = value;
+        return value;
+      };
+    }
     return (state) => {
       tick(state);
       if (!state.track) return slot.fn(state);
@@ -1477,6 +2266,9 @@ function createPlanner(config) {
     infixRight: buildInfixRight,
     as: buildAs,
     between: buildBetween,
+    flag: buildFlag,
+    pick: buildPick,
+    yields: buildYields,
     prefix: buildPrefix,
     postfix: buildPostfix,
     manySep: (arg) => planSeparated(arg, false),
@@ -1496,7 +2288,7 @@ function createPlanner(config) {
   }
   function planSeq(children) {
     if (!Array.isArray(children)) throw new Error("'seq' expects an array of parsers");
-    const idiom = recognizeSeqIdiom(children);
+    const idiom = idioms ? recognizeSeqIdiom(children) : null;
     if (idiom !== null) return planUncached(idiom);
     const PARSER = 0;
     const LABELLED = 1;
@@ -1506,6 +2298,7 @@ function createPlanner(config) {
     const kinds = [];
     const names = [];
     const tokens = [];
+    const tokenIdList = [];
     const fns = [];
     const actionLogic = [];
     let hasAction = false;
@@ -1526,11 +2319,13 @@ function createPlanner(config) {
           kinds.push(LABELLED_TERMINAL);
           names.push(name);
           tokens.push(terminal2);
+          tokenIdList.push(tokenIds ? tokenIds.get(terminal2) : void 0);
           fns.push(null);
         } else {
           kinds.push(LABELLED);
           names.push(name);
           tokens.push(null);
+          tokenIdList.push(void 0);
           fns.push(plan(inner));
         }
         continue;
@@ -1539,6 +2334,7 @@ function createPlanner(config) {
         kinds.push(ACTION);
         names.push(null);
         tokens.push(null);
+        tokenIdList.push(void 0);
         fns.push(planAction(child.action));
         actionLogic.push(child.action);
         hasAction = true;
@@ -1549,11 +2345,13 @@ function createPlanner(config) {
         kinds.push(TERMINAL);
         names.push(null);
         tokens.push(terminal);
+        tokenIdList.push(tokenIds ? tokenIds.get(terminal) : void 0);
         fns.push(null);
       } else {
         kinds.push(PARSER);
         names.push(null);
         tokens.push(null);
+        tokenIdList.push(void 0);
         fns.push(plan(child));
       }
     }
@@ -1574,7 +2372,7 @@ function createPlanner(config) {
         let value;
         if (kind === TERMINAL || kind === LABELLED_TERMINAL) {
           const token = state.tokens[state.idx];
-          if (token !== void 0 && token.type === tokens[i]) {
+          if (token !== void 0 && (token.id === tokenIdList[i] || token.id === void 0 && token.type === tokens[i])) {
             state.idx++;
             value = token;
           } else {
@@ -1604,18 +2402,46 @@ function createPlanner(config) {
   }
   function planAlt(branches) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches");
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null;
+    if (oneOf !== null) return planUncached(oneOf);
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null;
+    if (factored !== null) return planUncached(factored);
     const planned = branches.map(plan);
     const table = firsts ? buildDispatch(branches, firsts) : null;
     if (table !== null) {
       const byToken = /* @__PURE__ */ Object.create(null);
-      for (const [token, branch] of table) byToken[token] = planned[branches.indexOf(branch)];
+      const byId = tokenIds ? [] : null;
+      const groupClosures = /* @__PURE__ */ new Map();
+      for (const [token, branch] of table) {
+        let closure = groupClosures.get(branch);
+        if (closure === void 0) {
+          closure = Array.isArray(branch) ? (() => {
+            const group = branch.map((b) => planned[branches.indexOf(b)]);
+            return (state) => {
+              const start = state.idx;
+              for (let i = 0; i < group.length; i++) {
+                const value = group[i](state);
+                if (value !== FAIL) return value;
+                state.idx = start;
+              }
+              return FAIL;
+            };
+          })() : planned[branches.indexOf(branch)];
+          groupClosures.set(branch, closure);
+        }
+        byToken[token] = closure;
+        if (byId) {
+          const id = tokenIds.get(token);
+          if (id !== void 0) byId[id] = closure;
+        }
+      }
       const expectedTokens = [...table.keys()];
       const expectedCount = expectedTokens.length;
       return (state) => {
         tick(state);
         const token = state.tokens[state.idx];
         if (token !== void 0) {
-          const branch = byToken[token.type];
+          const branch = byId && token.id !== void 0 ? byId[token.id] : byToken[token.type];
           if (branch !== void 0) return branch(state);
         }
         for (let i = 0; i < expectedCount; i++) expect(state, expectedTokens[i]);
@@ -1700,6 +2526,9 @@ var OPERATORS = [
   "prefix",
   "postfix",
   "between",
+  "pick",
+  "yields",
+  "flag",
   "as",
   "subrule",
   "label",
@@ -1754,7 +2583,12 @@ var createValueMethods = () => ({
 
 // src/codegen.js
 function createCodegen(config) {
-  const { planner, rules, firsts, memo, maxSteps } = config;
+  const { planner, rules, firsts, memo, maxSteps, tokenIds = null, memoSlots = /* @__PURE__ */ new Map(), inlineRules = true, idioms = true } = config;
+  const typeCheck = (tokenVar, name) => {
+    const id = tokenIds ? tokenIds.get(name) : void 0;
+    if (id === void 0) return `${tokenVar}.type === ${JSON.stringify(name)}`;
+    return `(${tokenVar}.id === ${id} || (${tokenVar}.id === undefined && ${tokenVar}.type === ${JSON.stringify(name)}))`;
+  };
   const HELPERS = [];
   const ORIGINS = [];
   const RULES = [];
@@ -1770,6 +2604,33 @@ function createCodegen(config) {
   }
   const actionHelper = (logic) => helper(planner.planAction(logic), { kind: "action", logic });
   const constantHelper = (node) => helper(planner.plan(node), { kind: "constant", node });
+  const MAX_INLINE_DEPTH = 6;
+  const ONE_OF_SWITCH_AT = 12;
+  const INLINE_OPS = /* @__PURE__ */ new Set(["infixLeft", "infixRight", "prefix", "postfix"]);
+  const callSites = /* @__PURE__ */ new Map();
+  (function count(node) {
+    if (Array.isArray(node)) {
+      for (const child of node) count(child);
+      return;
+    }
+    if (!node || typeof node !== "object") return;
+    if ("subrule" in node) {
+      const target = Array.isArray(node.subrule) ? node.subrule[0] : node.subrule;
+      if (typeof target === "string") callSites.set(target, (callSites.get(target) || 0) + 1);
+    }
+    for (const key of Object.keys(node)) count(node[key]);
+  })(rules);
+  const inlining = /* @__PURE__ */ new Set();
+  let coldDepth = 0;
+  const cold = (write) => {
+    coldDepth++;
+    try {
+      return write();
+    } finally {
+      coldDepth--;
+    }
+  };
+  const canInline = (name) => inlineRules && coldDepth === 0 && !memo && !memoSlots.has(name) && !inlining.has(name) && inlining.size < MAX_INLINE_DEPTH && callSites.get(name) === 1 && INLINE_OPS.has(opOf(rules[name]));
   function indexOfRule(name) {
     let index = ruleIndex.get(name);
     if (index !== void 0) return index;
@@ -1796,7 +2657,7 @@ function createCodegen(config) {
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
-          `if (${token} !== undefined && ${token}.type === ${JSON.stringify(name)}) { c.idx++; ${result} = ${token}; }`,
+          `if (${token} !== undefined && ${typeCheck(token, name)}) { c.idx++; ${result} = ${token}; }`,
           `else { E(c, ${JSON.stringify(name)}); ${result} = F; }`
         );
         return result;
@@ -1830,6 +2691,16 @@ function createCodegen(config) {
         );
         return result;
       }
+      case "flag": {
+        const start = nextId();
+        out.push(`let ${result};`, `const ${start} = c.idx;`, "{");
+        const value = gen(arg, out, bindingsVar);
+        out.push(
+          `if (${value} === F) { c.idx = ${start}; ${result} = false; } else ${result} = true;`,
+          "}"
+        );
+        return result;
+      }
       case "many":
       case "many1": {
         const atLeastOne = op === "many1";
@@ -1847,7 +2718,7 @@ function createCodegen(config) {
           );
         }
         out.push("for (;;) {", `const ${before} = c.idx;`);
-        const value = gen(arg, out, bindingsVar);
+        const value = atLeastOne ? cold(() => gen(arg, out, bindingsVar)) : gen(arg, out, bindingsVar);
         out.push(
           `if (${value} === F) { c.idx = ${before}; break; }`,
           `if (c.idx === ${before}) P(${JSON.stringify(op)}, ${before});`,
@@ -1877,7 +2748,7 @@ function createCodegen(config) {
           operatorVar = gen(arg.operator, out, bindingsVar);
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`);
         }
-        const right = gen(arg.operand, out, bindingsVar);
+        const right = cold(() => gen(arg.operand, out, bindingsVar));
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${bindings} === null) ${bindings} = { left: null, op: null, right: null };`,
@@ -1892,12 +2763,27 @@ function createCodegen(config) {
       case "oneOf": {
         const names = Array.isArray(arg) ? arg : [arg];
         const token = nextId();
-        const test = names.map((n) => `${token}.type === ${JSON.stringify(n)}`).join(" || ");
+        const ids = tokenIds ? names.map((n) => tokenIds.get(n)) : [];
+        const failed = `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(" ")} ${result} = F; }`;
+        if (names.length > ONE_OF_SWITCH_AT && ids.every((id) => id !== void 0)) {
+          const hit = nextId();
+          const cases = [...new Set(ids)].map((id) => `case ${id}:`).join(" ");
+          out.push(
+            `let ${result};`,
+            `const ${token} = c.tokens[c.idx];`,
+            `let ${hit} = false;`,
+            `if (${token} !== undefined) { switch (${token}.id !== undefined ? ${token}.id : I[${token}.type]) { ${cases} ${hit} = true; } }`,
+            `if (${hit}) { c.idx++; ${result} = ${token}; }`,
+            failed
+          );
+          return result;
+        }
+        const test = names.map((n) => typeCheck(token, n)).join(" || ");
         out.push(
           `let ${result};`,
           `const ${token} = c.tokens[c.idx];`,
           `if (${token} !== undefined && (${test})) { c.idx++; ${result} = ${token}; }`,
-          `else { ${names.map((n) => `E(c, ${JSON.stringify(n)});`).join(" ")} ${result} = F; }`
+          failed
         );
         return result;
       }
@@ -1930,7 +2816,7 @@ function createCodegen(config) {
           operatorVar = gen(arg.operator, out, bindingsVar);
           out.push(`if (${operatorVar} === F) { c.idx = ${mark}; break; }`);
         }
-        const right = gen(arg.operand, out, bindingsVar);
+        const right = cold(() => gen(arg.operand, out, bindingsVar));
         out.push(
           `if (${right} === F) { c.idx = ${mark}; break; }`,
           `if (${items} === null) ${items} = [];`,
@@ -2005,6 +2891,19 @@ function createCodegen(config) {
         );
         return result;
       }
+      case "pick":
+      case "yields": {
+        const label2 = "L" + nextId();
+        const start = nextId();
+        out.push(tickSrc, `let ${result};`, `${label2}: {`, `const ${start} = c.idx;`);
+        const values = arg.parsers.map((parser) => {
+          const value = gen(parser, out, bindingsVar);
+          out.push(`if (${value} === F) { c.idx = ${start}; ${result} = F; break ${label2}; }`);
+          return value;
+        });
+        out.push(`${result} = ${op === "pick" ? values[arg.index] : JSON.stringify(arg.value)};`, "}");
+        return result;
+      }
       case "prefix": {
         const label2 = "L" + nextId();
         const start = nextId();
@@ -2081,7 +2980,7 @@ function createCodegen(config) {
         );
         const sep = gen(sepNode, out, bindingsVar);
         out.push(`if (${sep} === F) { c.idx = ${beforeSep}; break; }`, `const ${beforeItem} = c.idx;`);
-        const item = gen(arg.rule, out, bindingsVar);
+        const item = cold(() => gen(arg.rule, out, bindingsVar));
         out.push(
           `if (${item} === F) { c.idx = ${trailing ? beforeItem : beforeSep}; break; }`,
           `${result}.push(${item});`,
@@ -2096,8 +2995,20 @@ function createCodegen(config) {
           out.push(`const ${result} = ${helper(planner.plan(node), { kind: "unknownRule", name })}(c);`);
           return result;
         }
-        const index = indexOfRule(name);
         const parent = nextId();
+        if (canInline(name)) {
+          out.push(tickSrc);
+          out.push(
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`
+          );
+          inlining.add(name);
+          const inner = gen(rules[name], out, null);
+          inlining.delete(name);
+          out.push(`if (c.track) c.ruleDepth = ${parent};`);
+          return inner;
+        }
+        const index = indexOfRule(name);
         out.push(tickSrc);
         if (memo) {
           const key = nextId();
@@ -2118,6 +3029,26 @@ function createCodegen(config) {
             `c.memo.set(${key}, { value: ${result}, end: c.idx });`,
             "}"
           );
+        } else if (memoSlots.has(name)) {
+          const table = nextId();
+          const at = nextId();
+          const end = nextId();
+          out.push(
+            `let ${result};`,
+            `const ${table} = c.rm[${memoSlots.get(name)}];`,
+            `const ${at} = c.idx;`,
+            `const ${end} = ${table}.ends[${at}];`,
+            `if (${end} !== 0) { c.idx = ${end} - 1; ${result} = ${table}.vals[${at}]; }`,
+            "else {",
+            `const ${parent} = c.ruleDepth;`,
+            `if (c.track) { c.ruleNames[${parent}] = ${JSON.stringify(name)}; c.ruleDepth = ${parent} + 1; }`,
+            `${result} = R[${index}](c);`,
+            `if (c.track) c.ruleDepth = ${parent};`,
+            `if (${result} === F) c.idx = ${at};`,
+            `${table}.ends[${at}] = c.idx + 1;`,
+            `${table}.vals[${at}] = ${result};`,
+            "}"
+          );
         } else {
           out.push(
             `const ${parent} = c.ruleDepth;`,
@@ -2135,7 +3066,7 @@ function createCodegen(config) {
   }
   function genSeq(children, out, result) {
     if (!Array.isArray(children)) throw new Error("'seq' expects an array of parsers");
-    const idiom = recognizeSeqIdiom(children);
+    const idiom = idioms ? recognizeSeqIdiom(children) : null;
     if (idiom !== null) return gen(idiom, out, null);
     let hasLabels = false;
     let hasAction = false;
@@ -2171,24 +3102,53 @@ function createCodegen(config) {
     out.push(`${result} = ${hasAction ? actionResult : "[" + collected.join(", ") + "]"};`, "}");
     return result;
   }
+  function genOrderedBacktrack(list, out, bindingsVar) {
+    const result = nextId();
+    const label2 = "L" + nextId();
+    const start = nextId();
+    out.push(`let ${result};`, `${label2}: {`, `const ${start} = c.idx;`);
+    for (const branch of list) {
+      out.push("{");
+      const value = gen(branch, out, bindingsVar);
+      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label2}; }`, `c.idx = ${start};`, "}");
+    }
+    out.push(`${result} = F;`, "}");
+    return result;
+  }
   function genAlt(branches, out, result, bindingsVar) {
     if (!Array.isArray(branches)) throw new Error("'alt' expects an array of branches");
+    const oneOf = idioms ? recognizeAltConsumeIdiom(branches) : null;
+    if (oneOf !== null) return gen(oneOf, out, bindingsVar);
+    const factored = idioms ? recognizeAltPrefixIdiom(branches) : null;
+    if (factored !== null) return gen(factored, out, bindingsVar);
     const table = firsts ? buildDispatch(branches, firsts) : null;
     const label2 = "L" + nextId();
     if (table !== null) {
       const expected = [...table.keys()];
+      const expectedIds = tokenIds ? expected.map((t) => tokenIds.get(t)) : null;
+      const numeric = expectedIds !== null && expectedIds.every((id) => id !== void 0);
       const token = nextId();
       out.push(tickSrc, `let ${result};`, `${label2}: {`, `const ${token} = c.tokens[c.idx];`);
       out.push(`if (${token} === undefined) { ${expected.map((t) => `E(c, ${JSON.stringify(t)});`).join(" ")} ${result} = F; break ${label2}; }`);
-      out.push(`switch (${token}.type) {`);
+      let discriminant;
+      if (numeric) {
+        discriminant = nextId();
+        out.push(`const ${discriminant} = ${token}.id !== undefined ? ${token}.id : I[${token}.type];`);
+        out.push(`switch (${discriminant}) {`);
+      } else {
+        discriminant = `${token}.type`;
+        out.push(`switch (${discriminant}) {`);
+      }
       const byBranch = /* @__PURE__ */ new Map();
-      for (const [tokenName, branch] of table) {
+      for (const tokenName of expected) {
+        const branch = table.get(tokenName);
         if (!byBranch.has(branch)) byBranch.set(branch, []);
         byBranch.get(branch).push(tokenName);
       }
+      const caseLabel = (t) => numeric ? tokenIds.get(t) : JSON.stringify(t);
       for (const [branch, tokenNames] of byBranch) {
-        out.push(tokenNames.map((t) => `case ${JSON.stringify(t)}:`).join(" ") + " {");
-        const value = gen(branch, out, bindingsVar);
+        out.push(tokenNames.map((t) => `case ${caseLabel(t)}:`).join(" ") + " {");
+        const value = Array.isArray(branch) ? genOrderedBacktrack(branch, out, bindingsVar) : gen(branch, out, bindingsVar);
         out.push(`${result} = ${value}; break ${label2};`, "}");
       }
       out.push(
@@ -2198,16 +3158,10 @@ function createCodegen(config) {
       );
       return result;
     }
-    const start = nextId();
-    out.push(tickSrc, `let ${result};`, `${label2}: {`, `const ${start} = c.idx;`);
-    for (const branch of branches) {
-      out.push("{");
-      const value = gen(branch, out, bindingsVar);
-      out.push(`if (${value} !== F) { ${result} = ${value}; break ${label2}; }`, `c.idx = ${start};`, "}");
-    }
-    out.push(`${result} = F;`, "}");
-    return result;
+    out.push(tickSrc);
+    return genOrderedBacktrack(branches, out, bindingsVar);
   }
+  const idByName = tokenIds ? Object.fromEntries(tokenIds) : null;
   function buildRule(name) {
     const out = [];
     const value = gen(rules[name], out, null);
@@ -2217,8 +3171,8 @@ return ${value};
 }`;
     SOURCES[ruleIndex.get(name)] = body;
     const source = `return ${body}`;
-    const factory = new Function("F", "E", "P", "T", "R", "H", source);
-    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS);
+    const factory = new Function("F", "E", "P", "T", "R", "H", "I", source);
+    return factory(FAIL, expect, progressError, stepsError, RULES, HELPERS, idByName);
   }
   function progressError(op, at) {
     throw new Error(
@@ -2259,6 +3213,165 @@ return ${value};
   return { compile, functionFor, sourceFor, ruleIndex, RULES, SOURCES, ORIGINS, guard };
 }
 
+// src/memoplan.js
+function selectMemoRules(grammar, firsts, options = {}) {
+  const ll1 = options.ll1 !== false;
+  const names = Object.keys(grammar);
+  const leftEdges = /* @__PURE__ */ new Map();
+  const callEdges = /* @__PURE__ */ new Map();
+  for (const name of names) {
+    const left = /* @__PURE__ */ new Set();
+    leftmostCalls(grammar[name], firsts, left);
+    leftEdges.set(name, [...left].filter((r) => r in grammar));
+    const calls = /* @__PURE__ */ new Set();
+    walk(grammar[name], (node) => {
+      if (opOf(node) !== "subrule") return;
+      const arg = node.subrule;
+      const target = Array.isArray(arg) ? arg[0] : arg;
+      if (typeof target === "string" && target in grammar) calls.add(target);
+    });
+    callEdges.set(name, [...calls]);
+  }
+  const closure = (edges, from) => {
+    const seen = /* @__PURE__ */ new Set();
+    const stack = [...from];
+    while (stack.length > 0) {
+      const next = stack.pop();
+      if (seen.has(next)) continue;
+      seen.add(next);
+      for (const edge of edges.get(next) || []) stack.push(edge);
+    }
+    return seen;
+  };
+  const leftCache = /* @__PURE__ */ new Map();
+  const leftReach = (name) => {
+    let reach = leftCache.get(name);
+    if (reach === void 0) {
+      reach = closure(leftEdges, leftEdges.get(name) || []);
+      leftCache.set(name, reach);
+    }
+    return reach;
+  };
+  const startsWith = (branch) => {
+    const direct = /* @__PURE__ */ new Set();
+    leftmostCalls(branch, firsts, direct);
+    const all = /* @__PURE__ */ new Set();
+    for (const rule of direct) {
+      if (!(rule in grammar)) continue;
+      all.add(rule);
+      for (const deeper of leftReach(rule)) all.add(deeper);
+    }
+    return all;
+  };
+  const sites = [];
+  for (const name of names) {
+    walk(grammar[name], (node) => {
+      if (opOf(node) !== "alt" || !Array.isArray(node.alt) || node.alt.length < 2) return;
+      for (const group of collisionGroups(node.alt, firsts, ll1)) {
+        const counts = /* @__PURE__ */ new Map();
+        for (const branch of group) {
+          for (const rule of startsWith(branch)) counts.set(rule, (counts.get(rule) || 0) + 1);
+        }
+        const shared = [...counts].filter(([, n]) => n >= 2).map(([rule]) => rule);
+        const topmost = shared.filter((rule) => !shared.some((other) => other !== rule && leftReach(other).has(rule) && !leftReach(rule).has(other)));
+        if (topmost.length > 0) sites.push({ host: name, rules: topmost });
+      }
+    });
+  }
+  const hosts = new Set(sites.map((site) => site.host));
+  const chosen = /* @__PURE__ */ new Set();
+  for (const { rules } of sites) {
+    for (const rule of rules) {
+      const inside = closure(callEdges, [rule]);
+      inside.add(rule);
+      for (const host of hosts) {
+        if (inside.has(host)) {
+          chosen.add(rule);
+          break;
+        }
+      }
+    }
+  }
+  const dependsOnCaller = (name) => {
+    const reachable = closure(callEdges, [name]);
+    reachable.add(name);
+    for (const rule of reachable) if (usesOuterScope(grammar[rule])) return true;
+    return false;
+  };
+  return names.filter((name) => chosen.has(name) && !dependsOnCaller(name));
+}
+function collisionGroups(branches, firsts, ll1) {
+  const table = ll1 ? buildDispatch(branches, firsts) : null;
+  if (table === null) return [branches];
+  const groups = /* @__PURE__ */ new Set();
+  for (const entry of table.values()) if (Array.isArray(entry) && entry.length > 1) groups.add(entry);
+  return [...groups];
+}
+function walk(node, visit) {
+  if (Array.isArray(node)) {
+    for (const child of node) walk(child, visit);
+    return;
+  }
+  const op = opOf(node);
+  if (op === null) return;
+  visit(node);
+  const arg = node[op];
+  switch (op) {
+    case "seq":
+    case "alt":
+      if (Array.isArray(arg)) for (const child of arg) walk(child, visit);
+      break;
+    case "label":
+      if (Array.isArray(arg)) walk(arg[1], visit);
+      break;
+    case "many":
+    case "many1":
+    case "option":
+    case "flag":
+    case "text":
+    case "lookahead":
+    case "negLookahead":
+      walk(arg, visit);
+      break;
+    case "as":
+      walk(Array.isArray(arg) ? arg[0] : arg && arg.parser, visit);
+      break;
+    case "between":
+      if (arg) for (const part of [arg.open, arg.body, arg.close]) walk(part, visit);
+      break;
+    case "pick":
+    case "yields":
+      if (arg && Array.isArray(arg.parsers)) for (const part of arg.parsers) walk(part, visit);
+      break;
+    case "prefix":
+      if (arg) {
+        walk(arg.operator, visit);
+        walk(arg.operand, visit);
+      }
+      break;
+    case "postfix":
+      if (arg) {
+        walk(arg.operand, visit);
+        walk(arg.suffix, visit);
+      }
+      break;
+    case "infixLeft":
+    case "infixRight":
+      if (arg) {
+        walk(arg.operand, visit);
+        if (arg.operator !== void 0) walk(arg.operator, visit);
+      }
+      break;
+    case "manySep":
+    case "many1Sep":
+      if (arg) {
+        walk(arg.rule, visit);
+        if (typeof arg.sep !== "string") walk(arg.sep, visit);
+      }
+      break;
+  }
+}
+
 // src/parser.js
 var EXECUTION_MODES = /* @__PURE__ */ new Set(["generated", "interpreted"]);
 var GrammarError = class extends Error {
@@ -2274,6 +3387,9 @@ function createParser(spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec;
   const {
     memo = false,
+    autoMemo = true,
+    inlineRules = true,
+    idioms = true,
     strict = true,
     validate = true,
     maxSteps = 0,
@@ -2300,9 +3416,12 @@ function createParser(spec, options = {}) {
   for (const [name, method] of Object.entries(extraMethods || {})) engine.addMethod(name, method);
   const startRule = rules[start];
   const firsts = ll1 ? analysis.firsts : null;
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval });
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]));
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0 ? selectMemoRules(rules, analysis.firsts, { ll1 }) : [];
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]));
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval, tokenIds, memoSlots, idioms });
   engine.__jlGrammar = { planner };
-  const codegen = compile && unsafeEval ? createCodegen({ planner, rules, firsts, memo, maxSteps }) : null;
+  const codegen = compile && unsafeEval ? createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms }) : null;
   const compiledStart = codegen && codegen.compile(start);
   if (codegen && !engine.disableInterpretedOptimization) {
     for (const name of Object.keys(rules)) {
@@ -2311,7 +3430,7 @@ function createParser(spec, options = {}) {
     }
   }
   function parseTokens(tokens, source, track = false) {
-    const state = createState(tokens, rules, { memo, maxSteps, track });
+    const state = createState(tokens, rules, { memo, maxSteps, track, memoSlots: memoRules.length });
     state.firsts = firsts;
     state.source = source;
     state.positions = lexer.positions;
@@ -2341,6 +3460,8 @@ function createParser(spec, options = {}) {
     planner,
     codegen,
     execution: mode,
+    /** Rules memoized by `autoMemo`, for introspection. */
+    memoRules,
     compiled: Boolean(compiledStart),
     /** Rules whose `alt`s resolved to O(1) LL(1) dispatch, for introspection. */
     get firsts() {
@@ -2646,6 +3767,7 @@ var CONSTRUCTORS = {
   many: { arity: [1, 1], of: ([a]) => ({ many: a }) },
   many1: { arity: [1, 1], of: ([a]) => ({ many1: a }) },
   option: { arity: [1, 1], of: ([a]) => ({ option: a }) },
+  flag: { arity: [1, 1], of: ([a]) => ({ flag: a }) },
   action: { arity: [1, 1], of: ([a]) => ({ action: a }) },
   lookahead: { arity: [1, 1], of: ([a]) => ({ lookahead: a }) },
   negLookahead: { arity: [1, 1], of: ([a]) => ({ negLookahead: a }) },
@@ -2657,6 +3779,10 @@ var CONSTRUCTORS = {
   manySep: { arity: [2, 2], of: ([a, b]) => ({ manySep: { rule: a, sep: b } }) },
   many1Sep: { arity: [2, 2], of: ([a, b]) => ({ many1Sep: { rule: a, sep: b } }) },
   between: { arity: [3, 3], of: ([a, b, c]) => ({ between: { open: a, body: b, close: c } }) },
+  // pick(1, consume(Open), subrule(Body), consume(Close)) — the result at that index.
+  pick: { arity: [2, Infinity], of: ([index, ...parsers]) => ({ pick: { parsers, index } }) },
+  // yields(consume(True), true) — run the parsers, give back the constant.
+  yields: { arity: [2, Infinity], of: (args) => ({ yields: { parsers: args.slice(0, -1), value: args[args.length - 1] } }) },
   infixLeft: { arity: [3, 3], of: ([a, b, c]) => ({ infixLeft: { operand: a, operator: b, combine: logicOf(c) } }) },
   infixRight: { arity: [3, 3], of: ([a, b, c]) => ({ infixRight: { operand: a, operator: b, combine: logicOf(c) } }) },
   prefix: { arity: [3, 3], of: ([a, b, c]) => ({ prefix: { operator: a, operand: b, combine: logicOf(c) } }) },
@@ -2677,6 +3803,355 @@ function createParserFromSource(source, options = {}) {
   return createParser(grammar, { ...options, methods });
 }
 
+// src/lexgen.js
+function generateLexer(defs, options = {}) {
+  const positions = options.positions === "offset" ? "offset" : "full";
+  const trackLines = positions === "full";
+  const useScanners = options.scanners !== false;
+  const useFuse = options.fuse !== false;
+  const modeIds = /* @__PURE__ */ new Map([[DEFAULT_MODE2, 0]]);
+  const modeNames = [DEFAULT_MODE2];
+  const idOf = (name) => {
+    let id = modeIds.get(name);
+    if (id === void 0) {
+      id = modeNames.length;
+      modeIds.set(name, id);
+      modeNames.push(name);
+    }
+    return id;
+  };
+  for (const def of defs) for (const mode of modesOf2(def)) idOf(mode);
+  const compiled = defs.map((def, id) => {
+    const flags = def.ignoreCase ? "iy" : "y";
+    return {
+      id,
+      name: def.name,
+      skip: !!def.skip,
+      literal: def.literal,
+      pattern: def.pattern,
+      flags,
+      longerAlt: def.longerAlt || null,
+      scanner: useScanners && def.pattern ? buildScanner(def.pattern, !!def.ignoreCase) : null,
+      modes: modesOf2(def),
+      pushMode: def.pushMode ? idOf(def.pushMode) : -1,
+      popMode: !!def.popMode,
+      firstChars: def.literal !== void 0 ? def.literal.length > 0 ? /* @__PURE__ */ new Set([def.literal.charCodeAt(0)]) : null : firstCharSet(new RegExp(def.pattern, flags).source, !!def.ignoreCase)
+    };
+  });
+  const byName = new Map(compiled.map((c) => [c.name, c]));
+  const multiMode = modeNames.length > 1 || compiled.some((c) => c.popMode || c.pushMode !== -1);
+  const regexes = [];
+  const regexIndex = /* @__PURE__ */ new Map();
+  const rxFor = (c) => {
+    if (!regexIndex.has(c.id)) {
+      regexIndex.set(c.id, regexes.length);
+      regexes.push([c.id, c.flags]);
+    }
+    return (
+      /** @type {number} */
+      regexIndex.get(c.id)
+    );
+  };
+  const fusedRuns = [];
+  const helpers = [];
+  const keywords = [];
+  const lookFns = /* @__PURE__ */ new Map();
+  const need = { digits: false, hex: false, keyword: false };
+  function emitRegex(c) {
+    const rx = rxFor(c);
+    if (c.skip) {
+      return `RX[${rx}].lastIndex = offset; if (RX[${rx}].test(text) && RX[${rx}].lastIndex > offset) { id = ${c.id}; end = RX[${rx}].lastIndex; matched = null; break found }`;
+    }
+    return `RX[${rx}].lastIndex = offset; m = RX[${rx}].exec(text); if (m !== null && m[0].length > 0) { id = ${c.id}; matched = m[0]; end = offset + matched.length; break found }`;
+  }
+  const emitted = /* @__PURE__ */ new Map();
+  function emitDef(c) {
+    let code = emitted.get(c.id);
+    if (code === void 0) {
+      code = emitDefOnce(c);
+      emitted.set(c.id, code);
+    }
+    return code;
+  }
+  function emitDefOnce(c) {
+    if (c.literal !== void 0) {
+      const lit = c.literal;
+      const test = lit.length === 1 ? `code === ${lit.charCodeAt(0)}` : `text.startsWith(${q(lit)}, offset)`;
+      return `if (${test}) { id = ${c.id}; end = offset + ${lit.length}; matched = ${q(lit)}; break found }`;
+    }
+    const s = c.scanner;
+    if (s === null) return emitRegex(c);
+    const hit = `{ id = ${c.id}; end = e; matched = null; break found }`;
+    switch (s.kind) {
+      case SCANNER_KINDS.CLASS: {
+        if (s.wide) {
+          helpers.push(wideClassHelper(c.id, s));
+          return `{ const e = scan${c.id}(text, offset, length); if (e === -2) { ${emitRegex(c)} } else if (e !== -1) ${hit} }`;
+        }
+        return `if (${condition(s.first, "code")}) { let j = offset + 1; while (j < length) { const c = text.charCodeAt(j); if (!(${condition(s.rest, "c")})) break; j++ } id = ${c.id}; end = j; matched = null; break found }`;
+      }
+      case SCANNER_KINDS.SEQ2:
+        return `if (${condition(s.first, "code")}) { let j = offset + 1; while (j < length) { const c = text.charCodeAt(j); if (${condition(s.first, "c")}) j++; else break } const mid = j; while (j < length) { const c = text.charCodeAt(j); if (${condition(s.second, "c")}) j++; else break } if (j > mid) { id = ${c.id}; end = j; matched = null; break found } }`;
+      case SCANNER_KINDS.KEYWORD: {
+        need.keyword = true;
+        const cond = condition(s.look, "c");
+        let look = lookFns.get(cond);
+        if (look === void 0) {
+          look = `look${lookFns.size}`;
+          lookFns.set(cond, look);
+        }
+        keywords.push([s.lower, s.upper]);
+        return `{ const e = scanKeyword(text, offset, length, KW[${keywords.length - 1}][0], KW[${keywords.length - 1}][1], ${look}); if (e !== -1) ${hit} }`;
+      }
+      case SCANNER_KINDS.LINE:
+        return `if (text.startsWith(${q(s.prefix)}, offset)) { let j = offset + ${s.prefix.length}; while (j < length) { const c = text.charCodeAt(j); if (${condition(s.look, "c")}) break; j++ } id = ${c.id}; end = j; matched = null; break found }`;
+      case SCANNER_KINDS.QUOTED:
+        helpers.push(quotedHelper(c.id, s));
+        return `{ const e = scan${c.id}(text, offset); if (e !== -1) ${hit} }`;
+      case SCANNER_KINDS.NUMBER:
+        need.digits = true;
+        helpers.push(numberHelper(c.id, s));
+        return `{ const e = scan${c.id}(text, offset, length); if (e !== -1) ${hit} }`;
+      case SCANNER_KINDS.STRING:
+        if (s.uni) need.hex = true;
+        helpers.push(stringHelper(c.id, s));
+        return `{ const e = scan${c.id}(text, offset, length); if (e !== -1) ${hit} }`;
+      default:
+        return emitRegex(c);
+    }
+  }
+  const fusible = (c) => c.literal === void 0 && c.scanner === null && !/\\[1-9]|\\k</.test(c.pattern) && new RegExp(c.pattern + "|").exec("").length === 1;
+  function emitList(list) {
+    const out = [];
+    for (let i = 0; i < list.length; ) {
+      let j = i;
+      while (useFuse && j < list.length && fusible(list[j]) && list[j].flags === list[i].flags) j++;
+      if (j - i < 3) {
+        out.push(emitDef(list[i]));
+        i++;
+        continue;
+      }
+      const run = list.slice(i, j);
+      run.forEach(rxFor);
+      const k = fusedRuns.length;
+      fusedRuns.push({ flags: run[0].flags, ids: run.map((d) => d.id), rx: run.map((d) => (
+        /** @type {number} */
+        regexIndex.get(d.id)
+      )) });
+      out.push(
+        `FX[${k}].lastIndex = offset; m = FX[${k}].exec(text)
+          if (m !== null) {
+            if (m[0].length > 0) { let g = 1; while (m[g] === undefined) g++; id = FIDS[${k}][g - 1]; matched = m[0]; end = offset + matched.length; break found }
+            // An empty match is "no match" to a lexer, so the members after it still get their turn.
+            for (let k = 0; k < FRX[${k}].length; k++) { const r = RX[FRX[${k}][k]]; r.lastIndex = offset; const mm = r.exec(text); if (mm !== null && mm[0].length > 0) { id = FIDS[${k}][k]; matched = mm[0]; end = offset + matched.length; break found } }
+          }`
+      );
+      i = j;
+    }
+    return out.join("\n          ");
+  }
+  const lists = [{ list: (
+    /** @type {any[]} */
+    []
+  ) }];
+  const listIds = /* @__PURE__ */ new Map([["", 0]]);
+  const listFor = (list) => {
+    const key = list.map((c) => c.id).join(",");
+    let id = listIds.get(key);
+    if (id === void 0) {
+      id = lists.length;
+      listIds.set(key, id);
+      lists.push({ list });
+    }
+    return id;
+  };
+  const WIDTH = 129;
+  const table = [];
+  for (const name of modeNames) {
+    const members = compiled.filter((c) => c.modes.includes(name));
+    if (members.length === 0) throw new Error(`generateLexer: mode '${name}' has no tokens`);
+    const dispatch = members.some((c) => c.firstChars !== null);
+    for (let code = 0; code < 128; code++) {
+      table.push(listFor(dispatch ? members.filter((c) => c.firstChars === null || c.firstChars.has(code)) : members));
+    }
+    table.push(listFor(dispatch ? members.filter((c) => c.firstChars === null || [...c.firstChars].some((x) => x >= 128)) : members));
+  }
+  if (lists.length > 6e4) throw new Error("generateLexer: too many distinct candidate lists");
+  const dispatchTable = String.fromCharCode(...table.map((id) => 256 + id));
+  const listCases = lists.slice(1).map(({ list }, i) => `case ${i + 1}: {
+          ${emitList(list)}
+          break
+        }`);
+  const longer = compiled.filter((c) => c.longerAlt);
+  const longerCode = longer.length === 0 ? "" : `
+      switch (id) {
+${longer.map((c) => {
+    const alt2 = (
+      /** @type {any} */
+      byName.get(
+        /** @type {string} */
+        c.longerAlt
+      )
+    );
+    const altEnd = alt2.literal !== void 0 ? `text.startsWith(${q(alt2.literal)}, offset) ? offset + ${alt2.literal.length} : -1` : `(RX[${rxFor(alt2)}].lastIndex = offset, RX[${rxFor(alt2)}].test(text) ? RX[${rxFor(alt2)}].lastIndex : -1)`;
+    return `        case ${c.id}: { const altEnd = ${altEnd}; if (altEnd > end) { end = altEnd; id = ${alt2.id}; matched = ${alt2.literal !== void 0 ? q(alt2.literal) : "null"} } break }`;
+  }).join("\n")}
+      }`;
+  const transitions = compiled.filter((c) => c.popMode || c.pushMode !== -1);
+  const describe = trackLines ? "'line ' + line + ' column ' + (offset - lineStart + 1)" : "'offset ' + offset";
+  const transitionCode = transitions.length === 0 ? "" : `
+      switch (id) {
+${transitions.map((c) => c.popMode ? `        case ${c.id}:
+          if (depth === 0) throw new LexError('Token \\'' + NAMES[id] + '\\' popped the initial lexer mode at ' + ${describe}, { offset, line: ${trackLines ? "line" : "0"}, col: ${trackLines ? "offset - lineStart + 1" : "0"} })
+          mode = stack[--depth]
+          break` : `        case ${c.id}: stack[depth++] = mode; mode = ${c.pushMode}; break`).join("\n")}
+      }`;
+  const where = trackLines ? "'line ' + line + ' column ' + column" : "'offset ' + offset";
+  const inMode = modeNames.length > 1 ? " + ' (in lexer mode \\'' + MODE_NAMES[mode] + '\\')'" : "";
+  return `(() => {
+  const NAMES = tokens.map((t) => t.name)
+  const SKIP = tokens.map((t) => (t.skip ? 1 : 0))
+  const DISPATCH = ${q(dispatchTable)}
+  ${modeNames.length > 1 ? `const MODE_NAMES = ${q(modeNames)}` : ""}
+  // Patterns are read from the token table above rather than repeated here.
+  const RX = ${q(regexes)}.map(([i, flags]) => new RegExp(tokens[i].pattern, flags))
+  ${fusedRuns.length ? `const FX = ${q(fusedRuns.map((f) => [f.ids, f.flags]))}.map(([ids, flags]) => new RegExp(ids.map((i) => '(' + tokens[i].pattern + ')').join('|'), flags))
+  const FIDS = ${q(fusedRuns.map((f) => f.ids))}
+  const FRX = ${q(fusedRuns.map((f) => f.rx))}` : ""}
+  ${need.digits ? "const skipDigits = (text, i, length) => { while (i < length) { const c = text.charCodeAt(i); if (c < 48 || c > 57) break; i++ } return i }" : ""}
+  ${need.hex ? "const isHex = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 70) || (c >= 97 && c <= 102)" : ""}
+  ${need.keyword ? `const KW = ${q(keywords)}
+  ${[...lookFns].map(([cond, name]) => `const ${name} = (c) => ${cond}`).join("\n  ")}
+  ${KEYWORD_HELPER}` : ""}
+  ${helpers.join("\n")}
+
+  function tokenize (text) {
+    if (typeof text !== 'string') throw new TypeError('tokenize: expected a string')
+    const tokens_ = []
+    let offset = 0
+    let line = 1
+    let lineStart = 0
+    const length = text.length
+    ${multiMode ? "let mode = 0\n    const stack = []\n    let depth = 0" : ""}
+    let m = null
+    while (offset < length) {
+      const code = text.charCodeAt(offset)
+      let id = -1
+      let end = -1
+      let matched = null
+      found: {
+        switch (DISPATCH.charCodeAt(${multiMode ? `mode * ${WIDTH} + ` : ""}(code < 128 ? code : 128)) - 256) {
+        ${listCases.join("\n        ")}
+        }
+      }
+      if (id === -1) {
+        const column = offset - lineStart + 1
+        throw new LexError('Unexpected character \\'' + text[offset] + '\\' at ' + ${where}${inMode}, { offset, line: ${trackLines ? "line" : "0"}, col: ${trackLines ? "column" : "0"} })
+      }${longerCode}
+      if (SKIP[id] === 0) {
+        const image = matched !== null ? matched : text.slice(offset, end)
+        tokens_.push(${trackLines ? "{ type: NAMES[id], image, start: offset, end, line, col: offset - lineStart + 1 }" : "{ type: NAMES[id], id, image, start: offset, end }"})
+      }
+      ${trackLines ? "for (let j = offset; j < end; j++) { if (text.charCodeAt(j) !== 10) continue; line++; lineStart = j + 1 }" : ""}${transitionCode}
+      offset = end
+    }
+    return tokens_
+  }
+  return { tokenize, tokenNames: NAMES, positions: ${q(positions)}, modeNames: ${q(modeNames)} }
+})()`;
+}
+var DEFAULT_MODE2 = "default";
+var q = JSON.stringify;
+function modesOf2(def) {
+  if (Array.isArray(def.modes) && def.modes.length > 0) return def.modes;
+  if (def.mode) return [def.mode];
+  return [DEFAULT_MODE2];
+}
+function condition(table, v) {
+  const parts = [];
+  for (let i = 0; i < 128; ) {
+    if (table[i] !== 1) {
+      i++;
+      continue;
+    }
+    let j = i;
+    while (j + 1 < 128 && table[j + 1] === 1) j++;
+    parts.push(i === j ? `${v} === ${i}` : `(${v} >= ${i} && ${v} <= ${j})`);
+    i = j + 1;
+  }
+  return parts.length > 0 ? parts.join(" || ") : "false";
+}
+var KEYWORD_HELPER = `function scanKeyword (text, offset, length, lower, upper, look) {
+    const n = lower.length
+    if (offset + n > length) return -1
+    for (let i = 0; i < n; i++) {
+      const c = text.charCodeAt(offset + i)
+      if (c !== lower[i] && c !== upper[i]) return -1
+    }
+    const end = offset + n
+    if (end < length && look(text.charCodeAt(end))) return -1
+    return end
+  }`;
+function wideClassHelper(id, s) {
+  return `function scan${id} (text, offset, length) {
+    const code = text.charCodeAt(offset)
+    if (code >= 128) return -2
+    if (!(${condition(s.first, "code")})) return -1
+    let j = offset + 1
+    while (j < length) {
+      const c = text.charCodeAt(j)
+      if (c >= 128) return -2
+      if (!(${condition(s.rest, "c")})) break
+      j++
+    }
+    return j
+  }`;
+}
+function quotedHelper(id, s) {
+  return `function scan${id} (text, offset) {
+    if (text.charCodeAt(offset) !== ${s.quote}) return -1
+    let fallback = -1
+    let j = offset + 1
+    for (;;) {
+      j = text.indexOf(${q(String.fromCharCode(s.quote))}, j)
+      if (j === -1) return fallback
+      if (text.charCodeAt(j + 1) !== ${s.quote}) return j + 1
+      fallback = j + 1
+      j += 2
+    }
+  }`;
+}
+function numberHelper(id, s) {
+  return `function scan${id} (text, offset, length) {
+    let j = offset
+    ${s.sign ? "if (text.charCodeAt(j) === 45) j++" : ""}
+    const c = text.charCodeAt(j)
+    ${s.jsonInt ? "if (c === 48) j++\n    else if (c >= 49 && c <= 57) j = skipDigits(text, j + 1, length)\n    else return -1" : "if (!(c >= 48 && c <= 57)) return -1\n    j = skipDigits(text, j + 1, length)"}
+    ${s.frac ? "if (text.charCodeAt(j) === 46) { const k = skipDigits(text, j + 1, length); if (k > j + 1) j = k }" : ""}
+    ${s.exp ? "const e = text.charCodeAt(j)\n    if (e === 101 || e === 69) { let k = j + 1; const sg = text.charCodeAt(k); if (sg === 43 || sg === 45) k++; const end = skipDigits(text, k, length); if (end > k) j = end }" : ""}
+    return j
+  }`;
+}
+function stringHelper(id, s) {
+  return `function scan${id} (text, offset, length) {
+    if (text.charCodeAt(offset) !== ${s.quote}) return -1
+    let j = offset + 1
+    for (;;) {
+      if (j >= length) return -1
+      const c = text.charCodeAt(j)
+      if (c === ${s.quote}) return j + 1
+      if (c === 92) {
+        const d = text.charCodeAt(j + 1)
+        ${s.escAny ? "if (!(d >= 0) || d === 10 || d === 13 || d === 0x2028 || d === 0x2029) return -1\n        j += 2" : `if (${condition(s.esc, "d")}) j += 2
+        ${s.uni ? "else if (d === 117 && isHex(text.charCodeAt(j + 2)) && isHex(text.charCodeAt(j + 3)) && isHex(text.charCodeAt(j + 4)) && isHex(text.charCodeAt(j + 5))) j += 6\n        " : ""}else return -1`}
+        continue
+      }
+      ${s.ctrl ? "if (c < 32) return -1" : ""}
+      j++
+    }
+  }`;
+}
+
 // src/emit.js
 import { Compiler, LogicEngine as LogicEngine2 } from "json-logic-engine";
 var UNSUPPORTED = /\b(coerceArray|precoerceNumber|compareCheck|assertAllowedDepth|assertSize|asyncIterators|notTraversed)\b|(?<!engine\.)\bmethods\[|engine\.(?!methods\b)/;
@@ -2725,11 +4200,21 @@ function emitModule(spec, options = {}) {
   const { tokens: tokenDefs, rules, start } = spec;
   const {
     memo = false,
+    autoMemo = true,
+    // Off here, on in `createParser`. Inlining a precedence chain is worth 2-3x on a bare chain but only
+    // 1.02-1.04x on the formula and SQL examples, and it makes the file bigger: the formula module grows
+    // 32% minified (38 to 50 KB) even after the second copy of each operand is kept as a call. A parser
+    // built at run time pays nothing for that; a file someone ships does.
+    inlineRules = false,
+    idioms = true,
     strict = true,
     ll1 = true,
     maxSteps = 0,
     positions = "full",
     execution = "generated",
+    lexer: lexerMode = "generated",
+    scanners = true,
+    fuse = true,
     format = "esm",
     validate = true,
     moduleName = spec.name || "parser",
@@ -2738,6 +4223,7 @@ function emitModule(spec, options = {}) {
     methods: semanticMethods
   } = options;
   if (format !== "esm" && format !== "cjs") throw new Error(`emitModule: format must be 'esm' or 'cjs', got ${quote(format)}`);
+  if (lexerMode !== "generated" && lexerMode !== "runtime") throw new Error(`emitModule: lexer must be 'generated' or 'runtime', got ${quote(lexerMode)}`);
   const methodsBlock = options.methodsBlock !== void 0 ? options.methodsBlock : spec.methodsBlock;
   const block = methodsBlock ? emitMethodsBlock(methodsBlock, format) : null;
   if (!start || !(start in rules)) throw new Error(`emitModule: start rule '${start}' is not defined in rules`);
@@ -2745,12 +4231,15 @@ function emitModule(spec, options = {}) {
   const analysis = analyze(rules, { tokenNames: lexer.tokenNames });
   if (validate && analysis.errors.length > 0) throw new GrammarError(analysis.errors);
   const firsts = ll1 ? analysis.firsts : null;
+  const tokenIds = new Map(lexer.tokenNames.map((name, id) => [name, id]));
   const engine = { build: () => {
     throw new Error("not compiled during emit");
   }, run: () => {
   } };
-  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false });
-  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps });
+  const memoRules = autoMemo && !memo && analysis.errors.length === 0 ? selectMemoRules(rules, analysis.firsts, { ll1 }) : [];
+  const memoSlots = new Map(memoRules.map((name, slot) => [name, slot]));
+  const planner = createPlanner({ engine, rules, firsts, memo, maxSteps, unsafeEval: false, tokenIds, memoSlots, idioms });
+  const codegen = createCodegen({ planner, rules, firsts, memo, maxSteps, tokenIds, memoSlots, inlineRules, idioms });
   codegen.compile(start);
   const reached = [...codegen.ruleIndex.entries()].sort((a, b) => a[1] - b[1]);
   const unsupported = codegen.ORIGINS.map((origin, index) => ({ origin, index })).filter(({ origin }) => origin.kind === "unknownRule");
@@ -2791,7 +4280,8 @@ R[${index}] = ${codegen.SOURCES[index]}`
   ).join("\n\n");
   const interpretedActions = execution === "interpreted";
   const cjs = format === "cjs";
-  const runtimeImports = ["FAIL as F", "expect as E", "createState", "buildParseError", "createLexer"];
+  const generatedLexer = lexerMode === "generated";
+  const runtimeImports = ["FAIL as F", "expect as E", "createState", "buildParseError", generatedLexer ? "LexError" : "createLexer"];
   if (needsEngine) runtimeImports.push("createValueMethods");
   const importRuntime = cjs ? `const { ${runtimeImports.map((b) => b.replace(" as ", ": ")).join(", ")} } = require(${quote(runtimeSpecifier)})` : `import { ${runtimeImports.join(", ")} } from ${quote(runtimeSpecifier)}`;
   const importEngine = engineFree ? "" : cjs ? `const { LogicEngine } = require(${quote(engineSpecifier)})
@@ -2810,7 +4300,7 @@ export default { ${PUBLIC.join(", ")} }`;
 //
 // The grammar was analyzed when this file was written: FIRST sets, left-recursion
 // checks and rule specialization all happened then. Importing it does no grammar
-// work and generates no code.
+// work and generates no code.${generatedLexer ? "\n// The lexer was written out as source too, so there is no lexer engine to load." : ""}
 ${engineFree ? "// Every semantic action was compiled to source too, so this file has no\n// dependency on json-logic-engine at all." : interpretedActions ? "// Semantic actions are interpreted, so nothing here calls `eval`." : "// Semantic actions that could not be compiled to source are shipped as JSON\n// Logic and built by json-logic-engine when this module loads."}
 
 ${importRuntime}
@@ -2820,7 +4310,7 @@ const tokens = ${literal(tokenDefs)}
 
 const start = ${quote(start)}
 
-const lexer = createLexer(tokens, { positions: ${quote(positions)} })
+const lexer = ${generatedLexer ? generateLexer(tokenDefs, { positions, scanners, fuse }) : `createLexer(tokens, { positions: ${quote(positions)}, scanners: ${scanners}, fuse: ${fuse} })`}
 const tokenize = lexer.tokenize
 
 ${engineFree ? `// Every semantic action compiled to source, so the engine reduced to a table
@@ -2893,6 +4383,14 @@ const H = [
 ${helpers.map((source, index) => `  /* ${index} */ ${source}`).join(",\n")}
 ]
 
+/**
+ * Name -> id, the same numbering \`lexer\` assigns. A generated \`alt\`'s switch
+ * dispatches on a token's id when it has one; this is only read as a fallback,
+ * for a hand-built token from \`parseTokens\` that bypassed this file's own
+ * lexer and so has no id of its own.
+ */
+const I = ${literal(Object.fromEntries(tokenIds))}
+
 ${ruleSources}
 
 const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name, index])))}
@@ -2903,7 +4401,7 @@ const RULES = ${literal(Object.fromEntries(reached.map(([name, index]) => [name,
  * @param {string} [source]
  */
 function parseTokens (tokenList, source, track = false) {
-  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track })
+  const state = createState(tokenList, {}, { memo: ${memo}, maxSteps: ${maxSteps}, track, memoSlots: ${memoRules.length} })
   state.source = source
   state.positions = ${quote(positions)}
   state.ruleNames[0] = start