grammatik 0.1.1 → 0.1.2

package/dist/esm/runtime.js

@@ -1,5 +1,11 @@
 // src/state.js
 var FAIL = /* @__PURE__ */ Symbol("grammatik.fail");
+var memoTables = (slots, length) => {
+  if (!slots) return null;
+  const tables = new Array(slots);
+  for (let i = 0; i < slots; i++) tables[i] = { ends: new Int32Array(length + 1), vals: new Array(length + 1) };
+  return tables;
+};
 var createState = (tokens, grammar, options = {}) => ({
   tokens,
   idx: 0,
@@ -7,6 +13,7 @@ var createState = (tokens, grammar, options = {}) => ({
   expected: [],
   grammar,
   memo: options.memo ? /* @__PURE__ */ new Map() : null,
+  rm: memoTables(options.memoSlots, tokens.length),
   // Entering a rule is one array store rather than an object allocation, and
   // the stack is snapshotted only on a furthest failure — by the time an error
   // is built it would otherwise have unwound to just the start rule.
@@ -27,6 +34,19 @@ function expect(state, expectation) {
   if (!state.track || state.idx !== state.maxIdx) return;
   if (!state.expected.includes(expectation)) state.expected.push(expectation);
 }
+var LexError = class extends Error {
+  /**
+   * @param {string} message
+   * @param {{ offset: number, line: number, col: number }} position
+   */
+  constructor(message, position) {
+    super(message);
+    this.name = "LexError";
+    this.offset = position.offset;
+    this.line = position.line;
+    this.col = position.col;
+  }
+};
 var ParseError = class extends Error {
   /**
    * @param {string} message
@@ -367,20 +387,303 @@ function firstCharSet(source, ignoreCase) {
   return found;
 }
 
-// src/lexer.js
-var LexError = class extends Error {
-  /**
-   * @param {string} message
-   * @param {{ offset: number, line: number, col: number }} position
-   */
-  constructor(message, position) {
-    super(message);
-    this.name = "LexError";
-    this.offset = position.offset;
-    this.line = position.line;
-    this.col = position.col;
+// src/scan.js
+var DEFER = -2;
+var CLASS = 0;
+var SEQ2 = 1;
+var KEYWORD = 2;
+var LINE = 3;
+var QUOTED = 4;
+var NUMBER = 5;
+var STRING = 6;
+var SCANNER_KINDS = Object.freeze({ CLASS, SEQ2, KEYWORD, LINE, QUOTED, NUMBER, STRING });
+var NONE = new Uint8Array(128);
+function make(kind, fields) {
+  return {
+    kind,
+    wide: false,
+    first: NONE,
+    rest: NONE,
+    second: NONE,
+    look: NONE,
+    lower: [],
+    upper: [],
+    prefix: "",
+    quote: -1,
+    sign: false,
+    jsonInt: false,
+    frac: false,
+    exp: false,
+    ctrl: false,
+    uni: false,
+    escAny: false,
+    esc: NONE,
+    ...fields
+  };
+}
+var SHORTHAND = {
+  d: (t) => {
+    for (let k = 48; k < 58; k++) t[k] = 1;
+  },
+  w: (t) => {
+    for (let k = 0; k < 128; k++) if (/\w/.test(String.fromCharCode(k))) t[k] = 1;
+  },
+  s: (t) => {
+    for (const k of [9, 10, 11, 12, 13, 32]) t[k] = 1;
+  }
+};
+var SIMPLE_ESCAPES2 = { t: "	", r: "\r", n: "\n", f: "\f", v: "\v" };
+var ALNUM = /[a-zA-Z0-9]/;
+function parseClass(body) {
+  if (body.length === 0 || body[0] === "^") return null;
+  const table = new Uint8Array(128);
+  for (let i = 0; i < body.length; i++) {
+    let c = body[i];
+    if (c === "\\") {
+      const n = body[++i];
+      if (n === "d" || n === "w") {
+        SHORTHAND[n](table);
+        continue;
+      }
+      if (n === void 0) return null;
+      if (ALNUM.test(n) && !(n in SIMPLE_ESCAPES2)) return null;
+      c = SIMPLE_ESCAPES2[n] ?? n;
+    }
+    if (body[i + 1] === "-" && i + 2 < body.length) {
+      let hi = body[i + 2];
+      let advance = 2;
+      if (hi === "\\") {
+        hi = body[i + 3];
+        advance = 3;
+        if (hi === void 0 || ALNUM.test(hi)) return null;
+      }
+      const from = c.charCodeAt(0);
+      const to = hi.charCodeAt(0);
+      if (from > to || to >= 128) return null;
+      for (let k = from; k <= to; k++) table[k] = 1;
+      i += advance;
+      continue;
+    }
+    const code = c.charCodeAt(0);
+    if (code >= 128) return null;
+    table[code] = 1;
+  }
+  return table;
+}
+var BODY = String.raw`((?:[^\]\\]|\\.)*)`;
+var RE_CLASS_RUN = new RegExp(String.raw`^\[${BODY}\]([+*])$`);
+var RE_CLASS_PAIR = new RegExp(String.raw`^\[${BODY}\]\[${BODY}\]\*$`);
+var RE_CLASS_SEQ = new RegExp(String.raw`^\[${BODY}\]\+\[${BODY}\]\+$`);
+var RE_SHORTHAND = /^\\([sdw])[+*]$/;
+var RE_KEYWORD = new RegExp(String.raw`^([A-Za-z0-9_]+)\(\?!\[${BODY}\]\)$`);
+var RE_LINE = new RegExp(String.raw`^([^\\\[\]()*+?.|^$]+)\[\^${BODY}\]\*$`);
+var RE_QUOTED = /^(['"])\(\?:\1\1\|\[\^\1\]\)\*\1$/;
+var RE_NUMBER = /^(-\?)?(?:(\(\?:0\|\[1-9\]\\d\*\))|\\d\+)(\(\?:\\\.\\d\+\)\?)?(\(\?:\[eE\]\[\+-\]\?\\d\+\)\?)?$/;
+var RE_STRING_ESC = new RegExp(String.raw`^(["'])\(\?:\[\^\1\\\\(\\u0000-\\u001f|\\x00-\\x1f)?\]\|\\\\\(\?:\[${BODY}\](\|u\[0-9a-fA-F\]\{4\})?\)\)\*\1$`);
+var RE_STRING_ANY = new RegExp(String.raw`^(["'])\(\?:\[\^\1\\\\\]\|\\\\\.\)\*\1$`);
+function buildScanner(source, ignoreCase) {
+  let m = RE_KEYWORD.exec(source);
+  if (m) {
+    const look = parseClass(m[2]);
+    if (look === null) return null;
+    const word = m[1];
+    const lower = [];
+    const upper = [];
+    for (const ch of word) {
+      lower.push((ignoreCase ? ch.toLowerCase() : ch).charCodeAt(0));
+      upper.push((ignoreCase ? ch.toUpperCase() : ch).charCodeAt(0));
+    }
+    return make(KEYWORD, { look, lower, upper });
+  }
+  if (ignoreCase) return null;
+  m = RE_STRING_ESC.exec(source);
+  if (m) {
+    const esc = parseClass(m[3]);
+    if (esc === null) return null;
+    return make(STRING, { quote: m[1].charCodeAt(0), ctrl: m[2] !== void 0, uni: m[4] !== void 0, esc });
+  }
+  m = RE_STRING_ANY.exec(source);
+  if (m) return make(STRING, { quote: m[1].charCodeAt(0), escAny: true });
+  m = RE_NUMBER.exec(source.replace(/\[0-9\]/g, "\\d"));
+  if (m && (m[1] || m[2] || m[3] || m[4])) {
+    return make(NUMBER, { sign: m[1] !== void 0, jsonInt: m[2] !== void 0, frac: m[3] !== void 0, exp: m[4] !== void 0 });
   }
+  m = RE_CLASS_RUN.exec(source);
+  if (m) {
+    const table = parseClass(m[1]);
+    return table && make(CLASS, { first: table, rest: table });
+  }
+  m = RE_CLASS_PAIR.exec(source);
+  if (m) {
+    const first = parseClass(m[1]);
+    const rest = parseClass(m[2]);
+    return first && rest && make(CLASS, { first, rest });
+  }
+  m = RE_CLASS_SEQ.exec(source);
+  if (m) {
+    const first = parseClass(m[1]);
+    const second = parseClass(m[2]);
+    if (first && second && first.every((v, i) => !(v && second[i]))) {
+      return make(SEQ2, { first, rest: first, second });
+    }
+    return null;
+  }
+  m = RE_SHORTHAND.exec(source);
+  if (m) {
+    const table = new Uint8Array(128);
+    SHORTHAND[m[1]](table);
+    return make(CLASS, { first: table, rest: table, wide: m[1] === "s" });
+  }
+  m = RE_QUOTED.exec(source);
+  if (m) return make(QUOTED, { quote: m[1].charCodeAt(0) });
+  m = RE_LINE.exec(source);
+  if (m) {
+    const stop = parseClass(m[2]);
+    return stop && make(LINE, { prefix: m[1], look: stop });
+  }
+  return null;
+}
+var skipDigits = (text, i, length) => {
+  while (i < length) {
+    const c = text.charCodeAt(i);
+    if (c < 48 || c > 57) break;
+    i++;
+  }
+  return i;
 };
+var isHex = (c) => c >= 48 && c <= 57 || c >= 65 && c <= 70 || c >= 97 && c <= 102;
+function scanEnd(sc, text, offset, length) {
+  switch (sc.kind) {
+    case CLASS: {
+      const code = text.charCodeAt(offset);
+      if (code >= 128) return sc.wide ? DEFER : -1;
+      if (sc.first[code] !== 1) return -1;
+      const rest = sc.rest;
+      let j = offset + 1;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c >= 128) return sc.wide ? DEFER : j;
+        if (rest[c] !== 1) break;
+        j++;
+      }
+      return j;
+    }
+    case SEQ2: {
+      const c0 = text.charCodeAt(offset);
+      if (c0 >= 128 || sc.first[c0] !== 1) return -1;
+      const { first, second } = sc;
+      let j = offset + 1;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && first[c] === 1) j++;
+        else break;
+      }
+      const mid = j;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && second[c] === 1) j++;
+        else break;
+      }
+      return j > mid ? j : -1;
+    }
+    case KEYWORD: {
+      const { lower, upper } = sc;
+      const n = lower.length;
+      if (offset + n > length) return -1;
+      for (let i = 0; i < n; i++) {
+        const c = text.charCodeAt(offset + i);
+        if (c !== lower[i] && c !== upper[i]) return -1;
+      }
+      const end = offset + n;
+      if (end < length) {
+        const c = text.charCodeAt(end);
+        if (c < 128 && sc.look[c] === 1) return -1;
+      }
+      return end;
+    }
+    case LINE: {
+      if (!text.startsWith(sc.prefix, offset)) return -1;
+      let j = offset + sc.prefix.length;
+      const stop = sc.look;
+      while (j < length) {
+        const c = text.charCodeAt(j);
+        if (c < 128 && stop[c] === 1) break;
+        j++;
+      }
+      return j;
+    }
+    case NUMBER: {
+      let j = offset;
+      if (sc.sign && text.charCodeAt(j) === 45) j++;
+      const c = text.charCodeAt(j);
+      if (sc.jsonInt) {
+        if (c === 48) j++;
+        else if (c >= 49 && c <= 57) j = skipDigits(text, j + 1, length);
+        else return -1;
+      } else {
+        if (!(c >= 48 && c <= 57)) return -1;
+        j = skipDigits(text, j + 1, length);
+      }
+      if (sc.frac && text.charCodeAt(j) === 46) {
+        const k = skipDigits(text, j + 1, length);
+        if (k > j + 1) j = k;
+      }
+      if (sc.exp) {
+        const e = text.charCodeAt(j);
+        if (e === 101 || e === 69) {
+          let k = j + 1;
+          const s = text.charCodeAt(k);
+          if (s === 43 || s === 45) k++;
+          const end = skipDigits(text, k, length);
+          if (end > k) j = end;
+        }
+      }
+      return j;
+    }
+    case STRING: {
+      const q = sc.quote;
+      if (text.charCodeAt(offset) !== q) return -1;
+      let j = offset + 1;
+      for (; ; ) {
+        if (j >= length) return -1;
+        const c = text.charCodeAt(j);
+        if (c === q) return j + 1;
+        if (c === 92) {
+          const d = text.charCodeAt(j + 1);
+          if (sc.escAny) {
+            if (!(d >= 0) || d === 10 || d === 13 || d === 8232 || d === 8233) return -1;
+            j += 2;
+          } else if (d < 128 && sc.esc[d] === 1) {
+            j += 2;
+          } else if (sc.uni && d === 117 && isHex(text.charCodeAt(j + 2)) && isHex(text.charCodeAt(j + 3)) && isHex(text.charCodeAt(j + 4)) && isHex(text.charCodeAt(j + 5))) {
+            j += 6;
+          } else {
+            return -1;
+          }
+          continue;
+        }
+        if (sc.ctrl && c < 32) return -1;
+        j++;
+      }
+    }
+    default: {
+      const q = sc.quote;
+      if (text.charCodeAt(offset) !== q) return -1;
+      const quote = String.fromCharCode(q);
+      let fallback = -1;
+      let j = offset + 1;
+      for (; ; ) {
+        j = text.indexOf(quote, j);
+        if (j === -1) return fallback;
+        if (text.charCodeAt(j + 1) !== q) return j + 1;
+        fallback = j + 1;
+        j += 2;
+      }
+    }
+  }
+}
+
+// src/lexer.js
 var DEFAULT_MODE = "default";
 function modesOf(def) {
   if (Array.isArray(def.modes) && def.modes.length > 0) return def.modes;
@@ -390,6 +693,8 @@ function modesOf(def) {
 function createLexer(defs, options = {}) {
   const positions = options.positions === "offset" ? "offset" : "full";
   const trackLines = positions === "full";
+  const useScanners = options.scanners !== false;
+  const useFuse = options.fuse !== false;
   if (!Array.isArray(defs) || defs.length === 0) throw new Error("createLexer: expected a non-empty array of token definitions");
   const tokenNames = defs.map((d) => d.name);
   const seen = /* @__PURE__ */ new Set();
@@ -433,9 +738,15 @@ function createLexer(defs, options = {}) {
       literal: def.literal,
       longerAlt: def.longerAlt || null,
       regex: def.pattern ? new RegExp(def.pattern, def.ignoreCase ? "iy" : "y") : null,
+      // A table-driven stand-in for the regex where its shape is recognised;
+      // null otherwise (and always, with `scanners: false`). It is a field on
+      // every definition so they all keep one shape. See scan.js.
+      scanner: useScanners && def.pattern ? buildScanner(def.pattern, !!def.ignoreCase) : null,
       modes: modesOf(def),
       pushMode: def.pushMode ? idOf(def.pushMode) : -1,
-      popMode: !!def.popMode
+      popMode: !!def.popMode,
+      // Set only on the stand-in `fuse` builds for a run of regexes; see there.
+      members: null
     };
   });
   const byName = new Map(compiled.map((c) => [c.name, c]));
@@ -452,14 +763,47 @@ function createLexer(defs, options = {}) {
   for (const c of compiled) {
     c.firstChars = c.literal !== void 0 ? c.literal.length > 0 ? /* @__PURE__ */ new Set([c.literal.charCodeAt(0)]) : null : firstCharSet(c.regex.source, !!c.regex.ignoreCase);
   }
-  const modes = modeNames.map((name) => {
+  const FUSE_MIN = 3;
+  const fusible = (def) => def.literal === void 0 && def.scanner === null && def.regex !== null && !/\\[1-9]|\\k</.test(def.regex.source) && // `exec` of an always-matching `|` alternative returns one slot per capture group.
+  new RegExp(def.regex.source + "|").exec("").length === 1;
+  function fuse(list) {
+    if (!useFuse) return list;
+    const out = [];
+    for (let i = 0; i < list.length; ) {
+      let j = i;
+      while (j < list.length && fusible(list[j]) && list[j].regex.flags === list[i].regex.flags) j++;
+      if (j - i < FUSE_MIN) {
+        out.push(list[i++]);
+        continue;
+      }
+      const run = list.slice(i, j);
+      out.push({
+        name: run.map((d) => d.name).join("|"),
+        id: -1,
+        skip: false,
+        literal: void 0,
+        longerAlt: null,
+        longerAltDef: void 0,
+        regex: new RegExp(run.map((d) => `(${d.regex.source})`).join("|"), run[0].regex.flags),
+        scanner: null,
+        modes: run[0].modes,
+        pushMode: -1,
+        popMode: false,
+        members: run
+      });
+      i = j;
+    }
+    return out;
+  }
+  const modes = modeNames.map((name, id) => {
     const members = compiled.filter((c) => c.modes.includes(name));
     const alwaysTry = members.filter((c) => c.firstChars === null);
-    const candidatesFor = (code) => members.filter((c) => c.firstChars === null || c.firstChars.has(code));
+    const candidatesFor = (code) => fuse(members.filter((c) => c.firstChars === null || c.firstChars.has(code)));
     const ascii = new Array(128);
     for (let code = 0; code < 128; code++) ascii[code] = candidatesFor(code);
     return {
       name,
+      id,
       members,
       ascii,
       wide: /* @__PURE__ */ new Map(),
@@ -483,6 +827,30 @@ function createLexer(defs, options = {}) {
   const multiMode = modes.length > 1 || compiled.some((c) => c.popMode || c.pushMode !== -1);
   const namesModes = modes.length > 1;
   const initial = modes[0];
+  let fusedMember = null;
+  let fusedText = "";
+  function fusedEnd(run, text, offset) {
+    run.regex.lastIndex = offset;
+    const match = run.regex.exec(text);
+    if (match === null) return -1;
+    if (match[0].length === 0) {
+      for (const member of run.members) {
+        member.regex.lastIndex = offset;
+        const one = member.regex.exec(text);
+        if (one !== null && one[0].length > 0) {
+          fusedMember = member;
+          fusedText = one[0];
+          return offset + fusedText.length;
+        }
+      }
+      return -1;
+    }
+    let group = 1;
+    while (match[group] === void 0) group++;
+    fusedMember = run.members[group - 1];
+    fusedText = match[0];
+    return offset + fusedText.length;
+  }
   function longerAltEnd(def, text, offset) {
     const alt = def.longerAltDef;
     if (alt.literal !== void 0) return text.startsWith(alt.literal, offset) ? offset + alt.literal.length : -1;
@@ -502,19 +870,30 @@ function createLexer(defs, options = {}) {
     outer: while (offset < length) {
       const applicable = mode.dispatch ? candidates(mode, text.charCodeAt(offset)) : mode.members;
       for (let i = 0; i < applicable.length; i++) {
-        const def = applicable[i];
+        let def = applicable[i];
         let end = -1;
         let matched = null;
-        if (def.literal !== void 0) {
+        if (def.members !== null) {
+          end = fusedEnd(def, text, offset);
+          if (end === -1) continue;
+          def = fusedMember;
+          matched = fusedText;
+        } else if (def.literal !== void 0) {
           if (text.startsWith(def.literal, offset)) end = offset + def.literal.length;
-        } else if (def.skip) {
-          def.regex.lastIndex = offset;
-          if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
         } else {
-          def.regex.lastIndex = offset;
-          const match = def.regex.exec(text);
-          if (match !== null && match[0].length > 0) matched = match[0];
-          if (matched !== null) end = offset + matched.length;
+          end = def.scanner === null ? DEFER : scanEnd(def.scanner, text, offset, length);
+          if (end === DEFER) {
+            end = -1;
+            if (def.skip) {
+              def.regex.lastIndex = offset;
+              if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
+            } else {
+              def.regex.lastIndex = offset;
+              const match = def.regex.exec(text);
+              if (match !== null && match[0].length > 0) matched = match[0];
+              if (matched !== null) end = offset + matched.length;
+            }
+          }
         }
         if (end === -1) continue;
         let winner = def;
@@ -563,7 +942,238 @@ function createLexer(defs, options = {}) {
   }
   const inMode = (mode) => namesModes ? ` (in lexer mode '${mode.name}')` : "";
   const describe = (offset, line, lineStart) => trackLines ? `line ${line} column ${offset - lineStart + 1}` : `offset ${offset}`;
-  return { tokenize, tokenNames, positions, modeNames };
+  if (!options.incremental) return { tokenize, tokenNames, positions, modeNames };
+  const recover = !!options.recover;
+  const errorTokenName = options.errorToken || "Error";
+  function snapshotStack(stack, depth) {
+    const ids = new Array(depth);
+    for (let i = 0; i < depth; i++) ids[i] = stack[i].id;
+    return ids;
+  }
+  function sameStack(a, b) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
+  }
+  function scan(text, offset, mode, stack, depth, line, lineStart) {
+    const tokens = [];
+    const checkpoints = [];
+    const length = text.length;
+    outer: while (offset < length) {
+      checkpoints.push({ modeId: mode.id, stack: snapshotStack(stack, depth), line, lineStart, offset });
+      const applicable = mode.dispatch ? candidates(mode, text.charCodeAt(offset)) : mode.members;
+      for (let i = 0; i < applicable.length; i++) {
+        let def = applicable[i];
+        let end = -1;
+        let matched = null;
+        if (def.members !== null) {
+          end = fusedEnd(def, text, offset);
+          if (end === -1) continue;
+          def = fusedMember;
+          matched = fusedText;
+        } else if (def.literal !== void 0) {
+          if (text.startsWith(def.literal, offset)) end = offset + def.literal.length;
+        } else {
+          end = def.scanner === null ? DEFER : scanEnd(def.scanner, text, offset, length);
+          if (end === DEFER) {
+            end = -1;
+            if (def.skip) {
+              def.regex.lastIndex = offset;
+              if (def.regex.test(text) && def.regex.lastIndex > offset) end = def.regex.lastIndex;
+            } else {
+              def.regex.lastIndex = offset;
+              const match = def.regex.exec(text);
+              if (match !== null && match[0].length > 0) matched = match[0];
+              if (matched !== null) end = offset + matched.length;
+            }
+          }
+        }
+        if (end === -1) continue;
+        let winner = def;
+        if (def.longerAltDef) {
+          const altEnd = longerAltEnd(def, text, offset);
+          if (altEnd > end) {
+            end = altEnd;
+            winner = def.longerAltDef;
+          }
+        }
+        if (!winner.skip) {
+          const image = winner.literal !== void 0 ? winner.literal : winner === def && matched !== null ? matched : text.slice(offset, end);
+          tokens.push(trackLines ? { type: winner.name, image, start: offset, end, line, col: offset - lineStart + 1 } : { type: winner.name, id: winner.id, image, start: offset, end });
+        }
+        if (trackLines) {
+          for (let j = offset; j < end; j++) {
+            if (text.charCodeAt(j) !== 10) continue;
+            line++;
+            lineStart = j + 1;
+          }
+        }
+        if (multiMode) {
+          if (winner.popMode) {
+            if (depth === 0) {
+              if (!recover) {
+                throw new LexError(
+                  `Token '${winner.name}' popped the initial lexer mode at ${describe(offset, line, lineStart)}`,
+                  { offset, line: trackLines ? line : 0, col: trackLines ? offset - lineStart + 1 : 0 }
+                );
+              }
+            } else {
+              mode = stack[--depth];
+            }
+          } else if (winner.pushMode !== -1) {
+            stack[depth++] = mode;
+            mode = modes[winner.pushMode];
+          }
+        }
+        offset = end;
+        continue outer;
+      }
+      if (recover) {
+        tokens.push(trackLines ? { type: errorTokenName, image: text[offset], start: offset, end: offset + 1, line, col: offset - lineStart + 1 } : { type: errorTokenName, id: -1, image: text[offset], start: offset, end: offset + 1 });
+        if (trackLines && text.charCodeAt(offset) === 10) {
+          line++;
+          lineStart = offset + 1;
+        }
+        offset++;
+        continue outer;
+      }
+      const column = offset - lineStart + 1;
+      throw new LexError(
+        trackLines ? `Unexpected character '${text[offset]}' at line ${line} column ${column}${inMode(mode)}` : `Unexpected character '${text[offset]}' at offset ${offset}${inMode(mode)}`,
+        { offset, line: trackLines ? line : 0, col: trackLines ? column : 0 }
+      );
+    }
+    return { tokens, checkpoints, mode, stack, depth, line, lineStart };
+  }
+  function tokenizeIncremental(text) {
+    if (typeof text !== "string") throw new TypeError("tokenizeIncremental: expected a string");
+    const result = scan(text, 0, initial, multiMode ? [] : [], 0, 1, 0);
+    return { text, tokens: result.tokens, checkpoints: result.checkpoints, eof: { modeId: result.mode.id, stack: snapshotStack(result.stack, result.depth), line: result.line, lineStart: result.lineStart, offset: text.length } };
+  }
+  function tokenFloor(tokens, offset) {
+    let lo = 0;
+    let hi = tokens.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (tokens[mid].start < offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+  function checkpointFloor(checkpoints, eof, offset) {
+    let lo = 0;
+    let hi = checkpoints.length + 1;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      const at = mid < checkpoints.length ? checkpoints[mid].offset : eof.offset;
+      if (at <= offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo - 1;
+  }
+  function checkpointAt(checkpoints, eof, offset) {
+    if (eof.offset === offset) return eof;
+    let lo = 0;
+    let hi = checkpoints.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (checkpoints[mid].offset < offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo < checkpoints.length && checkpoints[lo].offset === offset ? checkpoints[lo] : null;
+  }
+  function relex(newText, prev, edit) {
+    if (typeof newText !== "string") throw new TypeError("relex: expected a string");
+    const tokens = prev.tokens;
+    const checkpoints = prev.checkpoints;
+    const delta = edit.newEnd - edit.oldEnd;
+    let resumeIndex = checkpointFloor(checkpoints, prev.eof, edit.start);
+    if (resumeIndex > 0) resumeIndex--;
+    let boundary = resumeIndex < checkpoints.length ? checkpoints[resumeIndex].offset : prev.eof.offset;
+    let ti = tokenFloor(tokens, boundary);
+    while (ti > 0 && tokens[ti - 1].end === boundary) {
+      ti--;
+      boundary = tokens[ti].start;
+    }
+    if (boundary < (resumeIndex < checkpoints.length ? checkpoints[resumeIndex].offset : prev.eof.offset)) {
+      resumeIndex = checkpointFloor(checkpoints, prev.eof, boundary);
+    }
+    const resume = resumeIndex < checkpoints.length ? checkpoints[resumeIndex] : prev.eof;
+    const resumeMode = modes[resume.modeId];
+    const resumeStack = resume.stack.map((id) => modes[id]);
+    const scanned = scan(newText, resume.offset, resumeMode, resumeStack, resumeStack.length, resume.line, resume.lineStart);
+    const scannedEof = { modeId: scanned.mode.id, stack: snapshotStack(scanned.stack, scanned.depth), line: scanned.line, lineStart: scanned.lineStart, offset: newText.length };
+    const VERIFY_WINDOW = 4;
+    function convergenceVerified(oldOffset, newOffset) {
+      const oldStart = tokenFloor(tokens, oldOffset);
+      const freshStart = tokenFloor(scanned.tokens, newOffset);
+      for (let k = 0; k < VERIFY_WINDOW; k++) {
+        const oldTok = tokens[oldStart + k];
+        const freshTok = scanned.tokens[freshStart + k];
+        if (oldTok === void 0 && freshTok === void 0) return true;
+        if (oldTok === void 0 || freshTok === void 0) return false;
+        if (oldTok.type !== freshTok.type || oldTok.image !== freshTok.image) return false;
+      }
+      return true;
+    }
+    let convergeNewOffset = -1;
+    let convergeOldOffset = -1;
+    let convergeOldCheckpoint = null;
+    let freshCheckpointCount = 0;
+    for (let s = 0; s <= scanned.checkpoints.length; s++) {
+      const cp = s < scanned.checkpoints.length ? scanned.checkpoints[s] : scannedEof;
+      if (cp.offset < edit.newEnd) continue;
+      const oldOffset = cp.offset - delta;
+      const oldCp = checkpointAt(checkpoints, prev.eof, oldOffset);
+      if (oldCp && oldCp.modeId === cp.modeId && sameStack(oldCp.stack, cp.stack) && convergenceVerified(oldOffset, cp.offset)) {
+        convergeNewOffset = cp.offset;
+        convergeOldOffset = oldOffset;
+        convergeOldCheckpoint = oldCp;
+        freshCheckpointCount = s;
+        break;
+      }
+    }
+    const tokenFrom = tokenFloor(tokens, resume.offset);
+    const checkpointFrom = resumeIndex;
+    if (convergeNewOffset === -1) {
+      tokens.splice(tokenFrom, tokens.length - tokenFrom, ...scanned.tokens);
+      checkpoints.splice(checkpointFrom, checkpoints.length - checkpointFrom, ...scanned.checkpoints);
+      prev.text = newText;
+      prev.eof = scannedEof;
+      return prev;
+    }
+    const freshTokenCount = tokenFloor(scanned.tokens, convergeNewOffset);
+    const lineDelta = trackLines ? scanned.checkpoints.length > freshCheckpointCount ? scanned.checkpoints[freshCheckpointCount].line - convergeOldCheckpoint.line : scannedEof.line - convergeOldCheckpoint.line : 0;
+    const boundaryOldLineStart = convergeOldCheckpoint.lineStart;
+    const boundaryNewLineStart = (freshCheckpointCount < scanned.checkpoints.length ? scanned.checkpoints[freshCheckpointCount] : scannedEof).lineStart;
+    const shiftedLineStart = (oldLineStart) => oldLineStart === boundaryOldLineStart ? boundaryNewLineStart : oldLineStart + delta;
+    const tailTokenFrom = tokenFloor(tokens, convergeOldOffset);
+    for (let i = tailTokenFrom; i < tokens.length; i++) {
+      const t = tokens[i];
+      if (trackLines) {
+        const oldLineStart = t.start - (t.col - 1);
+        t.col = t.start + delta - shiftedLineStart(oldLineStart) + 1;
+        t.line += lineDelta;
+      }
+      t.start += delta;
+      t.end += delta;
+    }
+    tokens.splice(tokenFrom, tailTokenFrom - tokenFrom, ...scanned.tokens.slice(0, freshTokenCount));
+    const tailCheckpointFrom = checkpointFloor(checkpoints, prev.eof, convergeOldOffset - 1) + 1;
+    for (let i = tailCheckpointFrom; i < checkpoints.length; i++) {
+      const cp = checkpoints[i];
+      cp.lineStart = shiftedLineStart(cp.lineStart);
+      cp.line += lineDelta;
+      cp.offset += delta;
+    }
+    checkpoints.splice(checkpointFrom, tailCheckpointFrom - checkpointFrom, ...scanned.checkpoints.slice(0, freshCheckpointCount));
+    prev.text = newText;
+    prev.eof.lineStart = shiftedLineStart(prev.eof.lineStart);
+    prev.eof.line += lineDelta;
+    prev.eof.offset += delta;
+    return prev;
+  }
+  return { tokenize, tokenizeIncremental, relex, tokenNames, positions, modeNames };
 }
 
 // src/values.js