@tabnas/parser 0.2.0 → 0.3.0

package/dist/lexer.js

@@ -42,27 +42,50 @@ exports.Point = Point;
 const makePoint = (...params) => new Point(...params);
 exports.makePoint = makePoint;
 // A single lexed token: numeric token id, JS-typed value, raw source text, and match position.
+//
+// NOTE: `src` is a prototype accessor over a lazily-materialized backing
+// field — matchers that only know the token's span (space, line,
+// comment, quoted-string raw text) defer the substring until someone
+// actually reads it, so ignored tokens never allocate one. Reading
+// tkn.src always yields the correct string; the one observable
+// difference from a plain data property is that `src` is not an OWN
+// enumerable property, so Object.keys(tkn), {...tkn}, and
+// JSON.stringify(tkn) do not include it.
 class Token {
-    constructor(name, tin, val, src, pnt, use, why) {
+    #src; // Materialized source text (undefined = not yet).
+    #ref; // Full source backing the [sI, sI+len) span.
+    constructor(name, tin, val, src, pnt, use, why, ref, len) {
         this.isToken = true; // Marker discriminating Tokens from other values.
         this.name = types_1.EMPTY; // Token name (e.g. '#NR', '#ST').
         this.tin = -1; // Numeric token id corresponding to name.
         this.val = undefined; // JS-typed value (e.g. a number for #NR).
-        this.src = types_1.EMPTY; // Raw matching source text.
         this.sI = -1; // Source index where the match started.
         this.rI = -1; // Row where the match started.
         this.cI = -1; // Column where the match started.
         this.len = -1; // Length of src.
         this.name = name;
         this.tin = tin;
-        this.src = src;
+        this.#src = src;
+        this.#ref = ref;
         this.val = val;
         this.sI = pnt.sI;
         this.rI = pnt.rI;
         this.cI = pnt.cI;
         this.use = use;
         this.why = why;
-        this.len = null == src ? 0 : src.length;
+        this.len = null != len ? len : null == src ? 0 : src.length;
+    }
+    get src() {
+        let s = this.#src;
+        if (undefined === s) {
+            const ref = this.#ref;
+            s = this.#src =
+                undefined === ref ? types_1.EMPTY : ref.substring(this.sI, this.sI + this.len);
+        }
+        return s;
+    }
+    set src(s) {
+        this.#src = s;
     }
     resolveVal(rule, ctx) {
         let out = 'function' === typeof this.val ? this.val(rule, ctx) : this.val;
@@ -116,6 +139,8 @@ function guardedMatcher(mcfg, body) {
         if (!mcfg.lex)
             return undefined;
         if (mcfg.check) {
+            // Check hooks are user code and may read lex.fwd directly.
+            lex.refwd();
             const r = mcfg.check(lex);
             if (r && r.done)
                 return r.token;
@@ -329,23 +354,48 @@ const STRING_BODY_TABLE = new Int32Array([
     CONSUME | IS_ROW,
 ]);
 let makeFixedMatcher = (cfg, _opts) => {
-    let fixed = (0, utility_1.regexp)(null, '^(', cfg.rePart.fixed, ')');
+    // First-char dispatch table replacing the old anchored alternation
+    // regex: match cost per position becomes one charCode index plus (for
+    // multi-char tokens) a startsWith verify, with no match-array or
+    // capture substring allocation. Candidates sharing a first char are
+    // ordered longest-first, preserving the regex's longest-match-wins
+    // ordering (utility.ts sorts the alternation the same way).
+    const table = new Array(256);
+    const wide = []; // Tokens whose first char is >= U+0100.
+    const byLenDesc = (a, b) => b.len - a.len || (a.src < b.src ? -1 : a.src > b.src ? 1 : 0);
+    for (const fsrc of (0, utility_1.keys)(cfg.fixed.token)) {
+        const tin = cfg.fixed.token[fsrc];
+        if (null == tin || 0 === fsrc.length)
+            continue;
+        const cand = { src: fsrc, len: fsrc.length, tin };
+        const cc = fsrc.charCodeAt(0);
+        if (cc < 256) {
+            ;
+            (table[cc] = table[cc] || []).push(cand);
+        }
+        else {
+            wide.push(cand);
+        }
+    }
+    for (const cands of table) {
+        if (cands)
+            cands.sort(byLenDesc);
+    }
+    wide.sort(byLenDesc);
     return guardedMatcher(cfg.fixed, function fixedBody(lex) {
-        const mcfg = cfg.fixed;
-        let pnt = lex.pnt;
-        let fwd = lex.fwd;
-        let m = fwd.match(fixed);
-        if (m) {
-            let msrc = m[1];
-            let mlen = msrc.length;
-            if (0 < mlen) {
-                let tkn = undefined;
-                let tin = mcfg.token[msrc];
-                if (null != tin) {
-                    tkn = lex.token(tin, undefined, msrc, pnt);
-                    pnt.sI += mlen;
-                    pnt.cI += mlen;
-                }
+        const pnt = lex.pnt;
+        const src = lex.src;
+        const cc = src.charCodeAt(pnt.sI);
+        const cands = cc < 256 ? table[cc] : wide;
+        if (undefined === cands)
+            return undefined;
+        for (const cand of cands) {
+            // Single-char Latin-1 candidates already matched via the table
+            // index; longer (or wide-char) candidates verify in place.
+            if ((1 === cand.len && cc < 256) || src.startsWith(cand.src, pnt.sI)) {
+                const tkn = lex.token(cand.tin, undefined, cand.src, pnt);
+                pnt.sI += cand.len;
+                pnt.cI += cand.len;
                 return tkn;
             }
         }
@@ -367,7 +417,9 @@ let makeMatchMatcher = (cfg, _opts) => {
     }
     return guardedMatcher(cfg.match, function matchBody(lex, rule, tI = 0) {
         let pnt = lex.pnt;
-        let fwd = lex.fwd;
+        // Value/token matcher regexes are documented to run against the
+        // remainder string, so materialize it (memoized per position).
+        let fwd = lex.refwd();
         let oc = 'o' === rule.state ? 0 : 1;
         for (let valueMatcher of valueMatchers) {
             if (valueMatcher.match instanceof RegExp) {
@@ -474,9 +526,11 @@ let makeCommentMatcher = (cfg, opts) => {
     // so the table and class array are reused across comment matches.
     const lineRunSpec = buildLineRunSpec(cfg.line);
     const scanOut = { sI: 0, rI: 0, cI: 0 };
+    // The body walks lex.src with absolute indices (aI) — no remainder
+    // slice is materialized per position.
     return guardedMatcher(cfg.comment, function commentBody(lex) {
         let pnt = lex.pnt;
-        let fwd = lex.fwd;
+        let src = lex.src;
         let rI = pnt.rI;
         let cI = pnt.cI;
         // Single line comment.
@@ -485,32 +539,32 @@ let makeCommentMatcher = (cfg, opts) => {
         const lineChars = cfg.line.chars;
         const rowChars = cfg.line.rowChars;
         for (let mc of lineComments) {
-            if (fwd.startsWith(mc.start)) {
-                let fwdlen = fwd.length;
-                let fI = mc.start.length;
+            if (src.startsWith(mc.start, pnt.sI)) {
+                let srclen = src.length;
+                let aI = pnt.sI + mc.start.length;
                 cI += mc.start.length;
                 let suffixLen = 0;
                 let cc;
-                while (fI < fwdlen &&
-                    !((cc = fwd.charCodeAt(fI)) < 256
+                while (aI < srclen &&
+                    !((cc = src.charCodeAt(aI)) < 256
                         ? lineBM[cc]
-                        : lineChars[fwd[fI]])) {
-                    let n = commentSuffixMatch(fwd, fI, mc.suffixes);
+                        : lineChars[src[aI]])) {
+                    let n = commentSuffixMatch(src, aI, mc.suffixes);
                     if (n > 0) {
                         suffixLen = n;
                         break;
                     }
-                    n = commentSuffixFnMatch(lex, fI, mc.suffixFn);
+                    n = commentSuffixFnMatch(lex, aI, mc.suffixFn);
                     if (n > 0) {
                         suffixLen = n;
                         break;
                     }
                     cI++;
-                    fI++;
+                    aI++;
                 }
                 if (suffixLen > 0) {
                     // Consume the suffix as the tail of the comment body.
-                    fI += suffixLen;
+                    aI += suffixLen;
                     cI += suffixLen;
                 }
                 else if (mc.eatline) {
@@ -518,13 +572,12 @@ let makeCommentMatcher = (cfg, opts) => {
                     // a line char (not from a suffix match). cI is intentionally
                     // NOT pulled from scanOut — current semantics leave the
                     // pnt.cI at end-of-comment-body even after eating newlines.
-                    scan(fwd, fI, rI, cI, lineRunSpec, scanOut);
+                    scan(src, aI, rI, cI, lineRunSpec, scanOut);
                     rI = scanOut.rI;
-                    fI = scanOut.sI;
+                    aI = scanOut.sI;
                 }
-                let csrc = fwd.substring(0, fI);
-                let tkn = lex.token('#CM', undefined, csrc, pnt);
-                pnt.sI += csrc.length;
+                let tkn = lex.token('#CM', undefined, undefined, pnt, undefined, undefined, aI - pnt.sI);
+                pnt.sI = aI;
                 pnt.cI = cI;
                 pnt.rI = rI;
                 return tkn;
@@ -532,59 +585,57 @@ let makeCommentMatcher = (cfg, opts) => {
         }
         // Multiline comment.
         for (let mc of blockComments) {
-            if (fwd.startsWith(mc.start)) {
-                let fwdlen = fwd.length;
-                let fI = mc.start.length;
+            if (src.startsWith(mc.start, pnt.sI)) {
+                let srclen = src.length;
+                let aI = pnt.sI + mc.start.length;
                 let end = mc.end;
                 cI += mc.start.length;
                 let suffixLen = 0;
                 let cc;
-                while (fI < fwdlen && !fwd.startsWith(end, fI)) {
-                    let n = commentSuffixMatch(fwd, fI, mc.suffixes);
+                while (aI < srclen && !src.startsWith(end, aI)) {
+                    let n = commentSuffixMatch(src, aI, mc.suffixes);
                     if (n > 0) {
                         suffixLen = n;
                         break;
                     }
-                    n = commentSuffixFnMatch(lex, fI, mc.suffixFn);
+                    n = commentSuffixFnMatch(lex, aI, mc.suffixFn);
                     if (n > 0) {
                         suffixLen = n;
                         break;
                     }
-                    cc = fwd.charCodeAt(fI);
-                    if (cc < 256 ? rowBM[cc] : rowChars[fwd[fI]]) {
+                    cc = src.charCodeAt(aI);
+                    if (cc < 256 ? rowBM[cc] : rowChars[src[aI]]) {
                         rI++;
                         cI = 0;
                     }
                     cI++;
-                    fI++;
+                    aI++;
                 }
                 if (suffixLen > 0) {
                     // Advance through the consumed suffix, tracking newlines.
                     for (let k = 0; k < suffixLen; k++) {
-                        cc = fwd.charCodeAt(fI + k);
-                        if (cc < 256 ? rowBM[cc] : rowChars[fwd[fI + k]]) {
+                        cc = src.charCodeAt(aI + k);
+                        if (cc < 256 ? rowBM[cc] : rowChars[src[aI + k]]) {
                             rI++;
                             cI = 0;
                         }
                         cI++;
                     }
-                    let csrc = fwd.substring(0, fI + suffixLen);
-                    let tkn = lex.token('#CM', undefined, csrc, pnt);
-                    pnt.sI += csrc.length;
+                    let tkn = lex.token('#CM', undefined, undefined, pnt, undefined, undefined, aI + suffixLen - pnt.sI);
+                    pnt.sI = aI + suffixLen;
                     pnt.rI = rI;
                     pnt.cI = cI;
                     return tkn;
                 }
-                if (fwd.startsWith(end, fI)) {
+                if (src.startsWith(end, aI)) {
                     cI += end.length;
                     if (mc.eatline) {
-                        scan(fwd, fI, rI, cI, lineRunSpec, scanOut);
+                        scan(src, aI, rI, cI, lineRunSpec, scanOut);
                         rI = scanOut.rI;
-                        fI = scanOut.sI;
+                        aI = scanOut.sI;
                     }
-                    let csrc = fwd.substring(0, fI + end.length);
-                    let tkn = lex.token('#CM', undefined, csrc, pnt);
-                    pnt.sI += csrc.length;
+                    let tkn = lex.token('#CM', undefined, undefined, pnt, undefined, undefined, aI + end.length - pnt.sI);
+                    pnt.sI = aI + end.length;
                     pnt.rI = rI;
                     pnt.cI = cI;
                     return tkn;
@@ -627,29 +678,30 @@ function normalizeCommentSuffix(raw) {
     };
 }
 // commentSuffixMatch returns the length of the best suffix match at
-// fwd[fI:] or 0 if none matches. Suffixes are pre-sorted longest-first,
+// src[aI:] or 0 if none matches. Suffixes are pre-sorted longest-first,
 // so the first match is the best match.
-function commentSuffixMatch(fwd, fI, suffixes) {
+function commentSuffixMatch(src, aI, suffixes) {
     if (!suffixes || 0 === suffixes.length)
         return 0;
     for (let s of suffixes) {
-        if (fwd.substring(fI, fI + s.length) === s)
+        if (src.startsWith(s, aI))
             return s.length;
     }
     return 0;
 }
 // commentSuffixFnMatch probes the LexMatcher-form suffix terminator at
-// offset fI. Returns the length of the returned token's src (to be
-// consumed) or 0 if no termination. The lex point is snapshotted and
-// restored so a misbehaving matcher can't advance the stream itself.
-function commentSuffixFnMatch(lex, fI, fn) {
+// absolute source offset aI. Returns the length of the returned token's
+// src (to be consumed) or 0 if no termination. The lex point is
+// snapshotted and restored so a misbehaving matcher can't advance the
+// stream itself.
+function commentSuffixFnMatch(lex, aI, fn) {
     if (!fn)
         return 0;
     let pnt = lex.pnt;
     let savedSI = pnt.sI;
     let savedRI = pnt.rI;
     let savedCI = pnt.cI;
-    pnt.sI = savedSI + fI;
+    pnt.sI = aI;
     let tkn;
     try {
         tkn = fn(lex, undefined);
@@ -666,9 +718,13 @@ function commentSuffixFnMatch(lex, fI, fn) {
 // Match text, checking for literal values, optionally followed by a fixed token.
 // Text strings are terminated by end markers.
 let makeTextMatcher = (cfg, opts) => {
-    let ender = (0, utility_1.regexp)(cfg.line.lex ? null : 's', '^(.*?)', ...cfg.rePart.ender);
+    // Sticky ('y') so the regex runs against the full source at an offset
+    // (ender.lastIndex = pnt.sI) instead of an allocated remainder slice.
+    let ender = (0, utility_1.regexp)(cfg.line.lex ? 'y' : 'ys', '(.*?)', ...cfg.rePart.ender);
     return function textMatcher(lex) {
         if (cfg.text.check) {
+            // Check hooks are user code and may read lex.fwd directly.
+            lex.refwd();
             let check = cfg.text.check(lex);
             if (check && check.done) {
                 return check.token;
@@ -676,10 +732,10 @@ let makeTextMatcher = (cfg, opts) => {
         }
         let mcfg = cfg.text;
         let pnt = lex.pnt;
-        let fwd = lex.fwd;
         let def = cfg.value.def;
         let defre = cfg.value.defre;
-        let m = fwd.match(ender);
+        ender.lastIndex = pnt.sI;
+        let m = ender.exec(lex.src);
         if (m) {
             let msrc = m[1];
             let tsrc = m[2];
@@ -702,8 +758,10 @@ let makeTextMatcher = (cfg, opts) => {
                             // utility.ts) — iteration order is deterministic.
                             for (let vspec of defre) {
                                 if (vspec.match) {
-                                    // If consume, assume regexp starts with ^.
-                                    let res = vspec.match.exec(vspec.consume ? fwd : msrc);
+                                    // If consume, assume regexp starts with ^. Consuming
+                                    // value regexes are user-supplied and match against the
+                                    // remainder string (materialized lazily, memoized).
+                                    let res = vspec.match.exec(vspec.consume ? lex.refwd() : msrc);
                                     // Must match entire text.
                                     if (res && (vspec.consume || res[0].length === msrc.length)) {
                                         let remsrc = res[0];
@@ -748,8 +806,10 @@ let makeTextMatcher = (cfg, opts) => {
 exports.makeTextMatcher = makeTextMatcher;
 let makeNumberMatcher = (cfg, _opts) => {
     let mcfg = cfg.number;
-    let ender = (0, utility_1.regexp)(null, [
-        '^([-+]?(0(',
+    // Sticky ('y') so the regex runs against the full source at an offset
+    // (ender.lastIndex = pnt.sI) instead of an allocated remainder slice.
+    let ender = (0, utility_1.regexp)('y', [
+        '([-+]?(0(',
         [
             mcfg.hex ? 'x[0-9a-fA-F_]+' : null,
             mcfg.oct ? 'o[0-7_]+' : null,
@@ -767,12 +827,13 @@ let makeNumberMatcher = (cfg, _opts) => {
     let numberSep = mcfg.sep
         ? (0, utility_1.regexp)('g', (0, utility_1.escre)(mcfg.sepChar))
         : undefined;
+    let numberSepChar = mcfg.sep ? mcfg.sepChar : undefined;
     return guardedMatcher(cfg.number, function numberBody(lex) {
         mcfg = cfg.number;
         let pnt = lex.pnt;
-        let fwd = lex.fwd;
         let valdef = cfg.value.def;
-        let m = fwd.match(ender);
+        ender.lastIndex = pnt.sI;
+        let m = ender.exec(lex.src);
         if (m) {
             let msrc = m[1];
             let tsrc = m[9]; // NOTE: count parens in numberEnder!
@@ -787,7 +848,11 @@ let makeNumberMatcher = (cfg, _opts) => {
                         out = lex.token('#VL', vs.val, msrc, pnt);
                     }
                     else {
-                        let nstr = numberSep ? msrc.replace(numberSep, '') : msrc;
+                        // Strip separators only when one is actually present — the
+                        // common separator-free number skips the regex replace.
+                        let nstr = numberSep && numberSepChar && -1 < msrc.indexOf(numberSepChar)
+                            ? msrc.replace(numberSep, '')
+                            : msrc;
                         let num = +nstr;
                         // Special case: +- prefix of 0x... format
                         if (isNaN(num)) {
@@ -881,7 +946,11 @@ let makeStringMatcher = (cfg, opts) => {
         let sI = startSI + 1;
         let rI = startRI;
         let cI = pnt.cI + 1;
-        const buf = [];
+        // Escape-free fast path: until the first escape or replace is
+        // processed the value is the contiguous source run after the opening
+        // quote, so no segment buffer is needed — a clean quoted string costs
+        // one substring instead of a buffer, per-segment pushes, and a join.
+        let buf = undefined;
         while (sI < srclen) {
             // Body scan: consume body chars (and multi-line newlines)
             // until something interesting (quote, escape, control, replace).
@@ -890,15 +959,16 @@ let makeStringMatcher = (cfg, opts) => {
             sI = scanOut.sI;
             rI = scanOut.rI;
             cI = scanOut.cI;
-            if (bodyStart < sI)
+            if (undefined !== buf && bodyStart < sI)
                 buf.push(src.substring(bodyStart, sI));
             if (sI >= srclen)
                 break;
             const cc = src.charCodeAt(sI);
             // Closing quote — string done.
             if (cc === qcc) {
+                const val = undefined === buf ? src.substring(startSI + 1, sI) : buf.join(types_1.EMPTY);
                 sI++;
-                const tkn = lex.token('#ST', buf.join(types_1.EMPTY), src.substring(startSI, sI), pnt);
+                const tkn = lex.token('#ST', val, undefined, pnt, undefined, undefined, sI - startSI);
                 pnt.sI = sI;
                 pnt.rI = rI;
                 pnt.cI = cI + 1;
@@ -908,6 +978,9 @@ let makeStringMatcher = (cfg, opts) => {
             if (hasReplace) {
                 const rs = replaceCodeMap[cc];
                 if (rs !== undefined) {
+                    if (undefined === buf) {
+                        buf = startSI + 1 < sI ? [src.substring(startSI + 1, sI)] : [];
+                    }
                     buf.push(rs);
                     sI++;
                     cI++;
@@ -916,6 +989,9 @@ let makeStringMatcher = (cfg, opts) => {
             }
             // Escape sequence.
             if (cc === escCharCode) {
+                if (undefined === buf) {
+                    buf = startSI + 1 < sI ? [src.substring(startSI + 1, sI)] : [];
+                }
                 sI++;
                 cI++;
                 if (sI >= srclen)
@@ -1048,8 +1124,7 @@ let makeLineMatcher = (cfg, _opts) => {
                 sI++;
             }
             if (pnt.sI < sI) {
-                const msrc = src.substring(pnt.sI, sI);
-                const tkn = lex.token('#LN', undefined, msrc, pnt);
+                const tkn = lex.token('#LN', undefined, undefined, pnt, undefined, undefined, sI - pnt.sI);
                 pnt.sI = sI;
                 pnt.rI = rI;
                 pnt.cI = 1;
@@ -1058,8 +1133,7 @@ let makeLineMatcher = (cfg, _opts) => {
             return;
         }
         if (scan(src, pnt.sI, pnt.rI, pnt.cI, spec, out)) {
-            const msrc = src.substring(pnt.sI, out.sI);
-            const tkn = lex.token('#LN', undefined, msrc, pnt);
+            const tkn = lex.token('#LN', undefined, undefined, pnt, undefined, undefined, out.sI - pnt.sI);
             pnt.sI = out.sI;
             pnt.rI = out.rI;
             pnt.cI = 1;
@@ -1078,8 +1152,7 @@ let makeSpaceMatcher = (cfg, _opts) => {
     return guardedMatcher(cfg.space, function spaceBody(lex) {
         const { pnt, src } = lex;
         if (scan(src, pnt.sI, pnt.rI, pnt.cI, spec, out)) {
-            const msrc = src.substring(pnt.sI, out.sI);
-            const tkn = lex.token('#SP', undefined, msrc, pnt);
+            const tkn = lex.token('#SP', undefined, undefined, pnt, undefined, undefined, out.sI - pnt.sI);
             pnt.sI = out.sI;
             pnt.rI = out.rI;
             pnt.cI = out.cI;
@@ -1113,10 +1186,23 @@ function subMatchFixed(lex, first, tsrc) {
     }
     return out;
 }
+// Built-in matcher names (the `matcher` annotation set in configure()).
+// These matchers scan lex.src at pnt offsets or call refwd() themselves;
+// anything else in the matcher list gets a fresh lex.fwd before running.
+const BUILTIN_MATCHER = {
+    match: 1, fixed: 1, space: 1, line: 1,
+    string: 1, comment: 1, number: 1, text: 1,
+};
 // Lexer driver: holds the scan Point and runs the configured matchers in order via next().
 class Lex {
+    // Slice the remainder lazily and memoize on the position — the slice
+    // is only materialized for consumers that genuinely need a remainder
+    // string (custom matchers, value.defre regexes), never once per token.
     refwd() {
-        this.fwd = this.src.substring(this.pnt.sI);
+        if (this.fwdSI !== this.pnt.sI) {
+            this.fwd = this.src.substring(this.pnt.sI);
+            this.fwdSI = this.pnt.sI;
+        }
         return this.fwd;
     }
     constructor(ctx) {
@@ -1125,12 +1211,16 @@ class Lex {
         this.cfg = {}; // Resolved configuration.
         this.pnt = makePoint(-1); // Current scan position.
         this.fwd = types_1.EMPTY; // Source from pnt.sI onward (the unconsumed remainder).
+        this.fwdSI = -1; // pnt.sI the current fwd slice was taken at (memo key).
         this.ctx = ctx;
         this.src = ctx.src();
         this.cfg = ctx.cfg;
         this.pnt = makePoint(this.src.length);
     }
-    token(ref, val, src, pnt, use, why) {
+    // Create a token. Pass `src` when the matcher already holds the
+    // matched text; pass src=undefined with an explicit `len` to defer
+    // the substring to first read (a span over this lexer's source).
+    token(ref, val, src, pnt, use, why, len) {
         let tin;
         let name;
         if ('string' === typeof ref) {
@@ -1141,7 +1231,7 @@ class Lex {
             tin = ref;
             name = (0, utility_1.tokenize)(ref, this.cfg);
         }
-        let tkn = makeToken(name, tin, val, src, pnt || this.pnt, use, why);
+        let tkn = makeToken(name, tin, val, src, pnt || this.pnt, use, why, this.src, len);
         return tkn;
     }
     next(rule, alt, altI, tI) {
@@ -1160,9 +1250,19 @@ class Lex {
             tkn = pnt.end;
         }
         else {
-            this.fwd = this.src.substring(pnt.sI);
+            // First-char dispatch: run only the matchers that could produce a
+            // token starting with this char (non-Latin-1 chars and configs
+            // without a table run the full pipeline).
+            const dispatch = this.cfg.lex.dispatch;
+            const cc = this.src.charCodeAt(pnt.sI);
+            const mats = undefined === dispatch
+                ? this.cfg.lex.match
+                : dispatch[cc < 256 ? cc : 256];
             try {
-                for (let mat of this.cfg.lex.match) {
+                for (let mat of mats) {
+                    if (undefined === BUILTIN_MATCHER[mat.matcher]) {
+                        this.refwd();
+                    }
                     if ((tkn = mat(this, rule, tI))) {
                         match = mat;
                         break;