@windrun-huaiin/third-ui 13.1.4 → 14.0.0

package/dist/node_modules/.pnpm/chevrotain@11.1.2/node_modules/chevrotain/lib/src/scan/lexer_public.mjs

@@ -1,657 +0,0 @@
-import { LineTerminatorOptimizedTester, DEFAULT_MODE, performRuntimeChecks, performWarningRuntimeChecks, validatePatterns, analyzeTokenTypes, cloneEmptyGroups, charCodeToOptimizedIndex } from './lexer.mjs';
-import { PRINT_WARNING } from '../../../../../../@chevrotain_utils@11.1.2/node_modules/@chevrotain/utils/lib/src/print.mjs';
-import { timer } from '../../../../../../@chevrotain_utils@11.1.2/node_modules/@chevrotain/utils/lib/src/timer.mjs';
-import { toFastProperties } from '../../../../../../@chevrotain_utils@11.1.2/node_modules/@chevrotain/utils/lib/src/to-fast-properties.mjs';
-import { augmentTokenTypes } from './tokens.mjs';
-import { defaultLexerErrorProvider } from './lexer_errors_public.mjs';
-import { clearRegExpParserCache } from './reg_exp_parser.mjs';
-import assign from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/assign.mjs';
-import isArray from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/isArray.mjs';
-import clone from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/clone.mjs';
-import forEach from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/forEach.mjs';
-import reject from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/reject.mjs';
-import keys from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/keys.mjs';
-import isEmpty from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/isEmpty.mjs';
-import map from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/map.mjs';
-import noop from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/noop.mjs';
-import identity from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/identity.mjs';
-import reduce from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/reduce.mjs';
-import last from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/last.mjs';
-import isUndefined from '../../../../../../lodash-es@4.17.23/node_modules/lodash-es/isUndefined.mjs';
-
-var LexerDefinitionErrorType;
-(function (LexerDefinitionErrorType) {
-    LexerDefinitionErrorType[LexerDefinitionErrorType["MISSING_PATTERN"] = 0] = "MISSING_PATTERN";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_PATTERN"] = 1] = "INVALID_PATTERN";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["EOI_ANCHOR_FOUND"] = 2] = "EOI_ANCHOR_FOUND";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["UNSUPPORTED_FLAGS_FOUND"] = 3] = "UNSUPPORTED_FLAGS_FOUND";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["DUPLICATE_PATTERNS_FOUND"] = 4] = "DUPLICATE_PATTERNS_FOUND";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_GROUP_TYPE_FOUND"] = 5] = "INVALID_GROUP_TYPE_FOUND";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["PUSH_MODE_DOES_NOT_EXIST"] = 6] = "PUSH_MODE_DOES_NOT_EXIST";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE"] = 7] = "MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY"] = 8] = "MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST"] = 9] = "MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED"] = 10] = "LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["SOI_ANCHOR_FOUND"] = 11] = "SOI_ANCHOR_FOUND";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["EMPTY_MATCH_PATTERN"] = 12] = "EMPTY_MATCH_PATTERN";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["NO_LINE_BREAKS_FLAGS"] = 13] = "NO_LINE_BREAKS_FLAGS";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["UNREACHABLE_PATTERN"] = 14] = "UNREACHABLE_PATTERN";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["IDENTIFY_TERMINATOR"] = 15] = "IDENTIFY_TERMINATOR";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["CUSTOM_LINE_BREAK"] = 16] = "CUSTOM_LINE_BREAK";
-    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE"] = 17] = "MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE";
-})(LexerDefinitionErrorType || (LexerDefinitionErrorType = {}));
-const DEFAULT_LEXER_CONFIG = {
-    deferDefinitionErrorsHandling: false,
-    positionTracking: "full",
-    lineTerminatorsPattern: /\n|\r\n?/g,
-    lineTerminatorCharacters: ["\n", "\r"],
-    ensureOptimizations: false,
-    safeMode: false,
-    errorMessageProvider: defaultLexerErrorProvider,
-    traceInitPerf: false,
-    skipValidations: false,
-    recoveryEnabled: true,
-};
-Object.freeze(DEFAULT_LEXER_CONFIG);
-class Lexer {
-    constructor(lexerDefinition, config = DEFAULT_LEXER_CONFIG) {
-        this.lexerDefinition = lexerDefinition;
-        this.lexerDefinitionErrors = [];
-        this.lexerDefinitionWarning = [];
-        this.patternIdxToConfig = {};
-        this.charCodeToPatternIdxToConfig = {};
-        this.modes = [];
-        this.emptyGroups = {};
-        this.trackStartLines = true;
-        this.trackEndLines = true;
-        this.hasCustom = false;
-        this.canModeBeOptimized = {};
-        // Duplicated from the parser's perf trace trait to allow future extraction
-        // of the lexer to a separate package.
-        this.TRACE_INIT = (phaseDesc, phaseImpl) => {
-            // No need to optimize this using NOOP pattern because
-            // It is not called in a hot spot...
-            if (this.traceInitPerf === true) {
-                this.traceInitIndent++;
-                const indent = new Array(this.traceInitIndent + 1).join("\t");
-                if (this.traceInitIndent < this.traceInitMaxIdent) {
-                    console.log(`${indent}--> <${phaseDesc}>`);
-                }
-                const { time, value } = timer(phaseImpl);
-                /* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
-                const traceMethod = time > 10 ? console.warn : console.log;
-                if (this.traceInitIndent < this.traceInitMaxIdent) {
-                    traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`);
-                }
-                this.traceInitIndent--;
-                return value;
-            }
-            else {
-                return phaseImpl();
-            }
-        };
-        if (typeof config === "boolean") {
-            throw Error("The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
-                "a boolean 2nd argument is no longer supported");
-        }
-        this.config = assign({}, DEFAULT_LEXER_CONFIG, config);
-        const traceInitVal = this.config.traceInitPerf;
-        if (traceInitVal === true) {
-            this.traceInitMaxIdent = Infinity;
-            this.traceInitPerf = true;
-        }
-        else if (typeof traceInitVal === "number") {
-            this.traceInitMaxIdent = traceInitVal;
-            this.traceInitPerf = true;
-        }
-        this.traceInitIndent = -1;
-        this.TRACE_INIT("Lexer Constructor", () => {
-            let actualDefinition;
-            let hasOnlySingleMode = true;
-            this.TRACE_INIT("Lexer Config handling", () => {
-                if (this.config.lineTerminatorsPattern ===
-                    DEFAULT_LEXER_CONFIG.lineTerminatorsPattern) {
-                    // optimized built-in implementation for the defaults definition of lineTerminators
-                    this.config.lineTerminatorsPattern = LineTerminatorOptimizedTester;
-                }
-                else {
-                    if (this.config.lineTerminatorCharacters ===
-                        DEFAULT_LEXER_CONFIG.lineTerminatorCharacters) {
-                        throw Error("Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
-                            "\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS");
-                    }
-                }
-                if (config.safeMode && config.ensureOptimizations) {
-                    throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');
-                }
-                this.trackStartLines = /full|onlyStart/i.test(this.config.positionTracking);
-                this.trackEndLines = /full/i.test(this.config.positionTracking);
-                // Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
-                if (isArray(lexerDefinition)) {
-                    actualDefinition = {
-                        modes: { defaultMode: clone(lexerDefinition) },
-                        defaultMode: DEFAULT_MODE,
-                    };
-                }
-                else {
-                    // no conversion needed, input should already be a IMultiModeLexerDefinition
-                    hasOnlySingleMode = false;
-                    actualDefinition = clone(lexerDefinition);
-                }
-            });
-            if (this.config.skipValidations === false) {
-                this.TRACE_INIT("performRuntimeChecks", () => {
-                    this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(performRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
-                });
-                this.TRACE_INIT("performWarningRuntimeChecks", () => {
-                    this.lexerDefinitionWarning = this.lexerDefinitionWarning.concat(performWarningRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
-                });
-            }
-            // for extra robustness to avoid throwing a none informative error message
-            actualDefinition.modes = actualDefinition.modes
-                ? actualDefinition.modes
-                : {};
-            // an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
-            // this transformation is to increase robustness in the case of partially invalid lexer definition.
-            forEach(actualDefinition.modes, (currModeValue, currModeName) => {
-                actualDefinition.modes[currModeName] = reject(currModeValue, (currTokType) => isUndefined(currTokType));
-            });
-            const allModeNames = keys(actualDefinition.modes);
-            forEach(actualDefinition.modes, (currModDef, currModName) => {
-                this.TRACE_INIT(`Mode: <${currModName}> processing`, () => {
-                    this.modes.push(currModName);
-                    if (this.config.skipValidations === false) {
-                        this.TRACE_INIT(`validatePatterns`, () => {
-                            this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(validatePatterns(currModDef, allModeNames));
-                        });
-                    }
-                    // If definition errors were encountered, the analysis phase may fail unexpectedly/
-                    // Considering a lexer with definition errors may never be used, there is no point
-                    // to performing the analysis anyhow...
-                    if (isEmpty(this.lexerDefinitionErrors)) {
-                        augmentTokenTypes(currModDef);
-                        let currAnalyzeResult;
-                        this.TRACE_INIT(`analyzeTokenTypes`, () => {
-                            currAnalyzeResult = analyzeTokenTypes(currModDef, {
-                                lineTerminatorCharacters: this.config.lineTerminatorCharacters,
-                                positionTracking: config.positionTracking,
-                                ensureOptimizations: config.ensureOptimizations,
-                                safeMode: config.safeMode,
-                                tracer: this.TRACE_INIT,
-                            });
-                        });
-                        this.patternIdxToConfig[currModName] =
-                            currAnalyzeResult.patternIdxToConfig;
-                        this.charCodeToPatternIdxToConfig[currModName] =
-                            currAnalyzeResult.charCodeToPatternIdxToConfig;
-                        this.emptyGroups = assign({}, this.emptyGroups, currAnalyzeResult.emptyGroups);
-                        this.hasCustom = currAnalyzeResult.hasCustom || this.hasCustom;
-                        this.canModeBeOptimized[currModName] =
-                            currAnalyzeResult.canBeOptimized;
-                    }
-                });
-            });
-            this.defaultMode = actualDefinition.defaultMode;
-            if (!isEmpty(this.lexerDefinitionErrors) &&
-                !this.config.deferDefinitionErrorsHandling) {
-                const allErrMessages = map(this.lexerDefinitionErrors, (error) => {
-                    return error.message;
-                });
-                const allErrMessagesString = allErrMessages.join("-----------------------\n");
-                throw new Error("Errors detected in definition of Lexer:\n" + allErrMessagesString);
-            }
-            // Only print warning if there are no errors, This will avoid pl
-            forEach(this.lexerDefinitionWarning, (warningDescriptor) => {
-                PRINT_WARNING(warningDescriptor.message);
-            });
-            this.TRACE_INIT("Choosing sub-methods implementations", () => {
-                // Choose the relevant internal implementations for this specific parser.
-                // These implementations should be in-lined by the JavaScript engine
-                // to provide optimal performance in each scenario.
-                if (hasOnlySingleMode) {
-                    this.handleModes = noop;
-                }
-                if (this.trackStartLines === false) {
-                    this.computeNewColumn = identity;
-                }
-                if (this.trackEndLines === false) {
-                    this.updateTokenEndLineColumnLocation = noop;
-                }
-                if (/full/i.test(this.config.positionTracking)) {
-                    this.createTokenInstance = this.createFullToken;
-                }
-                else if (/onlyStart/i.test(this.config.positionTracking)) {
-                    this.createTokenInstance = this.createStartOnlyToken;
-                }
-                else if (/onlyOffset/i.test(this.config.positionTracking)) {
-                    this.createTokenInstance = this.createOffsetOnlyToken;
-                }
-                else {
-                    throw Error(`Invalid <positionTracking> config option: "${this.config.positionTracking}"`);
-                }
-                if (this.hasCustom) {
-                    this.addToken = this.addTokenUsingPush;
-                    this.handlePayload = this.handlePayloadWithCustom;
-                }
-                else {
-                    this.addToken = this.addTokenUsingMemberAccess;
-                    this.handlePayload = this.handlePayloadNoCustom;
-                }
-            });
-            this.TRACE_INIT("Failed Optimization Warnings", () => {
-                const unOptimizedModes = reduce(this.canModeBeOptimized, (cannotBeOptimized, canBeOptimized, modeName) => {
-                    if (canBeOptimized === false) {
-                        cannotBeOptimized.push(modeName);
-                    }
-                    return cannotBeOptimized;
-                }, []);
-                if (config.ensureOptimizations && !isEmpty(unOptimizedModes)) {
-                    throw Error(`Lexer Modes: < ${unOptimizedModes.join(", ")} > cannot be optimized.\n` +
-                        '\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
-                        "\t Or inspect the console log for details on how to resolve these issues.");
-                }
-            });
-            this.TRACE_INIT("clearRegExpParserCache", () => {
-                clearRegExpParserCache();
-            });
-            this.TRACE_INIT("toFastProperties", () => {
-                toFastProperties(this);
-            });
-        });
-    }
-    tokenize(text, initialMode = this.defaultMode) {
-        if (!isEmpty(this.lexerDefinitionErrors)) {
-            const allErrMessages = map(this.lexerDefinitionErrors, (error) => {
-                return error.message;
-            });
-            const allErrMessagesString = allErrMessages.join("-----------------------\n");
-            throw new Error("Unable to Tokenize because Errors detected in definition of Lexer:\n" +
-                allErrMessagesString);
-        }
-        return this.tokenizeInternal(text, initialMode);
-    }
-    // There is quite a bit of duplication between this and "tokenizeInternalLazy"
-    // This is intentional due to performance considerations.
-    // this method also used quite a bit of `!` none null assertions because it is too optimized
-    // for `tsc` to always understand it is "safe"
-    tokenizeInternal(text, initialMode) {
-        let i, j, k, matchAltImage, longerAlt, matchedImage, payload, altPayload, imageLength, group, tokType, newToken, errLength, msg, match;
-        const orgText = text;
-        const orgLength = orgText.length;
-        let offset = 0;
-        let matchedTokensIndex = 0;
-        // initializing the tokensArray to the "guessed" size.
-        // guessing too little will still reduce the number of array re-sizes on pushes.
-        // guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
-        // but would still have a faster runtime by avoiding (All but one) array resizing.
-        const guessedNumberOfTokens = this.hasCustom
-            ? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
-            : Math.floor(text.length / 10);
-        const matchedTokens = new Array(guessedNumberOfTokens);
-        const errors = [];
-        let line = this.trackStartLines ? 1 : undefined;
-        let column = this.trackStartLines ? 1 : undefined;
-        const groups = cloneEmptyGroups(this.emptyGroups);
-        const trackLines = this.trackStartLines;
-        const lineTerminatorPattern = this.config.lineTerminatorsPattern;
-        let currModePatternsLength = 0;
-        let patternIdxToConfig = [];
-        let currCharCodeToPatternIdxToConfig = [];
-        const modeStack = [];
-        const emptyArray = [];
-        Object.freeze(emptyArray);
-        let isOptimizedMode = false;
-        const pop_mode = (popToken) => {
-            // TODO: perhaps avoid this error in the edge case there is no more input?
-            if (modeStack.length === 1 &&
-                // if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
-                // So no error should occur.
-                popToken.tokenType.PUSH_MODE === undefined) {
-                // if we try to pop the last mode there lexer will no longer have ANY mode.
-                // thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
-                const msg = this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(popToken);
-                errors.push({
-                    offset: popToken.startOffset,
-                    line: popToken.startLine,
-                    column: popToken.startColumn,
-                    length: popToken.image.length,
-                    message: msg,
-                });
-            }
-            else {
-                modeStack.pop();
-                const newMode = last(modeStack);
-                patternIdxToConfig = this.patternIdxToConfig[newMode];
-                currCharCodeToPatternIdxToConfig =
-                    this.charCodeToPatternIdxToConfig[newMode];
-                currModePatternsLength = patternIdxToConfig.length;
-                const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
-                if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
-                    isOptimizedMode = true;
-                }
-                else {
-                    isOptimizedMode = false;
-                }
-            }
-        };
-        function push_mode(newMode) {
-            modeStack.push(newMode);
-            currCharCodeToPatternIdxToConfig =
-                this.charCodeToPatternIdxToConfig[newMode];
-            patternIdxToConfig = this.patternIdxToConfig[newMode];
-            currModePatternsLength = patternIdxToConfig.length;
-            currModePatternsLength = patternIdxToConfig.length;
-            const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
-            if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
-                isOptimizedMode = true;
-            }
-            else {
-                isOptimizedMode = false;
-            }
-        }
-        // this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
-        // seem to matter performance wise.
-        push_mode.call(this, initialMode);
-        let currConfig;
-        const recoveryEnabled = this.config.recoveryEnabled;
-        while (offset < orgLength) {
-            matchedImage = null;
-            imageLength = -1;
-            const nextCharCode = orgText.charCodeAt(offset);
-            let chosenPatternIdxToConfig;
-            if (isOptimizedMode) {
-                const optimizedCharIdx = charCodeToOptimizedIndex(nextCharCode);
-                const possiblePatterns = currCharCodeToPatternIdxToConfig[optimizedCharIdx];
-                chosenPatternIdxToConfig =
-                    possiblePatterns !== undefined ? possiblePatterns : emptyArray;
-            }
-            else {
-                chosenPatternIdxToConfig = patternIdxToConfig;
-            }
-            const chosenPatternsLength = chosenPatternIdxToConfig.length;
-            for (i = 0; i < chosenPatternsLength; i++) {
-                currConfig = chosenPatternIdxToConfig[i];
-                const currPattern = currConfig.pattern;
-                payload = null;
-                // manually in-lined because > 600 chars won't be in-lined in V8
-                const singleCharCode = currConfig.short;
-                if (singleCharCode !== false) {
-                    if (nextCharCode === singleCharCode) {
-                        // single character string
-                        imageLength = 1;
-                        matchedImage = currPattern;
-                    }
-                }
-                else if (currConfig.isCustom === true) {
-                    match = currPattern.exec(orgText, offset, matchedTokens, groups);
-                    if (match !== null) {
-                        matchedImage = match[0];
-                        imageLength = matchedImage.length;
-                        if (match.payload !== undefined) {
-                            payload = match.payload;
-                        }
-                    }
-                    else {
-                        matchedImage = null;
-                    }
-                }
-                else {
-                    currPattern.lastIndex = offset;
-                    imageLength = this.matchLength(currPattern, text, offset);
-                }
-                // longer alts handling
-                if (imageLength !== -1) {
-                    // even though this pattern matched we must try a another longer alternative.
-                    // this can be used to prioritize keywords over identifiers
-                    longerAlt = currConfig.longerAlt;
-                    if (longerAlt !== undefined) {
-                        matchedImage = text.substring(offset, offset + imageLength);
-                        const longerAltLength = longerAlt.length;
-                        for (k = 0; k < longerAltLength; k++) {
-                            const longerAltConfig = patternIdxToConfig[longerAlt[k]];
-                            const longerAltPattern = longerAltConfig.pattern;
-                            altPayload = null;
-                            // single Char can never be a longer alt so no need to test it.
-                            // manually in-lined because > 600 chars won't be in-lined in V8
-                            if (longerAltConfig.isCustom === true) {
-                                match = longerAltPattern.exec(orgText, offset, matchedTokens, groups);
-                                if (match !== null) {
-                                    matchAltImage = match[0];
-                                    if (match.payload !== undefined) {
-                                        altPayload = match.payload;
-                                    }
-                                }
-                                else {
-                                    matchAltImage = null;
-                                }
-                            }
-                            else {
-                                longerAltPattern.lastIndex = offset;
-                                matchAltImage = this.match(longerAltPattern, text, offset);
-                            }
-                            if (matchAltImage && matchAltImage.length > matchedImage.length) {
-                                matchedImage = matchAltImage;
-                                imageLength = matchAltImage.length;
-                                payload = altPayload;
-                                currConfig = longerAltConfig;
-                                // Exit the loop early after matching one of the longer alternatives
-                                // The first matched alternative takes precedence
-                                break;
-                            }
-                        }
-                    }
-                    break;
-                }
-            }
-            // successful match
-            if (imageLength !== -1) {
-                group = currConfig.group;
-                if (group !== undefined) {
-                    matchedImage =
-                        matchedImage !== null
-                            ? matchedImage // for custom Tokens we will already have the `matchedImage`
-                            : text.substring(offset, offset + imageLength);
-                    tokType = currConfig.tokenTypeIdx;
-                    newToken = this.createTokenInstance(matchedImage, offset, tokType, currConfig.tokenType, line, column, imageLength);
-                    this.handlePayload(newToken, payload);
-                    if (group === false) {
-                        matchedTokensIndex = this.addToken(matchedTokens, matchedTokensIndex, newToken);
-                    }
-                    else {
-                        groups[group].push(newToken);
-                    }
-                }
-                // line terminator handling
-                if (trackLines === true && currConfig.canLineTerminator === true) {
-                    let numOfLTsInMatch = 0;
-                    let foundTerminator;
-                    let lastLTEndOffset;
-                    lineTerminatorPattern.lastIndex = 0;
-                    do {
-                        // only for skipped tokens the matchedImage may be null at this point
-                        matchedImage =
-                            matchedImage !== null
-                                ? matchedImage
-                                : text.substring(offset, offset + imageLength);
-                        foundTerminator = lineTerminatorPattern.test(matchedImage);
-                        if (foundTerminator === true) {
-                            lastLTEndOffset = lineTerminatorPattern.lastIndex - 1;
-                            numOfLTsInMatch++;
-                        }
-                    } while (foundTerminator === true);
-                    if (numOfLTsInMatch !== 0) {
-                        line = line + numOfLTsInMatch;
-                        column = imageLength - lastLTEndOffset;
-                        this.updateTokenEndLineColumnLocation(newToken, group, lastLTEndOffset, numOfLTsInMatch, line, column, imageLength);
-                    }
-                    else {
-                        column = this.computeNewColumn(column, imageLength);
-                    }
-                }
-                else {
-                    column = this.computeNewColumn(column, imageLength);
-                }
-                offset = offset + imageLength;
-                // will be NOOP if no modes present
-                this.handleModes(currConfig, pop_mode, push_mode, newToken);
-            }
-            else {
-                // error recovery, drop characters until we identify a valid token's start point
-                const errorStartOffset = offset;
-                const errorLine = line;
-                const errorColumn = column;
-                let foundResyncPoint = recoveryEnabled === false;
-                while (foundResyncPoint === false && offset < orgLength) {
-                    offset++;
-                    for (j = 0; j < currModePatternsLength; j++) {
-                        const currConfig = patternIdxToConfig[j];
-                        const currPattern = currConfig.pattern;
-                        // manually in-lined because > 600 chars won't be in-lined in V8
-                        const singleCharCode = currConfig.short;
-                        if (singleCharCode !== false) {
-                            if (orgText.charCodeAt(offset) === singleCharCode) {
-                                // single character string
-                                foundResyncPoint = true;
-                            }
-                        }
-                        else if (currConfig.isCustom === true) {
-                            foundResyncPoint =
-                                currPattern.exec(orgText, offset, matchedTokens, groups) !== null;
-                        }
-                        else {
-                            currPattern.lastIndex = offset;
-                            foundResyncPoint = currPattern.exec(text) !== null;
-                        }
-                        if (foundResyncPoint === true) {
-                            break;
-                        }
-                    }
-                }
-                errLength = offset - errorStartOffset;
-                column = this.computeNewColumn(column, errLength);
-                // at this point we either re-synced or reached the end of the input text
-                msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(orgText, errorStartOffset, errLength, errorLine, errorColumn, last(modeStack));
-                errors.push({
-                    offset: errorStartOffset,
-                    line: errorLine,
-                    column: errorColumn,
-                    length: errLength,
-                    message: msg,
-                });
-                if (recoveryEnabled === false) {
-                    break;
-                }
-            }
-        }
-        // if we do have custom patterns which push directly into the
-        // TODO: custom tokens should not push directly??
-        if (!this.hasCustom) {
-            // if we guessed a too large size for the tokens array this will shrink it to the right size.
-            matchedTokens.length = matchedTokensIndex;
-        }
-        return {
-            tokens: matchedTokens,
-            groups: groups,
-            errors: errors,
-        };
-    }
-    handleModes(config, pop_mode, push_mode, newToken) {
-        if (config.pop === true) {
-            // need to save the PUSH_MODE property as if the mode is popped
-            // patternIdxToPopMode is updated to reflect the new mode after popping the stack
-            const pushMode = config.push;
-            pop_mode(newToken);
-            if (pushMode !== undefined) {
-                push_mode.call(this, pushMode);
-            }
-        }
-        else if (config.push !== undefined) {
-            push_mode.call(this, config.push);
-        }
-    }
-    // TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
-    updateTokenEndLineColumnLocation(newToken, group, lastLTIdx, numOfLTsInMatch, line, column, imageLength) {
-        let lastCharIsLT, fixForEndingInLT;
-        if (group !== undefined) {
-            // a none skipped multi line Token, need to update endLine/endColumn
-            lastCharIsLT = lastLTIdx === imageLength - 1;
-            fixForEndingInLT = lastCharIsLT ? -1 : 0;
-            if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
-                // if a token ends in a LT that last LT only affects the line numbering of following Tokens
-                newToken.endLine = line + fixForEndingInLT;
-                // the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
-                // inclusive to exclusive range.
-                newToken.endColumn = column - 1 + -fixForEndingInLT;
-            }
-            // else single LT in the last character of a token, no need to modify the endLine/EndColumn
-        }
-    }
-    computeNewColumn(oldColumn, imageLength) {
-        return oldColumn + imageLength;
-    }
-    createOffsetOnlyToken(image, startOffset, tokenTypeIdx, tokenType) {
-        return {
-            image,
-            startOffset,
-            tokenTypeIdx,
-            tokenType,
-        };
-    }
-    createStartOnlyToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn) {
-        return {
-            image,
-            startOffset,
-            startLine,
-            startColumn,
-            tokenTypeIdx,
-            tokenType,
-        };
-    }
-    createFullToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn, imageLength) {
-        return {
-            image,
-            startOffset,
-            endOffset: startOffset + imageLength - 1,
-            startLine,
-            endLine: startLine,
-            startColumn,
-            endColumn: startColumn + imageLength - 1,
-            tokenTypeIdx,
-            tokenType,
-        };
-    }
-    addTokenUsingPush(tokenVector, index, tokenToAdd) {
-        tokenVector.push(tokenToAdd);
-        return index;
-    }
-    addTokenUsingMemberAccess(tokenVector, index, tokenToAdd) {
-        tokenVector[index] = tokenToAdd;
-        index++;
-        return index;
-    }
-    handlePayloadNoCustom(token, payload) { }
-    handlePayloadWithCustom(token, payload) {
-        if (payload !== null) {
-            token.payload = payload;
-        }
-    }
-    match(pattern, text, offset) {
-        const found = pattern.test(text);
-        if (found === true) {
-            return text.substring(offset, pattern.lastIndex);
-        }
-        return null;
-    }
-    matchLength(pattern, text, offset) {
-        const found = pattern.test(text);
-        if (found === true) {
-            return pattern.lastIndex - offset;
-        }
-        return -1;
-    }
-}
-Lexer.SKIPPED = "This marks a skipped Token pattern, this means each token identified by it will " +
-    "be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.";
-Lexer.NA = /NOT_APPLICABLE/;
-
-export { Lexer, LexerDefinitionErrorType };