less 3.10.1 → 3.11.1

package/packages/css-parser/node_modules/chevrotain/lib/src/scan/lexer_public.js

@@ -0,0 +1,674 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var lexer_1 = require("./lexer");
+var utils_1 = require("../utils/utils");
+var tokens_1 = require("./tokens");
+var lexer_errors_public_1 = require("../scan/lexer_errors_public");
+var reg_exp_parser_1 = require("./reg_exp_parser");
+var LexerDefinitionErrorType;
+(function (LexerDefinitionErrorType) {
+    LexerDefinitionErrorType[LexerDefinitionErrorType["MISSING_PATTERN"] = 0] = "MISSING_PATTERN";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_PATTERN"] = 1] = "INVALID_PATTERN";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["EOI_ANCHOR_FOUND"] = 2] = "EOI_ANCHOR_FOUND";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["UNSUPPORTED_FLAGS_FOUND"] = 3] = "UNSUPPORTED_FLAGS_FOUND";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["DUPLICATE_PATTERNS_FOUND"] = 4] = "DUPLICATE_PATTERNS_FOUND";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_GROUP_TYPE_FOUND"] = 5] = "INVALID_GROUP_TYPE_FOUND";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["PUSH_MODE_DOES_NOT_EXIST"] = 6] = "PUSH_MODE_DOES_NOT_EXIST";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE"] = 7] = "MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY"] = 8] = "MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST"] = 9] = "MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED"] = 10] = "LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["SOI_ANCHOR_FOUND"] = 11] = "SOI_ANCHOR_FOUND";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["EMPTY_MATCH_PATTERN"] = 12] = "EMPTY_MATCH_PATTERN";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["NO_LINE_BREAKS_FLAGS"] = 13] = "NO_LINE_BREAKS_FLAGS";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["UNREACHABLE_PATTERN"] = 14] = "UNREACHABLE_PATTERN";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["IDENTIFY_TERMINATOR"] = 15] = "IDENTIFY_TERMINATOR";
+    LexerDefinitionErrorType[LexerDefinitionErrorType["CUSTOM_LINE_BREAK"] = 16] = "CUSTOM_LINE_BREAK";
+})(LexerDefinitionErrorType = exports.LexerDefinitionErrorType || (exports.LexerDefinitionErrorType = {}));
+var DEFAULT_LEXER_CONFIG = {
+    deferDefinitionErrorsHandling: false,
+    positionTracking: "full",
+    lineTerminatorsPattern: /\n|\r\n?/g,
+    lineTerminatorCharacters: ["\n", "\r"],
+    ensureOptimizations: false,
+    safeMode: false,
+    errorMessageProvider: lexer_errors_public_1.defaultLexerErrorProvider,
+    traceInitPerf: false,
+    skipValidations: false
+};
+Object.freeze(DEFAULT_LEXER_CONFIG);
+var Lexer = /** @class */ (function () {
+    function Lexer(lexerDefinition, config) {
+        var _this = this;
+        if (config === void 0) { config = DEFAULT_LEXER_CONFIG; }
+        this.lexerDefinition = lexerDefinition;
+        this.lexerDefinitionErrors = [];
+        this.lexerDefinitionWarning = [];
+        this.patternIdxToConfig = {};
+        this.charCodeToPatternIdxToConfig = {};
+        this.modes = [];
+        this.emptyGroups = {};
+        this.config = undefined;
+        this.trackStartLines = true;
+        this.trackEndLines = true;
+        this.hasCustom = false;
+        this.canModeBeOptimized = {};
+        if (typeof config === "boolean") {
+            throw Error("The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
+                "a boolean 2nd argument is no longer supported");
+        }
+        // todo: defaults func?
+        this.config = utils_1.merge(DEFAULT_LEXER_CONFIG, config);
+        var traceInitVal = this.config.traceInitPerf;
+        if (traceInitVal === true) {
+            this.traceInitMaxIdent = Infinity;
+            this.traceInitPerf = true;
+        }
+        else if (typeof traceInitVal === "number") {
+            this.traceInitMaxIdent = traceInitVal;
+            this.traceInitPerf = true;
+        }
+        this.traceInitIndent = -1;
+        this.TRACE_INIT("Lexer Constructor", function () {
+            var actualDefinition;
+            var hasOnlySingleMode = true;
+            _this.TRACE_INIT("Lexer Config handling", function () {
+                if (_this.config.lineTerminatorsPattern ===
+                    DEFAULT_LEXER_CONFIG.lineTerminatorsPattern) {
+                    // optimized built-in implementation for the defaults definition of lineTerminators
+                    _this.config.lineTerminatorsPattern = lexer_1.LineTerminatorOptimizedTester;
+                }
+                else {
+                    if (_this.config.lineTerminatorCharacters ===
+                        DEFAULT_LEXER_CONFIG.lineTerminatorCharacters) {
+                        throw Error("Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
+                            "\tFor details See: https://sap.github.io/chevrotain/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS");
+                    }
+                }
+                if (config.safeMode && config.ensureOptimizations) {
+                    throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');
+                }
+                _this.trackStartLines = /full|onlyStart/i.test(_this.config.positionTracking);
+                _this.trackEndLines = /full/i.test(_this.config.positionTracking);
+                // Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
+                if (utils_1.isArray(lexerDefinition)) {
+                    actualDefinition = { modes: {} };
+                    actualDefinition.modes[lexer_1.DEFAULT_MODE] = utils_1.cloneArr(lexerDefinition);
+                    actualDefinition[lexer_1.DEFAULT_MODE] = lexer_1.DEFAULT_MODE;
+                }
+                else {
+                    // no conversion needed, input should already be a IMultiModeLexerDefinition
+                    hasOnlySingleMode = false;
+                    actualDefinition = utils_1.cloneObj((lexerDefinition));
+                }
+            });
+            if (_this.config.skipValidations === false) {
+                _this.TRACE_INIT("performRuntimeChecks", function () {
+                    _this.lexerDefinitionErrors = _this.lexerDefinitionErrors.concat(lexer_1.performRuntimeChecks(actualDefinition, _this.trackStartLines, _this.config.lineTerminatorCharacters));
+                });
+                _this.TRACE_INIT("performWarningRuntimeChecks", function () {
+                    _this.lexerDefinitionWarning = _this.lexerDefinitionWarning.concat(lexer_1.performWarningRuntimeChecks(actualDefinition, _this.trackStartLines, _this.config.lineTerminatorCharacters));
+                });
+            }
+            // for extra robustness to avoid throwing an none informative error message
+            actualDefinition.modes = actualDefinition.modes
+                ? actualDefinition.modes
+                : {};
+            // an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
+            // this transformation is to increase robustness in the case of partially invalid lexer definition.
+            utils_1.forEach(actualDefinition.modes, function (currModeValue, currModeName) {
+                actualDefinition.modes[currModeName] = utils_1.reject(currModeValue, function (currTokType) { return utils_1.isUndefined(currTokType); });
+            });
+            var allModeNames = utils_1.keys(actualDefinition.modes);
+            utils_1.forEach(actualDefinition.modes, function (currModDef, currModName) {
+                _this.TRACE_INIT("Mode: <" + currModName + "> processing", function () {
+                    _this.modes.push(currModName);
+                    if (_this.config.skipValidations === false) {
+                        _this.TRACE_INIT("validatePatterns", function () {
+                            _this.lexerDefinitionErrors = _this.lexerDefinitionErrors.concat(lexer_1.validatePatterns(currModDef, allModeNames));
+                        });
+                    }
+                    // If definition errors were encountered, the analysis phase may fail unexpectedly/
+                    // Considering a lexer with definition errors may never be used, there is no point
+                    // to performing the analysis anyhow...
+                    if (utils_1.isEmpty(_this.lexerDefinitionErrors)) {
+                        tokens_1.augmentTokenTypes(currModDef);
+                        var currAnalyzeResult_1;
+                        _this.TRACE_INIT("analyzeTokenTypes", function () {
+                            currAnalyzeResult_1 = lexer_1.analyzeTokenTypes(currModDef, {
+                                lineTerminatorCharacters: _this.config
+                                    .lineTerminatorCharacters,
+                                positionTracking: config.positionTracking,
+                                ensureOptimizations: config.ensureOptimizations,
+                                safeMode: config.safeMode,
+                                tracer: _this.TRACE_INIT.bind(_this)
+                            });
+                        });
+                        _this.patternIdxToConfig[currModName] =
+                            currAnalyzeResult_1.patternIdxToConfig;
+                        _this.charCodeToPatternIdxToConfig[currModName] =
+                            currAnalyzeResult_1.charCodeToPatternIdxToConfig;
+                        _this.emptyGroups = utils_1.merge(_this.emptyGroups, currAnalyzeResult_1.emptyGroups);
+                        _this.hasCustom =
+                            currAnalyzeResult_1.hasCustom || _this.hasCustom;
+                        _this.canModeBeOptimized[currModName] =
+                            currAnalyzeResult_1.canBeOptimized;
+                    }
+                });
+            });
+            _this.defaultMode = actualDefinition.defaultMode;
+            if (!utils_1.isEmpty(_this.lexerDefinitionErrors) &&
+                !_this.config.deferDefinitionErrorsHandling) {
+                var allErrMessages = utils_1.map(_this.lexerDefinitionErrors, function (error) {
+                    return error.message;
+                });
+                var allErrMessagesString = allErrMessages.join("-----------------------\n");
+                throw new Error("Errors detected in definition of Lexer:\n" +
+                    allErrMessagesString);
+            }
+            // Only print warning if there are no errors, This will avoid pl
+            utils_1.forEach(_this.lexerDefinitionWarning, function (warningDescriptor) {
+                utils_1.PRINT_WARNING(warningDescriptor.message);
+            });
+            _this.TRACE_INIT("Choosing sub-methods implementations", function () {
+                // Choose the relevant internal implementations for this specific parser.
+                // These implementations should be in-lined by the JavaScript engine
+                // to provide optimal performance in each scenario.
+                if (lexer_1.SUPPORT_STICKY) {
+                    _this.chopInput = utils_1.IDENTITY;
+                    _this.match = _this.matchWithTest;
+                }
+                else {
+                    _this.updateLastIndex = utils_1.NOOP;
+                    _this.match = _this.matchWithExec;
+                }
+                if (hasOnlySingleMode) {
+                    _this.handleModes = utils_1.NOOP;
+                }
+                if (_this.trackStartLines === false) {
+                    _this.computeNewColumn = utils_1.IDENTITY;
+                }
+                if (_this.trackEndLines === false) {
+                    _this.updateTokenEndLineColumnLocation = utils_1.NOOP;
+                }
+                if (/full/i.test(_this.config.positionTracking)) {
+                    _this.createTokenInstance = _this.createFullToken;
+                }
+                else if (/onlyStart/i.test(_this.config.positionTracking)) {
+                    _this.createTokenInstance = _this.createStartOnlyToken;
+                }
+                else if (/onlyOffset/i.test(_this.config.positionTracking)) {
+                    _this.createTokenInstance = _this.createOffsetOnlyToken;
+                }
+                else {
+                    throw Error("Invalid <positionTracking> config option: \"" + _this.config.positionTracking + "\"");
+                }
+                if (_this.hasCustom) {
+                    _this.addToken = _this.addTokenUsingPush;
+                    _this.handlePayload = _this.handlePayloadWithCustom;
+                }
+                else {
+                    _this.addToken = _this.addTokenUsingMemberAccess;
+                    _this.handlePayload = _this.handlePayloadNoCustom;
+                }
+            });
+            _this.TRACE_INIT("Failed Optimization Warnings", function () {
+                var unOptimizedModes = utils_1.reduce(_this.canModeBeOptimized, function (cannotBeOptimized, canBeOptimized, modeName) {
+                    if (canBeOptimized === false) {
+                        cannotBeOptimized.push(modeName);
+                    }
+                    return cannotBeOptimized;
+                }, []);
+                if (config.ensureOptimizations && !utils_1.isEmpty(unOptimizedModes)) {
+                    throw Error("Lexer Modes: < " + unOptimizedModes.join(", ") + " > cannot be optimized.\n" +
+                        '\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
+                        "\t Or inspect the console log for details on how to resolve these issues.");
+                }
+            });
+            _this.TRACE_INIT("clearRegExpParserCache", function () {
+                reg_exp_parser_1.clearRegExpParserCache();
+            });
+            _this.TRACE_INIT("toFastProperties", function () {
+                utils_1.toFastProperties(_this);
+            });
+        });
+    }
+    Lexer.prototype.tokenize = function (text, initialMode) {
+        if (initialMode === void 0) { initialMode = this.defaultMode; }
+        if (!utils_1.isEmpty(this.lexerDefinitionErrors)) {
+            var allErrMessages = utils_1.map(this.lexerDefinitionErrors, function (error) {
+                return error.message;
+            });
+            var allErrMessagesString = allErrMessages.join("-----------------------\n");
+            throw new Error("Unable to Tokenize because Errors detected in definition of Lexer:\n" +
+                allErrMessagesString);
+        }
+        var lexResult = this.tokenizeInternal(text, initialMode);
+        return lexResult;
+    };
+    // There is quite a bit of duplication between this and "tokenizeInternalLazy"
+    // This is intentional due to performance considerations.
+    Lexer.prototype.tokenizeInternal = function (text, initialMode) {
+        var _this = this;
+        var i, j, matchAltImage, longerAltIdx, matchedImage, payload, altPayload, imageLength, group, tokType, newToken, errLength, droppedChar, msg, match;
+        var orgText = text;
+        var orgLength = orgText.length;
+        var offset = 0;
+        var matchedTokensIndex = 0;
+        // initializing the tokensArray to the "guessed" size.
+        // guessing too little will still reduce the number of array re-sizes on pushes.
+        // guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
+        // but would still have a faster runtime by avoiding (All but one) array resizing.
+        var guessedNumberOfTokens = this.hasCustom
+            ? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
+            : Math.floor(text.length / 10);
+        var matchedTokens = new Array(guessedNumberOfTokens);
+        var errors = [];
+        var line = this.trackStartLines ? 1 : undefined;
+        var column = this.trackStartLines ? 1 : undefined;
+        var groups = lexer_1.cloneEmptyGroups(this.emptyGroups);
+        var trackLines = this.trackStartLines;
+        var lineTerminatorPattern = this.config.lineTerminatorsPattern;
+        var currModePatternsLength = 0;
+        var patternIdxToConfig = [];
+        var currCharCodeToPatternIdxToConfig = [];
+        var modeStack = [];
+        var emptyArray = [];
+        Object.freeze(emptyArray);
+        var getPossiblePatterns = undefined;
+        function getPossiblePatternsSlow() {
+            return patternIdxToConfig;
+        }
+        function getPossiblePatternsOptimized(charCode) {
+            var optimizedCharIdx = lexer_1.charCodeToOptimizedIndex(charCode);
+            var possiblePatterns = currCharCodeToPatternIdxToConfig[optimizedCharIdx];
+            if (possiblePatterns === undefined) {
+                return emptyArray;
+            }
+            else {
+                return possiblePatterns;
+            }
+        }
+        var pop_mode = function (popToken) {
+            // TODO: perhaps avoid this error in the edge case there is no more input?
+            if (modeStack.length === 1 &&
+                // if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
+                // So no error should occur.
+                popToken.tokenType.PUSH_MODE === undefined) {
+                // if we try to pop the last mode there lexer will no longer have ANY mode.
+                // thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
+                var msg_1 = _this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(popToken);
+                errors.push({
+                    offset: popToken.startOffset,
+                    line: popToken.startLine !== undefined
+                        ? popToken.startLine
+                        : undefined,
+                    column: popToken.startColumn !== undefined
+                        ? popToken.startColumn
+                        : undefined,
+                    length: popToken.image.length,
+                    message: msg_1
+                });
+            }
+            else {
+                modeStack.pop();
+                var newMode = utils_1.last(modeStack);
+                patternIdxToConfig = _this.patternIdxToConfig[newMode];
+                currCharCodeToPatternIdxToConfig = _this
+                    .charCodeToPatternIdxToConfig[newMode];
+                currModePatternsLength = patternIdxToConfig.length;
+                var modeCanBeOptimized = _this.canModeBeOptimized[newMode] &&
+                    _this.config.safeMode === false;
+                if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
+                    getPossiblePatterns = getPossiblePatternsOptimized;
+                }
+                else {
+                    getPossiblePatterns = getPossiblePatternsSlow;
+                }
+            }
+        };
+        function push_mode(newMode) {
+            modeStack.push(newMode);
+            currCharCodeToPatternIdxToConfig = this
+                .charCodeToPatternIdxToConfig[newMode];
+            patternIdxToConfig = this.patternIdxToConfig[newMode];
+            currModePatternsLength = patternIdxToConfig.length;
+            currModePatternsLength = patternIdxToConfig.length;
+            var modeCanBeOptimized = this.canModeBeOptimized[newMode] &&
+                this.config.safeMode === false;
+            if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
+                getPossiblePatterns = getPossiblePatternsOptimized;
+            }
+            else {
+                getPossiblePatterns = getPossiblePatternsSlow;
+            }
+        }
+        // this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
+        // seem to matter performance wise.
+        push_mode.call(this, initialMode);
+        var currConfig;
+        while (offset < orgLength) {
+            matchedImage = null;
+            var nextCharCode = orgText.charCodeAt(offset);
+            var chosenPatternIdxToConfig = getPossiblePatterns(nextCharCode);
+            var chosenPatternsLength = chosenPatternIdxToConfig.length;
+            for (i = 0; i < chosenPatternsLength; i++) {
+                currConfig = chosenPatternIdxToConfig[i];
+                var currPattern = currConfig.pattern;
+                payload = null;
+                // manually in-lined because > 600 chars won't be in-lined in V8
+                var singleCharCode = currConfig.short;
+                if (singleCharCode !== false) {
+                    if (nextCharCode === singleCharCode) {
+                        // single character string
+                        matchedImage = currPattern;
+                    }
+                }
+                else if (currConfig.isCustom === true) {
+                    match = currPattern.exec(orgText, offset, matchedTokens, groups);
+                    if (match !== null) {
+                        matchedImage = match[0];
+                        if (match.payload !== undefined) {
+                            payload = match.payload;
+                        }
+                    }
+                    else {
+                        matchedImage = null;
+                    }
+                }
+                else {
+                    this.updateLastIndex(currPattern, offset);
+                    matchedImage = this.match(currPattern, text, offset);
+                }
+                if (matchedImage !== null) {
+                    // even though this pattern matched we must try a another longer alternative.
+                    // this can be used to prioritize keywords over identifiers
+                    longerAltIdx = currConfig.longerAlt;
+                    if (longerAltIdx !== undefined) {
+                        // TODO: micro optimize, avoid extra prop access
+                        // by saving/linking longerAlt on the original config?
+                        var longerAltConfig = patternIdxToConfig[longerAltIdx];
+                        var longerAltPattern = longerAltConfig.pattern;
+                        altPayload = null;
+                        // single Char can never be a longer alt so no need to test it.
+                        // manually in-lined because > 600 chars won't be in-lined in V8
+                        if (longerAltConfig.isCustom === true) {
+                            match = longerAltPattern.exec(orgText, offset, matchedTokens, groups);
+                            if (match !== null) {
+                                matchAltImage = match[0];
+                                if (match.payload !== undefined) {
+                                    altPayload = match.payload;
+                                }
+                            }
+                            else {
+                                matchAltImage = null;
+                            }
+                        }
+                        else {
+                            this.updateLastIndex(longerAltPattern, offset);
+                            matchAltImage = this.match(longerAltPattern, text, offset);
+                        }
+                        if (matchAltImage &&
+                            matchAltImage.length > matchedImage.length) {
+                            matchedImage = matchAltImage;
+                            payload = altPayload;
+                            currConfig = longerAltConfig;
+                        }
+                    }
+                    break;
+                }
+            }
+            // successful match
+            if (matchedImage !== null) {
+                imageLength = matchedImage.length;
+                group = currConfig.group;
+                if (group !== undefined) {
+                    tokType = currConfig.tokenTypeIdx;
+                    // TODO: "offset + imageLength" and the new column may be computed twice in case of "full" location information inside
+                    // createFullToken method
+                    newToken = this.createTokenInstance(matchedImage, offset, tokType, currConfig.tokenType, line, column, imageLength);
+                    this.handlePayload(newToken, payload);
+                    // TODO: optimize NOOP in case there are no special groups?
+                    if (group === false) {
+                        matchedTokensIndex = this.addToken(matchedTokens, matchedTokensIndex, newToken);
+                    }
+                    else {
+                        groups[group].push(newToken);
+                    }
+                }
+                text = this.chopInput(text, imageLength);
+                offset = offset + imageLength;
+                // TODO: with newlines the column may be assigned twice
+                column = this.computeNewColumn(column, imageLength);
+                if (trackLines === true &&
+                    currConfig.canLineTerminator === true) {
+                    var numOfLTsInMatch = 0;
+                    var foundTerminator = void 0;
+                    var lastLTEndOffset = void 0;
+                    lineTerminatorPattern.lastIndex = 0;
+                    do {
+                        foundTerminator = lineTerminatorPattern.test(matchedImage);
+                        if (foundTerminator === true) {
+                            lastLTEndOffset =
+                                lineTerminatorPattern.lastIndex - 1;
+                            numOfLTsInMatch++;
+                        }
+                    } while (foundTerminator === true);
+                    if (numOfLTsInMatch !== 0) {
+                        line = line + numOfLTsInMatch;
+                        column = imageLength - lastLTEndOffset;
+                        this.updateTokenEndLineColumnLocation(newToken, group, lastLTEndOffset, numOfLTsInMatch, line, column, imageLength);
+                    }
+                }
+                // will be NOOP if no modes present
+                this.handleModes(currConfig, pop_mode, push_mode, newToken);
+            }
+            else {
+                // error recovery, drop characters until we identify a valid token's start point
+                var errorStartOffset = offset;
+                var errorLine = line;
+                var errorColumn = column;
+                var foundResyncPoint = false;
+                while (!foundResyncPoint && offset < orgLength) {
+                    // drop chars until we succeed in matching something
+                    droppedChar = orgText.charCodeAt(offset);
+                    // Identity Func (when sticky flag is enabled)
+                    text = this.chopInput(text, 1);
+                    offset++;
+                    for (j = 0; j < currModePatternsLength; j++) {
+                        var currConfig_1 = patternIdxToConfig[j];
+                        var currPattern = currConfig_1.pattern;
+                        // manually in-lined because > 600 chars won't be in-lined in V8
+                        var singleCharCode = currConfig_1.short;
+                        if (singleCharCode !== false) {
+                            if (orgText.charCodeAt(offset) === singleCharCode) {
+                                // single character string
+                                foundResyncPoint = true;
+                            }
+                        }
+                        else if (currConfig_1.isCustom === true) {
+                            foundResyncPoint =
+                                currPattern.exec(orgText, offset, matchedTokens, groups) !== null;
+                        }
+                        else {
+                            this.updateLastIndex(currPattern, offset);
+                            foundResyncPoint = currPattern.exec(text) !== null;
+                        }
+                        if (foundResyncPoint === true) {
+                            break;
+                        }
+                    }
+                }
+                errLength = offset - errorStartOffset;
+                // at this point we either re-synced or reached the end of the input text
+                msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(orgText, errorStartOffset, errLength, errorLine, errorColumn);
+                errors.push({
+                    offset: errorStartOffset,
+                    line: errorLine,
+                    column: errorColumn,
+                    length: errLength,
+                    message: msg
+                });
+            }
+        }
+        // if we do have custom patterns which push directly into the
+        // TODO: custom tokens should not push directly??
+        if (!this.hasCustom) {
+            // if we guessed a too large size for the tokens array this will shrink it to the right size.
+            matchedTokens.length = matchedTokensIndex;
+        }
+        return {
+            tokens: matchedTokens,
+            groups: groups,
+            errors: errors
+        };
+    };
+    Lexer.prototype.handleModes = function (config, pop_mode, push_mode, newToken) {
+        if (config.pop === true) {
+            // need to save the PUSH_MODE property as if the mode is popped
+            // patternIdxToPopMode is updated to reflect the new mode after popping the stack
+            var pushMode = config.push;
+            pop_mode(newToken);
+            if (pushMode !== undefined) {
+                push_mode.call(this, pushMode);
+            }
+        }
+        else if (config.push !== undefined) {
+            push_mode.call(this, config.push);
+        }
+    };
+    Lexer.prototype.chopInput = function (text, length) {
+        return text.substring(length);
+    };
+    Lexer.prototype.updateLastIndex = function (regExp, newLastIndex) {
+        regExp.lastIndex = newLastIndex;
+    };
+    // TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
+    Lexer.prototype.updateTokenEndLineColumnLocation = function (newToken, group, lastLTIdx, numOfLTsInMatch, line, column, imageLength) {
+        var lastCharIsLT, fixForEndingInLT;
+        if (group !== undefined) {
+            // a none skipped multi line Token, need to update endLine/endColumn
+            lastCharIsLT = lastLTIdx === imageLength - 1;
+            fixForEndingInLT = lastCharIsLT ? -1 : 0;
+            if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
+                // if a token ends in a LT that last LT only affects the line numbering of following Tokens
+                newToken.endLine = line + fixForEndingInLT;
+                // the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
+                // inclusive to exclusive range.
+                newToken.endColumn = column - 1 + -fixForEndingInLT;
+            }
+            // else single LT in the last character of a token, no need to modify the endLine/EndColumn
+        }
+    };
+    Lexer.prototype.computeNewColumn = function (oldColumn, imageLength) {
+        return oldColumn + imageLength;
+    };
+    // Place holder, will be replaced by the correct variant according to the locationTracking option at runtime.
+    /* istanbul ignore next - place holder */
+    Lexer.prototype.createTokenInstance = function () {
+        var args = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            args[_i] = arguments[_i];
+        }
+        return null;
+    };
+    Lexer.prototype.createOffsetOnlyToken = function (image, startOffset, tokenTypeIdx, tokenType) {
+        return {
+            image: image,
+            startOffset: startOffset,
+            tokenTypeIdx: tokenTypeIdx,
+            tokenType: tokenType
+        };
+    };
+    Lexer.prototype.createStartOnlyToken = function (image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn) {
+        return {
+            image: image,
+            startOffset: startOffset,
+            startLine: startLine,
+            startColumn: startColumn,
+            tokenTypeIdx: tokenTypeIdx,
+            tokenType: tokenType
+        };
+    };
+    Lexer.prototype.createFullToken = function (image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn, imageLength) {
+        return {
+            image: image,
+            startOffset: startOffset,
+            endOffset: startOffset + imageLength - 1,
+            startLine: startLine,
+            endLine: startLine,
+            startColumn: startColumn,
+            endColumn: startColumn + imageLength - 1,
+            tokenTypeIdx: tokenTypeIdx,
+            tokenType: tokenType
+        };
+    };
+    // Place holder, will be replaced by the correct variant according to the locationTracking option at runtime.
+    /* istanbul ignore next - place holder */
+    Lexer.prototype.addToken = function (tokenVector, index, tokenToAdd) {
+        return 666;
+    };
+    Lexer.prototype.addTokenUsingPush = function (tokenVector, index, tokenToAdd) {
+        tokenVector.push(tokenToAdd);
+        return index;
+    };
+    Lexer.prototype.addTokenUsingMemberAccess = function (tokenVector, index, tokenToAdd) {
+        tokenVector[index] = tokenToAdd;
+        index++;
+        return index;
+    };
+    // Place holder, will be replaced by the correct variant according to the hasCustom flag option at runtime.
+    /* istanbul ignore next - place holder */
+    Lexer.prototype.handlePayload = function (token, payload) { };
+    Lexer.prototype.handlePayloadNoCustom = function (token, payload) { };
+    Lexer.prototype.handlePayloadWithCustom = function (token, payload) {
+        if (payload !== null) {
+            token.payload = payload;
+        }
+    };
+    /* istanbul ignore next - place holder to be replaced with chosen alternative at runtime */
+    Lexer.prototype.match = function (pattern, text, offset) {
+        return null;
+    };
+    Lexer.prototype.matchWithTest = function (pattern, text, offset) {
+        var found = pattern.test(text);
+        if (found === true) {
+            return text.substring(offset, pattern.lastIndex);
+        }
+        return null;
+    };
+    Lexer.prototype.matchWithExec = function (pattern, text) {
+        var regExpArray = pattern.exec(text);
+        return regExpArray !== null ? regExpArray[0] : regExpArray;
+    };
+    // Duplicated from the parser's perf trace trait to allow future extraction
+    // of the lexer to a separate package.
+    Lexer.prototype.TRACE_INIT = function (phaseDesc, phaseImpl) {
+        // No need to optimize this using NOOP pattern because
+        // It is not called in a hot spot...
+        if (this.traceInitPerf === true) {
+            this.traceInitIndent++;
+            var indent = new Array(this.traceInitIndent + 1).join("\t");
+            if (this.traceInitIndent < this.traceInitMaxIdent) {
+                console.log(indent + "--> <" + phaseDesc + ">");
+            }
+            var _a = utils_1.timer(phaseImpl), time = _a.time, value = _a.value;
+            /* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
+            var traceMethod = time > 10 ? console.warn : console.log;
+            if (this.traceInitIndent < this.traceInitMaxIdent) {
+                traceMethod(indent + "<-- <" + phaseDesc + "> time: " + time + "ms");
+            }
+            this.traceInitIndent--;
+            return value;
+        }
+        else {
+            return phaseImpl();
+        }
+    };
+    Lexer.SKIPPED = "This marks a skipped Token pattern, this means each token identified by it will" +
+        "be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.";
+    Lexer.NA = /NOT_APPLICABLE/;
+    return Lexer;
+}());
+exports.Lexer = Lexer;
+//# sourceMappingURL=lexer_public.js.map