@elmethis/core 1.0.0-alpha.149 → 1.0.0-alpha.150

package/dist/node_modules/.pnpm/chevrotain@11.0.3/node_modules/chevrotain/lib/src/scan/lexer.mjs

@@ -1,720 +0,0 @@
-import "../../../../../../@chevrotain_regexp-to-ast@11.0.3/node_modules/@chevrotain/regexp-to-ast/lib/src/character-classes.mjs";
-import { BaseRegExpVisitor } from "../../../../../../@chevrotain_regexp-to-ast@11.0.3/node_modules/@chevrotain/regexp-to-ast/lib/src/base-regexp-visitor.mjs";
-import { LexerDefinitionErrorType, Lexer } from "./lexer_public.mjs";
-import { has, forEach, isUndefined, isArray, includes, compact, flatten, values, reject, defaults, map, isRegExp, isFunction, isString, indexOf, reduce, isEmpty, keys, filter, difference, first, find } from "lodash-es";
-import { PRINT_ERROR } from "../../../../../../@chevrotain_utils@11.0.3/node_modules/@chevrotain/utils/lib/src/print.mjs";
-import { canMatchCharCode, failedOptimizationPrefixMsg, getOptimizedStartCodesIndices } from "./reg_exp.mjs";
-import { getRegExpAst } from "./reg_exp_parser.mjs";
-const PATTERN = "PATTERN";
-const DEFAULT_MODE = "defaultMode";
-const MODES = "modes";
-let SUPPORT_STICKY = typeof new RegExp("(?:)").sticky === "boolean";
-function analyzeTokenTypes(tokenTypes, options) {
-  options = defaults(options, {
-    useSticky: SUPPORT_STICKY,
-    debug: false,
-    safeMode: false,
-    positionTracking: "full",
-    lineTerminatorCharacters: ["\r", "\n"],
-    tracer: (msg, action) => action()
-  });
-  const tracer = options.tracer;
-  tracer("initCharCodeToOptimizedIndexMap", () => {
-    initCharCodeToOptimizedIndexMap();
-  });
-  let onlyRelevantTypes;
-  tracer("Reject Lexer.NA", () => {
-    onlyRelevantTypes = reject(tokenTypes, (currType) => {
-      return currType[PATTERN] === Lexer.NA;
-    });
-  });
-  let hasCustom = false;
-  let allTransformedPatterns;
-  tracer("Transform Patterns", () => {
-    hasCustom = false;
-    allTransformedPatterns = map(onlyRelevantTypes, (currType) => {
-      const currPattern = currType[PATTERN];
-      if (isRegExp(currPattern)) {
-        const regExpSource = currPattern.source;
-        if (regExpSource.length === 1 && // only these regExp meta characters which can appear in a length one regExp
-        regExpSource !== "^" && regExpSource !== "$" && regExpSource !== "." && !currPattern.ignoreCase) {
-          return regExpSource;
-        } else if (regExpSource.length === 2 && regExpSource[0] === "\\" && // not a meta character
-        !includes([
-          "d",
-          "D",
-          "s",
-          "S",
-          "t",
-          "r",
-          "n",
-          "t",
-          "0",
-          "c",
-          "b",
-          "B",
-          "f",
-          "v",
-          "w",
-          "W"
-        ], regExpSource[1])) {
-          return regExpSource[1];
-        } else {
-          return options.useSticky ? addStickyFlag(currPattern) : addStartOfInput(currPattern);
-        }
-      } else if (isFunction(currPattern)) {
-        hasCustom = true;
-        return { exec: currPattern };
-      } else if (typeof currPattern === "object") {
-        hasCustom = true;
-        return currPattern;
-      } else if (typeof currPattern === "string") {
-        if (currPattern.length === 1) {
-          return currPattern;
-        } else {
-          const escapedRegExpString = currPattern.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
-          const wrappedRegExp = new RegExp(escapedRegExpString);
-          return options.useSticky ? addStickyFlag(wrappedRegExp) : addStartOfInput(wrappedRegExp);
-        }
-      } else {
-        throw Error("non exhaustive match");
-      }
-    });
-  });
-  let patternIdxToType;
-  let patternIdxToGroup;
-  let patternIdxToLongerAltIdxArr;
-  let patternIdxToPushMode;
-  let patternIdxToPopMode;
-  tracer("misc mapping", () => {
-    patternIdxToType = map(onlyRelevantTypes, (currType) => currType.tokenTypeIdx);
-    patternIdxToGroup = map(onlyRelevantTypes, (clazz) => {
-      const groupName = clazz.GROUP;
-      if (groupName === Lexer.SKIPPED) {
-        return void 0;
-      } else if (isString(groupName)) {
-        return groupName;
-      } else if (isUndefined(groupName)) {
-        return false;
-      } else {
-        throw Error("non exhaustive match");
-      }
-    });
-    patternIdxToLongerAltIdxArr = map(onlyRelevantTypes, (clazz) => {
-      const longerAltType = clazz.LONGER_ALT;
-      if (longerAltType) {
-        const longerAltIdxArr = isArray(longerAltType) ? map(longerAltType, (type) => indexOf(onlyRelevantTypes, type)) : [indexOf(onlyRelevantTypes, longerAltType)];
-        return longerAltIdxArr;
-      }
-    });
-    patternIdxToPushMode = map(onlyRelevantTypes, (clazz) => clazz.PUSH_MODE);
-    patternIdxToPopMode = map(onlyRelevantTypes, (clazz) => has(clazz, "POP_MODE"));
-  });
-  let patternIdxToCanLineTerminator;
-  tracer("Line Terminator Handling", () => {
-    const lineTerminatorCharCodes = getCharCodes(options.lineTerminatorCharacters);
-    patternIdxToCanLineTerminator = map(onlyRelevantTypes, (tokType) => false);
-    if (options.positionTracking !== "onlyOffset") {
-      patternIdxToCanLineTerminator = map(onlyRelevantTypes, (tokType) => {
-        if (has(tokType, "LINE_BREAKS")) {
-          return !!tokType.LINE_BREAKS;
-        } else {
-          return checkLineBreaksIssues(tokType, lineTerminatorCharCodes) === false && canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN);
-        }
-      });
-    }
-  });
-  let patternIdxToIsCustom;
-  let patternIdxToShort;
-  let emptyGroups;
-  let patternIdxToConfig;
-  tracer("Misc Mapping #2", () => {
-    patternIdxToIsCustom = map(onlyRelevantTypes, isCustomPattern);
-    patternIdxToShort = map(allTransformedPatterns, isShortPattern);
-    emptyGroups = reduce(onlyRelevantTypes, (acc, clazz) => {
-      const groupName = clazz.GROUP;
-      if (isString(groupName) && !(groupName === Lexer.SKIPPED)) {
-        acc[groupName] = [];
-      }
-      return acc;
-    }, {});
-    patternIdxToConfig = map(allTransformedPatterns, (x, idx) => {
-      return {
-        pattern: allTransformedPatterns[idx],
-        longerAlt: patternIdxToLongerAltIdxArr[idx],
-        canLineTerminator: patternIdxToCanLineTerminator[idx],
-        isCustom: patternIdxToIsCustom[idx],
-        short: patternIdxToShort[idx],
-        group: patternIdxToGroup[idx],
-        push: patternIdxToPushMode[idx],
-        pop: patternIdxToPopMode[idx],
-        tokenTypeIdx: patternIdxToType[idx],
-        tokenType: onlyRelevantTypes[idx]
-      };
-    });
-  });
-  let canBeOptimized = true;
-  let charCodeToPatternIdxToConfig = [];
-  if (!options.safeMode) {
-    tracer("First Char Optimization", () => {
-      charCodeToPatternIdxToConfig = reduce(onlyRelevantTypes, (result, currTokType, idx) => {
-        if (typeof currTokType.PATTERN === "string") {
-          const charCode = currTokType.PATTERN.charCodeAt(0);
-          const optimizedIdx = charCodeToOptimizedIndex(charCode);
-          addToMapOfArrays(result, optimizedIdx, patternIdxToConfig[idx]);
-        } else if (isArray(currTokType.START_CHARS_HINT)) {
-          let lastOptimizedIdx;
-          forEach(currTokType.START_CHARS_HINT, (charOrInt) => {
-            const charCode = typeof charOrInt === "string" ? charOrInt.charCodeAt(0) : charOrInt;
-            const currOptimizedIdx = charCodeToOptimizedIndex(charCode);
-            if (lastOptimizedIdx !== currOptimizedIdx) {
-              lastOptimizedIdx = currOptimizedIdx;
-              addToMapOfArrays(result, currOptimizedIdx, patternIdxToConfig[idx]);
-            }
-          });
-        } else if (isRegExp(currTokType.PATTERN)) {
-          if (currTokType.PATTERN.unicode) {
-            canBeOptimized = false;
-            if (options.ensureOptimizations) {
-              PRINT_ERROR(`${failedOptimizationPrefixMsg}	Unable to analyze < ${currTokType.PATTERN.toString()} > pattern.
-	The regexp unicode flag is not currently supported by the regexp-to-ast library.
-	This will disable the lexer's first char optimizations.
-	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNICODE_OPTIMIZE`);
-            }
-          } else {
-            const optimizedCodes = getOptimizedStartCodesIndices(currTokType.PATTERN, options.ensureOptimizations);
-            if (isEmpty(optimizedCodes)) {
-              canBeOptimized = false;
-            }
-            forEach(optimizedCodes, (code) => {
-              addToMapOfArrays(result, code, patternIdxToConfig[idx]);
-            });
-          }
-        } else {
-          if (options.ensureOptimizations) {
-            PRINT_ERROR(`${failedOptimizationPrefixMsg}	TokenType: <${currTokType.name}> is using a custom token pattern without providing <start_chars_hint> parameter.
-	This will disable the lexer's first char optimizations.
-	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_OPTIMIZE`);
-          }
-          canBeOptimized = false;
-        }
-        return result;
-      }, []);
-    });
-  }
-  return {
-    emptyGroups,
-    patternIdxToConfig,
-    charCodeToPatternIdxToConfig,
-    hasCustom,
-    canBeOptimized
-  };
-}
-function validatePatterns(tokenTypes, validModesNames) {
-  let errors = [];
-  const missingResult = findMissingPatterns(tokenTypes);
-  errors = errors.concat(missingResult.errors);
-  const invalidResult = findInvalidPatterns(missingResult.valid);
-  const validTokenTypes = invalidResult.valid;
-  errors = errors.concat(invalidResult.errors);
-  errors = errors.concat(validateRegExpPattern(validTokenTypes));
-  errors = errors.concat(findInvalidGroupType(validTokenTypes));
-  errors = errors.concat(findModesThatDoNotExist(validTokenTypes, validModesNames));
-  errors = errors.concat(findUnreachablePatterns(validTokenTypes));
-  return errors;
-}
-function validateRegExpPattern(tokenTypes) {
-  let errors = [];
-  const withRegExpPatterns = filter(tokenTypes, (currTokType) => isRegExp(currTokType[PATTERN]));
-  errors = errors.concat(findEndOfInputAnchor(withRegExpPatterns));
-  errors = errors.concat(findStartOfInputAnchor(withRegExpPatterns));
-  errors = errors.concat(findUnsupportedFlags(withRegExpPatterns));
-  errors = errors.concat(findDuplicatePatterns(withRegExpPatterns));
-  errors = errors.concat(findEmptyMatchRegExps(withRegExpPatterns));
-  return errors;
-}
-function findMissingPatterns(tokenTypes) {
-  const tokenTypesWithMissingPattern = filter(tokenTypes, (currType) => {
-    return !has(currType, PATTERN);
-  });
-  const errors = map(tokenTypesWithMissingPattern, (currType) => {
-    return {
-      message: "Token Type: ->" + currType.name + "<- missing static 'PATTERN' property",
-      type: LexerDefinitionErrorType.MISSING_PATTERN,
-      tokenTypes: [currType]
-    };
-  });
-  const valid = difference(tokenTypes, tokenTypesWithMissingPattern);
-  return { errors, valid };
-}
-function findInvalidPatterns(tokenTypes) {
-  const tokenTypesWithInvalidPattern = filter(tokenTypes, (currType) => {
-    const pattern = currType[PATTERN];
-    return !isRegExp(pattern) && !isFunction(pattern) && !has(pattern, "exec") && !isString(pattern);
-  });
-  const errors = map(tokenTypesWithInvalidPattern, (currType) => {
-    return {
-      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' can only be a RegExp, a Function matching the {CustomPatternMatcherFunc} type or an Object matching the {ICustomPattern} interface.",
-      type: LexerDefinitionErrorType.INVALID_PATTERN,
-      tokenTypes: [currType]
-    };
-  });
-  const valid = difference(tokenTypes, tokenTypesWithInvalidPattern);
-  return { errors, valid };
-}
-const end_of_input = /[^\\][$]/;
-function findEndOfInputAnchor(tokenTypes) {
-  class EndAnchorFinder extends BaseRegExpVisitor {
-    constructor() {
-      super(...arguments);
-      this.found = false;
-    }
-    visitEndAnchor(node) {
-      this.found = true;
-    }
-  }
-  const invalidRegex = filter(tokenTypes, (currType) => {
-    const pattern = currType.PATTERN;
-    try {
-      const regexpAst = getRegExpAst(pattern);
-      const endAnchorVisitor = new EndAnchorFinder();
-      endAnchorVisitor.visit(regexpAst);
-      return endAnchorVisitor.found;
-    } catch (e) {
-      return end_of_input.test(pattern.source);
-    }
-  });
-  const errors = map(invalidRegex, (currType) => {
-    return {
-      message: "Unexpected RegExp Anchor Error:\n	Token Type: ->" + currType.name + "<- static 'PATTERN' cannot contain end of input anchor '$'\n	See chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS	for details.",
-      type: LexerDefinitionErrorType.EOI_ANCHOR_FOUND,
-      tokenTypes: [currType]
-    };
-  });
-  return errors;
-}
-function findEmptyMatchRegExps(tokenTypes) {
-  const matchesEmptyString = filter(tokenTypes, (currType) => {
-    const pattern = currType.PATTERN;
-    return pattern.test("");
-  });
-  const errors = map(matchesEmptyString, (currType) => {
-    return {
-      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' must not match an empty string",
-      type: LexerDefinitionErrorType.EMPTY_MATCH_PATTERN,
-      tokenTypes: [currType]
-    };
-  });
-  return errors;
-}
-const start_of_input = /[^\\[][\^]|^\^/;
-function findStartOfInputAnchor(tokenTypes) {
-  class StartAnchorFinder extends BaseRegExpVisitor {
-    constructor() {
-      super(...arguments);
-      this.found = false;
-    }
-    visitStartAnchor(node) {
-      this.found = true;
-    }
-  }
-  const invalidRegex = filter(tokenTypes, (currType) => {
-    const pattern = currType.PATTERN;
-    try {
-      const regexpAst = getRegExpAst(pattern);
-      const startAnchorVisitor = new StartAnchorFinder();
-      startAnchorVisitor.visit(regexpAst);
-      return startAnchorVisitor.found;
-    } catch (e) {
-      return start_of_input.test(pattern.source);
-    }
-  });
-  const errors = map(invalidRegex, (currType) => {
-    return {
-      message: "Unexpected RegExp Anchor Error:\n	Token Type: ->" + currType.name + "<- static 'PATTERN' cannot contain start of input anchor '^'\n	See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS	for details.",
-      type: LexerDefinitionErrorType.SOI_ANCHOR_FOUND,
-      tokenTypes: [currType]
-    };
-  });
-  return errors;
-}
-function findUnsupportedFlags(tokenTypes) {
-  const invalidFlags = filter(tokenTypes, (currType) => {
-    const pattern = currType[PATTERN];
-    return pattern instanceof RegExp && (pattern.multiline || pattern.global);
-  });
-  const errors = map(invalidFlags, (currType) => {
-    return {
-      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' may NOT contain global('g') or multiline('m')",
-      type: LexerDefinitionErrorType.UNSUPPORTED_FLAGS_FOUND,
-      tokenTypes: [currType]
-    };
-  });
-  return errors;
-}
-function findDuplicatePatterns(tokenTypes) {
-  const found = [];
-  let identicalPatterns = map(tokenTypes, (outerType) => {
-    return reduce(tokenTypes, (result, innerType) => {
-      if (outerType.PATTERN.source === innerType.PATTERN.source && !includes(found, innerType) && innerType.PATTERN !== Lexer.NA) {
-        found.push(innerType);
-        result.push(innerType);
-        return result;
-      }
-      return result;
-    }, []);
-  });
-  identicalPatterns = compact(identicalPatterns);
-  const duplicatePatterns = filter(identicalPatterns, (currIdenticalSet) => {
-    return currIdenticalSet.length > 1;
-  });
-  const errors = map(duplicatePatterns, (setOfIdentical) => {
-    const tokenTypeNames = map(setOfIdentical, (currType) => {
-      return currType.name;
-    });
-    const dupPatternSrc = first(setOfIdentical).PATTERN;
-    return {
-      message: `The same RegExp pattern ->${dupPatternSrc}<-has been used in all of the following Token Types: ${tokenTypeNames.join(", ")} <-`,
-      type: LexerDefinitionErrorType.DUPLICATE_PATTERNS_FOUND,
-      tokenTypes: setOfIdentical
-    };
-  });
-  return errors;
-}
-function findInvalidGroupType(tokenTypes) {
-  const invalidTypes = filter(tokenTypes, (clazz) => {
-    if (!has(clazz, "GROUP")) {
-      return false;
-    }
-    const group = clazz.GROUP;
-    return group !== Lexer.SKIPPED && group !== Lexer.NA && !isString(group);
-  });
-  const errors = map(invalidTypes, (currType) => {
-    return {
-      message: "Token Type: ->" + currType.name + "<- static 'GROUP' can only be Lexer.SKIPPED/Lexer.NA/A String",
-      type: LexerDefinitionErrorType.INVALID_GROUP_TYPE_FOUND,
-      tokenTypes: [currType]
-    };
-  });
-  return errors;
-}
-function findModesThatDoNotExist(tokenTypes, validModes) {
-  const invalidModes = filter(tokenTypes, (clazz) => {
-    return clazz.PUSH_MODE !== void 0 && !includes(validModes, clazz.PUSH_MODE);
-  });
-  const errors = map(invalidModes, (tokType) => {
-    const msg = `Token Type: ->${tokType.name}<- static 'PUSH_MODE' value cannot refer to a Lexer Mode ->${tokType.PUSH_MODE}<-which does not exist`;
-    return {
-      message: msg,
-      type: LexerDefinitionErrorType.PUSH_MODE_DOES_NOT_EXIST,
-      tokenTypes: [tokType]
-    };
-  });
-  return errors;
-}
-function findUnreachablePatterns(tokenTypes) {
-  const errors = [];
-  const canBeTested = reduce(tokenTypes, (result, tokType, idx) => {
-    const pattern = tokType.PATTERN;
-    if (pattern === Lexer.NA) {
-      return result;
-    }
-    if (isString(pattern)) {
-      result.push({ str: pattern, idx, tokenType: tokType });
-    } else if (isRegExp(pattern) && noMetaChar(pattern)) {
-      result.push({ str: pattern.source, idx, tokenType: tokType });
-    }
-    return result;
-  }, []);
-  forEach(tokenTypes, (tokType, testIdx) => {
-    forEach(canBeTested, ({ str, idx, tokenType }) => {
-      if (testIdx < idx && testTokenType(str, tokType.PATTERN)) {
-        const msg = `Token: ->${tokenType.name}<- can never be matched.
-Because it appears AFTER the Token Type ->${tokType.name}<-in the lexer's definition.
-See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNREACHABLE`;
-        errors.push({
-          message: msg,
-          type: LexerDefinitionErrorType.UNREACHABLE_PATTERN,
-          tokenTypes: [tokType, tokenType]
-        });
-      }
-    });
-  });
-  return errors;
-}
-function testTokenType(str, pattern) {
-  if (isRegExp(pattern)) {
-    const regExpArray = pattern.exec(str);
-    return regExpArray !== null && regExpArray.index === 0;
-  } else if (isFunction(pattern)) {
-    return pattern(str, 0, [], {});
-  } else if (has(pattern, "exec")) {
-    return pattern.exec(str, 0, [], {});
-  } else if (typeof pattern === "string") {
-    return pattern === str;
-  } else {
-    throw Error("non exhaustive match");
-  }
-}
-function noMetaChar(regExp) {
-  const metaChars = [
-    ".",
-    "\\",
-    "[",
-    "]",
-    "|",
-    "^",
-    "$",
-    "(",
-    ")",
-    "?",
-    "*",
-    "+",
-    "{"
-  ];
-  return find(metaChars, (char) => regExp.source.indexOf(char) !== -1) === void 0;
-}
-function addStartOfInput(pattern) {
-  const flags = pattern.ignoreCase ? "i" : "";
-  return new RegExp(`^(?:${pattern.source})`, flags);
-}
-function addStickyFlag(pattern) {
-  const flags = pattern.ignoreCase ? "iy" : "y";
-  return new RegExp(`${pattern.source}`, flags);
-}
-function performRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
-  const errors = [];
-  if (!has(lexerDefinition, DEFAULT_MODE)) {
-    errors.push({
-      message: "A MultiMode Lexer cannot be initialized without a <" + DEFAULT_MODE + "> property in its definition\n",
-      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE
-    });
-  }
-  if (!has(lexerDefinition, MODES)) {
-    errors.push({
-      message: "A MultiMode Lexer cannot be initialized without a <" + MODES + "> property in its definition\n",
-      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY
-    });
-  }
-  if (has(lexerDefinition, MODES) && has(lexerDefinition, DEFAULT_MODE) && !has(lexerDefinition.modes, lexerDefinition.defaultMode)) {
-    errors.push({
-      message: `A MultiMode Lexer cannot be initialized with a ${DEFAULT_MODE}: <${lexerDefinition.defaultMode}>which does not exist
-`,
-      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST
-    });
-  }
-  if (has(lexerDefinition, MODES)) {
-    forEach(lexerDefinition.modes, (currModeValue, currModeName) => {
-      forEach(currModeValue, (currTokType, currIdx) => {
-        if (isUndefined(currTokType)) {
-          errors.push({
-            message: `A Lexer cannot be initialized using an undefined Token Type. Mode:<${currModeName}> at index: <${currIdx}>
-`,
-            type: LexerDefinitionErrorType.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED
-          });
-        } else if (has(currTokType, "LONGER_ALT")) {
-          const longerAlt = isArray(currTokType.LONGER_ALT) ? currTokType.LONGER_ALT : [currTokType.LONGER_ALT];
-          forEach(longerAlt, (currLongerAlt) => {
-            if (!isUndefined(currLongerAlt) && !includes(currModeValue, currLongerAlt)) {
-              errors.push({
-                message: `A MultiMode Lexer cannot be initialized with a longer_alt <${currLongerAlt.name}> on token <${currTokType.name}> outside of mode <${currModeName}>
-`,
-                type: LexerDefinitionErrorType.MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE
-              });
-            }
-          });
-        }
-      });
-    });
-  }
-  return errors;
-}
-function performWarningRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
-  const warnings = [];
-  let hasAnyLineBreak = false;
-  const allTokenTypes = compact(flatten(values(lexerDefinition.modes)));
-  const concreteTokenTypes = reject(allTokenTypes, (currType) => currType[PATTERN] === Lexer.NA);
-  const terminatorCharCodes = getCharCodes(lineTerminatorCharacters);
-  if (trackLines) {
-    forEach(concreteTokenTypes, (tokType) => {
-      const currIssue = checkLineBreaksIssues(tokType, terminatorCharCodes);
-      if (currIssue !== false) {
-        const message = buildLineBreakIssueMessage(tokType, currIssue);
-        const warningDescriptor = {
-          message,
-          type: currIssue.issue,
-          tokenType: tokType
-        };
-        warnings.push(warningDescriptor);
-      } else {
-        if (has(tokType, "LINE_BREAKS")) {
-          if (tokType.LINE_BREAKS === true) {
-            hasAnyLineBreak = true;
-          }
-        } else {
-          if (canMatchCharCode(terminatorCharCodes, tokType.PATTERN)) {
-            hasAnyLineBreak = true;
-          }
-        }
-      }
-    });
-  }
-  if (trackLines && !hasAnyLineBreak) {
-    warnings.push({
-      message: "Warning: No LINE_BREAKS Found.\n	This Lexer has been defined to track line and column information,\n	But none of the Token Types can be identified as matching a line terminator.\n	See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#LINE_BREAKS \n	for details.",
-      type: LexerDefinitionErrorType.NO_LINE_BREAKS_FLAGS
-    });
-  }
-  return warnings;
-}
-function cloneEmptyGroups(emptyGroups) {
-  const clonedResult = {};
-  const groupKeys = keys(emptyGroups);
-  forEach(groupKeys, (currKey) => {
-    const currGroupValue = emptyGroups[currKey];
-    if (isArray(currGroupValue)) {
-      clonedResult[currKey] = [];
-    } else {
-      throw Error("non exhaustive match");
-    }
-  });
-  return clonedResult;
-}
-function isCustomPattern(tokenType) {
-  const pattern = tokenType.PATTERN;
-  if (isRegExp(pattern)) {
-    return false;
-  } else if (isFunction(pattern)) {
-    return true;
-  } else if (has(pattern, "exec")) {
-    return true;
-  } else if (isString(pattern)) {
-    return false;
-  } else {
-    throw Error("non exhaustive match");
-  }
-}
-function isShortPattern(pattern) {
-  if (isString(pattern) && pattern.length === 1) {
-    return pattern.charCodeAt(0);
-  } else {
-    return false;
-  }
-}
-const LineTerminatorOptimizedTester = {
-  // implements /\n|\r\n?/g.test
-  test: function(text) {
-    const len = text.length;
-    for (let i = this.lastIndex; i < len; i++) {
-      const c = text.charCodeAt(i);
-      if (c === 10) {
-        this.lastIndex = i + 1;
-        return true;
-      } else if (c === 13) {
-        if (text.charCodeAt(i + 1) === 10) {
-          this.lastIndex = i + 2;
-        } else {
-          this.lastIndex = i + 1;
-        }
-        return true;
-      }
-    }
-    return false;
-  },
-  lastIndex: 0
-};
-function checkLineBreaksIssues(tokType, lineTerminatorCharCodes) {
-  if (has(tokType, "LINE_BREAKS")) {
-    return false;
-  } else {
-    if (isRegExp(tokType.PATTERN)) {
-      try {
-        canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN);
-      } catch (e) {
-        return {
-          issue: LexerDefinitionErrorType.IDENTIFY_TERMINATOR,
-          errMsg: e.message
-        };
-      }
-      return false;
-    } else if (isString(tokType.PATTERN)) {
-      return false;
-    } else if (isCustomPattern(tokType)) {
-      return { issue: LexerDefinitionErrorType.CUSTOM_LINE_BREAK };
-    } else {
-      throw Error("non exhaustive match");
-    }
-  }
-}
-function buildLineBreakIssueMessage(tokType, details) {
-  if (details.issue === LexerDefinitionErrorType.IDENTIFY_TERMINATOR) {
-    return `Warning: unable to identify line terminator usage in pattern.
-	The problem is in the <${tokType.name}> Token Type
-	 Root cause: ${details.errMsg}.
-	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#IDENTIFY_TERMINATOR`;
-  } else if (details.issue === LexerDefinitionErrorType.CUSTOM_LINE_BREAK) {
-    return `Warning: A Custom Token Pattern should specify the <line_breaks> option.
-	The problem is in the <${tokType.name}> Token Type
-	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_LINE_BREAK`;
-  } else {
-    throw Error("non exhaustive match");
-  }
-}
-function getCharCodes(charsOrCodes) {
-  const charCodes = map(charsOrCodes, (numOrString) => {
-    if (isString(numOrString)) {
-      return numOrString.charCodeAt(0);
-    } else {
-      return numOrString;
-    }
-  });
-  return charCodes;
-}
-function addToMapOfArrays(map2, key, value) {
-  if (map2[key] === void 0) {
-    map2[key] = [value];
-  } else {
-    map2[key].push(value);
-  }
-}
-const minOptimizationVal = 256;
-let charCodeToOptimizedIdxMap = [];
-function charCodeToOptimizedIndex(charCode) {
-  return charCode < minOptimizationVal ? charCode : charCodeToOptimizedIdxMap[charCode];
-}
-function initCharCodeToOptimizedIndexMap() {
-  if (isEmpty(charCodeToOptimizedIdxMap)) {
-    charCodeToOptimizedIdxMap = new Array(65536);
-    for (let i = 0; i < 65536; i++) {
-      charCodeToOptimizedIdxMap[i] = i > 255 ? 255 + ~~(i / 255) : i;
-    }
-  }
-}
-export {
-  DEFAULT_MODE,
-  LineTerminatorOptimizedTester,
-  MODES,
-  SUPPORT_STICKY,
-  addStartOfInput,
-  addStickyFlag,
-  analyzeTokenTypes,
-  buildLineBreakIssueMessage,
-  charCodeToOptimizedIndex,
-  cloneEmptyGroups,
-  findDuplicatePatterns,
-  findEmptyMatchRegExps,
-  findEndOfInputAnchor,
-  findInvalidGroupType,
-  findInvalidPatterns,
-  findMissingPatterns,
-  findModesThatDoNotExist,
-  findStartOfInputAnchor,
-  findUnreachablePatterns,
-  findUnsupportedFlags,
-  isCustomPattern,
-  isShortPattern,
-  minOptimizationVal,
-  performRuntimeChecks,
-  performWarningRuntimeChecks,
-  validatePatterns
-};