@elmethis/core 1.0.0-alpha.147 → 1.0.0-alpha.148

package/dist/node_modules/.pnpm/chevrotain@11.0.3/node_modules/chevrotain/lib/src/scan/lexer.mjs

@@ -0,0 +1,720 @@
+import "../../../../../../@chevrotain_regexp-to-ast@11.0.3/node_modules/@chevrotain/regexp-to-ast/lib/src/character-classes.mjs";
+import { BaseRegExpVisitor } from "../../../../../../@chevrotain_regexp-to-ast@11.0.3/node_modules/@chevrotain/regexp-to-ast/lib/src/base-regexp-visitor.mjs";
+import { LexerDefinitionErrorType, Lexer } from "./lexer_public.mjs";
+import { has, forEach, isUndefined, isArray, includes, compact, flatten, values, reject, defaults, map, isRegExp, isFunction, isString, indexOf, reduce, isEmpty, keys, filter, difference, first, find } from "lodash-es";
+import { PRINT_ERROR } from "../../../../../../@chevrotain_utils@11.0.3/node_modules/@chevrotain/utils/lib/src/print.mjs";
+import { canMatchCharCode, failedOptimizationPrefixMsg, getOptimizedStartCodesIndices } from "./reg_exp.mjs";
+import { getRegExpAst } from "./reg_exp_parser.mjs";
+const PATTERN = "PATTERN";
+const DEFAULT_MODE = "defaultMode";
+const MODES = "modes";
+let SUPPORT_STICKY = typeof new RegExp("(?:)").sticky === "boolean";
+function analyzeTokenTypes(tokenTypes, options) {
+  options = defaults(options, {
+    useSticky: SUPPORT_STICKY,
+    debug: false,
+    safeMode: false,
+    positionTracking: "full",
+    lineTerminatorCharacters: ["\r", "\n"],
+    tracer: (msg, action) => action()
+  });
+  const tracer = options.tracer;
+  tracer("initCharCodeToOptimizedIndexMap", () => {
+    initCharCodeToOptimizedIndexMap();
+  });
+  let onlyRelevantTypes;
+  tracer("Reject Lexer.NA", () => {
+    onlyRelevantTypes = reject(tokenTypes, (currType) => {
+      return currType[PATTERN] === Lexer.NA;
+    });
+  });
+  let hasCustom = false;
+  let allTransformedPatterns;
+  tracer("Transform Patterns", () => {
+    hasCustom = false;
+    allTransformedPatterns = map(onlyRelevantTypes, (currType) => {
+      const currPattern = currType[PATTERN];
+      if (isRegExp(currPattern)) {
+        const regExpSource = currPattern.source;
+        if (regExpSource.length === 1 && // only these regExp meta characters which can appear in a length one regExp
+        regExpSource !== "^" && regExpSource !== "$" && regExpSource !== "." && !currPattern.ignoreCase) {
+          return regExpSource;
+        } else if (regExpSource.length === 2 && regExpSource[0] === "\\" && // not a meta character
+        !includes([
+          "d",
+          "D",
+          "s",
+          "S",
+          "t",
+          "r",
+          "n",
+          "t",
+          "0",
+          "c",
+          "b",
+          "B",
+          "f",
+          "v",
+          "w",
+          "W"
+        ], regExpSource[1])) {
+          return regExpSource[1];
+        } else {
+          return options.useSticky ? addStickyFlag(currPattern) : addStartOfInput(currPattern);
+        }
+      } else if (isFunction(currPattern)) {
+        hasCustom = true;
+        return { exec: currPattern };
+      } else if (typeof currPattern === "object") {
+        hasCustom = true;
+        return currPattern;
+      } else if (typeof currPattern === "string") {
+        if (currPattern.length === 1) {
+          return currPattern;
+        } else {
+          const escapedRegExpString = currPattern.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+          const wrappedRegExp = new RegExp(escapedRegExpString);
+          return options.useSticky ? addStickyFlag(wrappedRegExp) : addStartOfInput(wrappedRegExp);
+        }
+      } else {
+        throw Error("non exhaustive match");
+      }
+    });
+  });
+  let patternIdxToType;
+  let patternIdxToGroup;
+  let patternIdxToLongerAltIdxArr;
+  let patternIdxToPushMode;
+  let patternIdxToPopMode;
+  tracer("misc mapping", () => {
+    patternIdxToType = map(onlyRelevantTypes, (currType) => currType.tokenTypeIdx);
+    patternIdxToGroup = map(onlyRelevantTypes, (clazz) => {
+      const groupName = clazz.GROUP;
+      if (groupName === Lexer.SKIPPED) {
+        return void 0;
+      } else if (isString(groupName)) {
+        return groupName;
+      } else if (isUndefined(groupName)) {
+        return false;
+      } else {
+        throw Error("non exhaustive match");
+      }
+    });
+    patternIdxToLongerAltIdxArr = map(onlyRelevantTypes, (clazz) => {
+      const longerAltType = clazz.LONGER_ALT;
+      if (longerAltType) {
+        const longerAltIdxArr = isArray(longerAltType) ? map(longerAltType, (type) => indexOf(onlyRelevantTypes, type)) : [indexOf(onlyRelevantTypes, longerAltType)];
+        return longerAltIdxArr;
+      }
+    });
+    patternIdxToPushMode = map(onlyRelevantTypes, (clazz) => clazz.PUSH_MODE);
+    patternIdxToPopMode = map(onlyRelevantTypes, (clazz) => has(clazz, "POP_MODE"));
+  });
+  let patternIdxToCanLineTerminator;
+  tracer("Line Terminator Handling", () => {
+    const lineTerminatorCharCodes = getCharCodes(options.lineTerminatorCharacters);
+    patternIdxToCanLineTerminator = map(onlyRelevantTypes, (tokType) => false);
+    if (options.positionTracking !== "onlyOffset") {
+      patternIdxToCanLineTerminator = map(onlyRelevantTypes, (tokType) => {
+        if (has(tokType, "LINE_BREAKS")) {
+          return !!tokType.LINE_BREAKS;
+        } else {
+          return checkLineBreaksIssues(tokType, lineTerminatorCharCodes) === false && canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN);
+        }
+      });
+    }
+  });
+  let patternIdxToIsCustom;
+  let patternIdxToShort;
+  let emptyGroups;
+  let patternIdxToConfig;
+  tracer("Misc Mapping #2", () => {
+    patternIdxToIsCustom = map(onlyRelevantTypes, isCustomPattern);
+    patternIdxToShort = map(allTransformedPatterns, isShortPattern);
+    emptyGroups = reduce(onlyRelevantTypes, (acc, clazz) => {
+      const groupName = clazz.GROUP;
+      if (isString(groupName) && !(groupName === Lexer.SKIPPED)) {
+        acc[groupName] = [];
+      }
+      return acc;
+    }, {});
+    patternIdxToConfig = map(allTransformedPatterns, (x, idx) => {
+      return {
+        pattern: allTransformedPatterns[idx],
+        longerAlt: patternIdxToLongerAltIdxArr[idx],
+        canLineTerminator: patternIdxToCanLineTerminator[idx],
+        isCustom: patternIdxToIsCustom[idx],
+        short: patternIdxToShort[idx],
+        group: patternIdxToGroup[idx],
+        push: patternIdxToPushMode[idx],
+        pop: patternIdxToPopMode[idx],
+        tokenTypeIdx: patternIdxToType[idx],
+        tokenType: onlyRelevantTypes[idx]
+      };
+    });
+  });
+  let canBeOptimized = true;
+  let charCodeToPatternIdxToConfig = [];
+  if (!options.safeMode) {
+    tracer("First Char Optimization", () => {
+      charCodeToPatternIdxToConfig = reduce(onlyRelevantTypes, (result, currTokType, idx) => {
+        if (typeof currTokType.PATTERN === "string") {
+          const charCode = currTokType.PATTERN.charCodeAt(0);
+          const optimizedIdx = charCodeToOptimizedIndex(charCode);
+          addToMapOfArrays(result, optimizedIdx, patternIdxToConfig[idx]);
+        } else if (isArray(currTokType.START_CHARS_HINT)) {
+          let lastOptimizedIdx;
+          forEach(currTokType.START_CHARS_HINT, (charOrInt) => {
+            const charCode = typeof charOrInt === "string" ? charOrInt.charCodeAt(0) : charOrInt;
+            const currOptimizedIdx = charCodeToOptimizedIndex(charCode);
+            if (lastOptimizedIdx !== currOptimizedIdx) {
+              lastOptimizedIdx = currOptimizedIdx;
+              addToMapOfArrays(result, currOptimizedIdx, patternIdxToConfig[idx]);
+            }
+          });
+        } else if (isRegExp(currTokType.PATTERN)) {
+          if (currTokType.PATTERN.unicode) {
+            canBeOptimized = false;
+            if (options.ensureOptimizations) {
+              PRINT_ERROR(`${failedOptimizationPrefixMsg}	Unable to analyze < ${currTokType.PATTERN.toString()} > pattern.
+	The regexp unicode flag is not currently supported by the regexp-to-ast library.
+	This will disable the lexer's first char optimizations.
+	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNICODE_OPTIMIZE`);
+            }
+          } else {
+            const optimizedCodes = getOptimizedStartCodesIndices(currTokType.PATTERN, options.ensureOptimizations);
+            if (isEmpty(optimizedCodes)) {
+              canBeOptimized = false;
+            }
+            forEach(optimizedCodes, (code) => {
+              addToMapOfArrays(result, code, patternIdxToConfig[idx]);
+            });
+          }
+        } else {
+          if (options.ensureOptimizations) {
+            PRINT_ERROR(`${failedOptimizationPrefixMsg}	TokenType: <${currTokType.name}> is using a custom token pattern without providing <start_chars_hint> parameter.
+	This will disable the lexer's first char optimizations.
+	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_OPTIMIZE`);
+          }
+          canBeOptimized = false;
+        }
+        return result;
+      }, []);
+    });
+  }
+  return {
+    emptyGroups,
+    patternIdxToConfig,
+    charCodeToPatternIdxToConfig,
+    hasCustom,
+    canBeOptimized
+  };
+}
+function validatePatterns(tokenTypes, validModesNames) {
+  let errors = [];
+  const missingResult = findMissingPatterns(tokenTypes);
+  errors = errors.concat(missingResult.errors);
+  const invalidResult = findInvalidPatterns(missingResult.valid);
+  const validTokenTypes = invalidResult.valid;
+  errors = errors.concat(invalidResult.errors);
+  errors = errors.concat(validateRegExpPattern(validTokenTypes));
+  errors = errors.concat(findInvalidGroupType(validTokenTypes));
+  errors = errors.concat(findModesThatDoNotExist(validTokenTypes, validModesNames));
+  errors = errors.concat(findUnreachablePatterns(validTokenTypes));
+  return errors;
+}
+function validateRegExpPattern(tokenTypes) {
+  let errors = [];
+  const withRegExpPatterns = filter(tokenTypes, (currTokType) => isRegExp(currTokType[PATTERN]));
+  errors = errors.concat(findEndOfInputAnchor(withRegExpPatterns));
+  errors = errors.concat(findStartOfInputAnchor(withRegExpPatterns));
+  errors = errors.concat(findUnsupportedFlags(withRegExpPatterns));
+  errors = errors.concat(findDuplicatePatterns(withRegExpPatterns));
+  errors = errors.concat(findEmptyMatchRegExps(withRegExpPatterns));
+  return errors;
+}
+function findMissingPatterns(tokenTypes) {
+  const tokenTypesWithMissingPattern = filter(tokenTypes, (currType) => {
+    return !has(currType, PATTERN);
+  });
+  const errors = map(tokenTypesWithMissingPattern, (currType) => {
+    return {
+      message: "Token Type: ->" + currType.name + "<- missing static 'PATTERN' property",
+      type: LexerDefinitionErrorType.MISSING_PATTERN,
+      tokenTypes: [currType]
+    };
+  });
+  const valid = difference(tokenTypes, tokenTypesWithMissingPattern);
+  return { errors, valid };
+}
+function findInvalidPatterns(tokenTypes) {
+  const tokenTypesWithInvalidPattern = filter(tokenTypes, (currType) => {
+    const pattern = currType[PATTERN];
+    return !isRegExp(pattern) && !isFunction(pattern) && !has(pattern, "exec") && !isString(pattern);
+  });
+  const errors = map(tokenTypesWithInvalidPattern, (currType) => {
+    return {
+      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' can only be a RegExp, a Function matching the {CustomPatternMatcherFunc} type or an Object matching the {ICustomPattern} interface.",
+      type: LexerDefinitionErrorType.INVALID_PATTERN,
+      tokenTypes: [currType]
+    };
+  });
+  const valid = difference(tokenTypes, tokenTypesWithInvalidPattern);
+  return { errors, valid };
+}
+const end_of_input = /[^\\][$]/;
+function findEndOfInputAnchor(tokenTypes) {
+  class EndAnchorFinder extends BaseRegExpVisitor {
+    constructor() {
+      super(...arguments);
+      this.found = false;
+    }
+    visitEndAnchor(node) {
+      this.found = true;
+    }
+  }
+  const invalidRegex = filter(tokenTypes, (currType) => {
+    const pattern = currType.PATTERN;
+    try {
+      const regexpAst = getRegExpAst(pattern);
+      const endAnchorVisitor = new EndAnchorFinder();
+      endAnchorVisitor.visit(regexpAst);
+      return endAnchorVisitor.found;
+    } catch (e) {
+      return end_of_input.test(pattern.source);
+    }
+  });
+  const errors = map(invalidRegex, (currType) => {
+    return {
+      message: "Unexpected RegExp Anchor Error:\n	Token Type: ->" + currType.name + "<- static 'PATTERN' cannot contain end of input anchor '$'\n	See chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS	for details.",
+      type: LexerDefinitionErrorType.EOI_ANCHOR_FOUND,
+      tokenTypes: [currType]
+    };
+  });
+  return errors;
+}
+function findEmptyMatchRegExps(tokenTypes) {
+  const matchesEmptyString = filter(tokenTypes, (currType) => {
+    const pattern = currType.PATTERN;
+    return pattern.test("");
+  });
+  const errors = map(matchesEmptyString, (currType) => {
+    return {
+      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' must not match an empty string",
+      type: LexerDefinitionErrorType.EMPTY_MATCH_PATTERN,
+      tokenTypes: [currType]
+    };
+  });
+  return errors;
+}
+const start_of_input = /[^\\[][\^]|^\^/;
+function findStartOfInputAnchor(tokenTypes) {
+  class StartAnchorFinder extends BaseRegExpVisitor {
+    constructor() {
+      super(...arguments);
+      this.found = false;
+    }
+    visitStartAnchor(node) {
+      this.found = true;
+    }
+  }
+  const invalidRegex = filter(tokenTypes, (currType) => {
+    const pattern = currType.PATTERN;
+    try {
+      const regexpAst = getRegExpAst(pattern);
+      const startAnchorVisitor = new StartAnchorFinder();
+      startAnchorVisitor.visit(regexpAst);
+      return startAnchorVisitor.found;
+    } catch (e) {
+      return start_of_input.test(pattern.source);
+    }
+  });
+  const errors = map(invalidRegex, (currType) => {
+    return {
+      message: "Unexpected RegExp Anchor Error:\n	Token Type: ->" + currType.name + "<- static 'PATTERN' cannot contain start of input anchor '^'\n	See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS	for details.",
+      type: LexerDefinitionErrorType.SOI_ANCHOR_FOUND,
+      tokenTypes: [currType]
+    };
+  });
+  return errors;
+}
+function findUnsupportedFlags(tokenTypes) {
+  const invalidFlags = filter(tokenTypes, (currType) => {
+    const pattern = currType[PATTERN];
+    return pattern instanceof RegExp && (pattern.multiline || pattern.global);
+  });
+  const errors = map(invalidFlags, (currType) => {
+    return {
+      message: "Token Type: ->" + currType.name + "<- static 'PATTERN' may NOT contain global('g') or multiline('m')",
+      type: LexerDefinitionErrorType.UNSUPPORTED_FLAGS_FOUND,
+      tokenTypes: [currType]
+    };
+  });
+  return errors;
+}
+function findDuplicatePatterns(tokenTypes) {
+  const found = [];
+  let identicalPatterns = map(tokenTypes, (outerType) => {
+    return reduce(tokenTypes, (result, innerType) => {
+      if (outerType.PATTERN.source === innerType.PATTERN.source && !includes(found, innerType) && innerType.PATTERN !== Lexer.NA) {
+        found.push(innerType);
+        result.push(innerType);
+        return result;
+      }
+      return result;
+    }, []);
+  });
+  identicalPatterns = compact(identicalPatterns);
+  const duplicatePatterns = filter(identicalPatterns, (currIdenticalSet) => {
+    return currIdenticalSet.length > 1;
+  });
+  const errors = map(duplicatePatterns, (setOfIdentical) => {
+    const tokenTypeNames = map(setOfIdentical, (currType) => {
+      return currType.name;
+    });
+    const dupPatternSrc = first(setOfIdentical).PATTERN;
+    return {
+      message: `The same RegExp pattern ->${dupPatternSrc}<-has been used in all of the following Token Types: ${tokenTypeNames.join(", ")} <-`,
+      type: LexerDefinitionErrorType.DUPLICATE_PATTERNS_FOUND,
+      tokenTypes: setOfIdentical
+    };
+  });
+  return errors;
+}
+function findInvalidGroupType(tokenTypes) {
+  const invalidTypes = filter(tokenTypes, (clazz) => {
+    if (!has(clazz, "GROUP")) {
+      return false;
+    }
+    const group = clazz.GROUP;
+    return group !== Lexer.SKIPPED && group !== Lexer.NA && !isString(group);
+  });
+  const errors = map(invalidTypes, (currType) => {
+    return {
+      message: "Token Type: ->" + currType.name + "<- static 'GROUP' can only be Lexer.SKIPPED/Lexer.NA/A String",
+      type: LexerDefinitionErrorType.INVALID_GROUP_TYPE_FOUND,
+      tokenTypes: [currType]
+    };
+  });
+  return errors;
+}
+function findModesThatDoNotExist(tokenTypes, validModes) {
+  const invalidModes = filter(tokenTypes, (clazz) => {
+    return clazz.PUSH_MODE !== void 0 && !includes(validModes, clazz.PUSH_MODE);
+  });
+  const errors = map(invalidModes, (tokType) => {
+    const msg = `Token Type: ->${tokType.name}<- static 'PUSH_MODE' value cannot refer to a Lexer Mode ->${tokType.PUSH_MODE}<-which does not exist`;
+    return {
+      message: msg,
+      type: LexerDefinitionErrorType.PUSH_MODE_DOES_NOT_EXIST,
+      tokenTypes: [tokType]
+    };
+  });
+  return errors;
+}
+function findUnreachablePatterns(tokenTypes) {
+  const errors = [];
+  const canBeTested = reduce(tokenTypes, (result, tokType, idx) => {
+    const pattern = tokType.PATTERN;
+    if (pattern === Lexer.NA) {
+      return result;
+    }
+    if (isString(pattern)) {
+      result.push({ str: pattern, idx, tokenType: tokType });
+    } else if (isRegExp(pattern) && noMetaChar(pattern)) {
+      result.push({ str: pattern.source, idx, tokenType: tokType });
+    }
+    return result;
+  }, []);
+  forEach(tokenTypes, (tokType, testIdx) => {
+    forEach(canBeTested, ({ str, idx, tokenType }) => {
+      if (testIdx < idx && testTokenType(str, tokType.PATTERN)) {
+        const msg = `Token: ->${tokenType.name}<- can never be matched.
+Because it appears AFTER the Token Type ->${tokType.name}<-in the lexer's definition.
+See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNREACHABLE`;
+        errors.push({
+          message: msg,
+          type: LexerDefinitionErrorType.UNREACHABLE_PATTERN,
+          tokenTypes: [tokType, tokenType]
+        });
+      }
+    });
+  });
+  return errors;
+}
+function testTokenType(str, pattern) {
+  if (isRegExp(pattern)) {
+    const regExpArray = pattern.exec(str);
+    return regExpArray !== null && regExpArray.index === 0;
+  } else if (isFunction(pattern)) {
+    return pattern(str, 0, [], {});
+  } else if (has(pattern, "exec")) {
+    return pattern.exec(str, 0, [], {});
+  } else if (typeof pattern === "string") {
+    return pattern === str;
+  } else {
+    throw Error("non exhaustive match");
+  }
+}
+function noMetaChar(regExp) {
+  const metaChars = [
+    ".",
+    "\\",
+    "[",
+    "]",
+    "|",
+    "^",
+    "$",
+    "(",
+    ")",
+    "?",
+    "*",
+    "+",
+    "{"
+  ];
+  return find(metaChars, (char) => regExp.source.indexOf(char) !== -1) === void 0;
+}
+function addStartOfInput(pattern) {
+  const flags = pattern.ignoreCase ? "i" : "";
+  return new RegExp(`^(?:${pattern.source})`, flags);
+}
+function addStickyFlag(pattern) {
+  const flags = pattern.ignoreCase ? "iy" : "y";
+  return new RegExp(`${pattern.source}`, flags);
+}
+function performRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
+  const errors = [];
+  if (!has(lexerDefinition, DEFAULT_MODE)) {
+    errors.push({
+      message: "A MultiMode Lexer cannot be initialized without a <" + DEFAULT_MODE + "> property in its definition\n",
+      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE
+    });
+  }
+  if (!has(lexerDefinition, MODES)) {
+    errors.push({
+      message: "A MultiMode Lexer cannot be initialized without a <" + MODES + "> property in its definition\n",
+      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY
+    });
+  }
+  if (has(lexerDefinition, MODES) && has(lexerDefinition, DEFAULT_MODE) && !has(lexerDefinition.modes, lexerDefinition.defaultMode)) {
+    errors.push({
+      message: `A MultiMode Lexer cannot be initialized with a ${DEFAULT_MODE}: <${lexerDefinition.defaultMode}>which does not exist
+`,
+      type: LexerDefinitionErrorType.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST
+    });
+  }
+  if (has(lexerDefinition, MODES)) {
+    forEach(lexerDefinition.modes, (currModeValue, currModeName) => {
+      forEach(currModeValue, (currTokType, currIdx) => {
+        if (isUndefined(currTokType)) {
+          errors.push({
+            message: `A Lexer cannot be initialized using an undefined Token Type. Mode:<${currModeName}> at index: <${currIdx}>
+`,
+            type: LexerDefinitionErrorType.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED
+          });
+        } else if (has(currTokType, "LONGER_ALT")) {
+          const longerAlt = isArray(currTokType.LONGER_ALT) ? currTokType.LONGER_ALT : [currTokType.LONGER_ALT];
+          forEach(longerAlt, (currLongerAlt) => {
+            if (!isUndefined(currLongerAlt) && !includes(currModeValue, currLongerAlt)) {
+              errors.push({
+                message: `A MultiMode Lexer cannot be initialized with a longer_alt <${currLongerAlt.name}> on token <${currTokType.name}> outside of mode <${currModeName}>
+`,
+                type: LexerDefinitionErrorType.MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE
+              });
+            }
+          });
+        }
+      });
+    });
+  }
+  return errors;
+}
+function performWarningRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
+  const warnings = [];
+  let hasAnyLineBreak = false;
+  const allTokenTypes = compact(flatten(values(lexerDefinition.modes)));
+  const concreteTokenTypes = reject(allTokenTypes, (currType) => currType[PATTERN] === Lexer.NA);
+  const terminatorCharCodes = getCharCodes(lineTerminatorCharacters);
+  if (trackLines) {
+    forEach(concreteTokenTypes, (tokType) => {
+      const currIssue = checkLineBreaksIssues(tokType, terminatorCharCodes);
+      if (currIssue !== false) {
+        const message = buildLineBreakIssueMessage(tokType, currIssue);
+        const warningDescriptor = {
+          message,
+          type: currIssue.issue,
+          tokenType: tokType
+        };
+        warnings.push(warningDescriptor);
+      } else {
+        if (has(tokType, "LINE_BREAKS")) {
+          if (tokType.LINE_BREAKS === true) {
+            hasAnyLineBreak = true;
+          }
+        } else {
+          if (canMatchCharCode(terminatorCharCodes, tokType.PATTERN)) {
+            hasAnyLineBreak = true;
+          }
+        }
+      }
+    });
+  }
+  if (trackLines && !hasAnyLineBreak) {
+    warnings.push({
+      message: "Warning: No LINE_BREAKS Found.\n	This Lexer has been defined to track line and column information,\n	But none of the Token Types can be identified as matching a line terminator.\n	See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#LINE_BREAKS \n	for details.",
+      type: LexerDefinitionErrorType.NO_LINE_BREAKS_FLAGS
+    });
+  }
+  return warnings;
+}
+function cloneEmptyGroups(emptyGroups) {
+  const clonedResult = {};
+  const groupKeys = keys(emptyGroups);
+  forEach(groupKeys, (currKey) => {
+    const currGroupValue = emptyGroups[currKey];
+    if (isArray(currGroupValue)) {
+      clonedResult[currKey] = [];
+    } else {
+      throw Error("non exhaustive match");
+    }
+  });
+  return clonedResult;
+}
+function isCustomPattern(tokenType) {
+  const pattern = tokenType.PATTERN;
+  if (isRegExp(pattern)) {
+    return false;
+  } else if (isFunction(pattern)) {
+    return true;
+  } else if (has(pattern, "exec")) {
+    return true;
+  } else if (isString(pattern)) {
+    return false;
+  } else {
+    throw Error("non exhaustive match");
+  }
+}
+function isShortPattern(pattern) {
+  if (isString(pattern) && pattern.length === 1) {
+    return pattern.charCodeAt(0);
+  } else {
+    return false;
+  }
+}
+const LineTerminatorOptimizedTester = {
+  // implements /\n|\r\n?/g.test
+  test: function(text) {
+    const len = text.length;
+    for (let i = this.lastIndex; i < len; i++) {
+      const c = text.charCodeAt(i);
+      if (c === 10) {
+        this.lastIndex = i + 1;
+        return true;
+      } else if (c === 13) {
+        if (text.charCodeAt(i + 1) === 10) {
+          this.lastIndex = i + 2;
+        } else {
+          this.lastIndex = i + 1;
+        }
+        return true;
+      }
+    }
+    return false;
+  },
+  lastIndex: 0
+};
+function checkLineBreaksIssues(tokType, lineTerminatorCharCodes) {
+  if (has(tokType, "LINE_BREAKS")) {
+    return false;
+  } else {
+    if (isRegExp(tokType.PATTERN)) {
+      try {
+        canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN);
+      } catch (e) {
+        return {
+          issue: LexerDefinitionErrorType.IDENTIFY_TERMINATOR,
+          errMsg: e.message
+        };
+      }
+      return false;
+    } else if (isString(tokType.PATTERN)) {
+      return false;
+    } else if (isCustomPattern(tokType)) {
+      return { issue: LexerDefinitionErrorType.CUSTOM_LINE_BREAK };
+    } else {
+      throw Error("non exhaustive match");
+    }
+  }
+}
+function buildLineBreakIssueMessage(tokType, details) {
+  if (details.issue === LexerDefinitionErrorType.IDENTIFY_TERMINATOR) {
+    return `Warning: unable to identify line terminator usage in pattern.
+	The problem is in the <${tokType.name}> Token Type
+	 Root cause: ${details.errMsg}.
+	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#IDENTIFY_TERMINATOR`;
+  } else if (details.issue === LexerDefinitionErrorType.CUSTOM_LINE_BREAK) {
+    return `Warning: A Custom Token Pattern should specify the <line_breaks> option.
+	The problem is in the <${tokType.name}> Token Type
+	For details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_LINE_BREAK`;
+  } else {
+    throw Error("non exhaustive match");
+  }
+}
+function getCharCodes(charsOrCodes) {
+  const charCodes = map(charsOrCodes, (numOrString) => {
+    if (isString(numOrString)) {
+      return numOrString.charCodeAt(0);
+    } else {
+      return numOrString;
+    }
+  });
+  return charCodes;
+}
+function addToMapOfArrays(map2, key, value) {
+  if (map2[key] === void 0) {
+    map2[key] = [value];
+  } else {
+    map2[key].push(value);
+  }
+}
+const minOptimizationVal = 256;
+let charCodeToOptimizedIdxMap = [];
+function charCodeToOptimizedIndex(charCode) {
+  return charCode < minOptimizationVal ? charCode : charCodeToOptimizedIdxMap[charCode];
+}
+function initCharCodeToOptimizedIndexMap() {
+  if (isEmpty(charCodeToOptimizedIdxMap)) {
+    charCodeToOptimizedIdxMap = new Array(65536);
+    for (let i = 0; i < 65536; i++) {
+      charCodeToOptimizedIdxMap[i] = i > 255 ? 255 + ~~(i / 255) : i;
+    }
+  }
+}
+export {
+  DEFAULT_MODE,
+  LineTerminatorOptimizedTester,
+  MODES,
+  SUPPORT_STICKY,
+  addStartOfInput,
+  addStickyFlag,
+  analyzeTokenTypes,
+  buildLineBreakIssueMessage,
+  charCodeToOptimizedIndex,
+  cloneEmptyGroups,
+  findDuplicatePatterns,
+  findEmptyMatchRegExps,
+  findEndOfInputAnchor,
+  findInvalidGroupType,
+  findInvalidPatterns,
+  findMissingPatterns,
+  findModesThatDoNotExist,
+  findStartOfInputAnchor,
+  findUnreachablePatterns,
+  findUnsupportedFlags,
+  isCustomPattern,
+  isShortPattern,
+  minOptimizationVal,
+  performRuntimeChecks,
+  performWarningRuntimeChecks,
+  validatePatterns
+};