@clear-capabilities/agentic-security-scanner 0.149.1 → 0.149.4

package/vendor/java-parser/node_modules/chevrotain/src/parse/grammar/checks.ts

@@ -0,0 +1,712 @@
+import {
+  clone,
+  compact,
+  difference,
+  drop,
+  dropRight,
+  filter,
+  first,
+  flatMap,
+  flatten,
+  forEach,
+  groupBy,
+  includes,
+  isEmpty,
+  map,
+  pickBy,
+  reduce,
+  reject,
+  values,
+} from "lodash-es";
+import {
+  IParserAmbiguousAlternativesDefinitionError,
+  IParserDuplicatesDefinitionError,
+  IParserEmptyAlternativeDefinitionError,
+  ParserDefinitionErrorType,
+} from "../parser/parser.js";
+import {
+  Alternation,
+  Alternative as AlternativeGAST,
+  GAstVisitor,
+  getProductionDslName,
+  isOptionalProd,
+  NonTerminal,
+  Option,
+  Repetition,
+  RepetitionMandatory,
+  RepetitionMandatoryWithSeparator,
+  RepetitionWithSeparator,
+  Terminal,
+} from "@chevrotain/gast";
+import {
+  Alternative,
+  containsPath,
+  getLookaheadPathsForOptionalProd,
+  getLookaheadPathsForOr,
+  getProdType,
+  isStrictPrefixOfPath,
+} from "./lookahead.js";
+import { nextPossibleTokensAfter } from "./interpreter.js";
+import {
+  ILookaheadStrategy,
+  IProduction,
+  IProductionWithOccurrence,
+  Rule,
+  TokenType,
+} from "@chevrotain/types";
+import {
+  IGrammarValidatorErrorMessageProvider,
+  IParserDefinitionError,
+} from "./types.js";
+import { tokenStructuredMatcher } from "../../scan/tokens.js";
+
+export function validateLookahead(options: {
+  lookaheadStrategy: ILookaheadStrategy;
+  rules: Rule[];
+  tokenTypes: TokenType[];
+  grammarName: string;
+}): IParserDefinitionError[] {
+  const lookaheadValidationErrorMessages = options.lookaheadStrategy.validate({
+    rules: options.rules,
+    tokenTypes: options.tokenTypes,
+    grammarName: options.grammarName,
+  });
+  return map(lookaheadValidationErrorMessages, (errorMessage) => ({
+    type: ParserDefinitionErrorType.CUSTOM_LOOKAHEAD_VALIDATION,
+    ...errorMessage,
+  }));
+}
+
+export function validateGrammar(
+  topLevels: Rule[],
+  tokenTypes: TokenType[],
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+  grammarName: string,
+): IParserDefinitionError[] {
+  const duplicateErrors: IParserDefinitionError[] = flatMap(
+    topLevels,
+    (currTopLevel) =>
+      validateDuplicateProductions(currTopLevel, errMsgProvider),
+  );
+
+  const termsNamespaceConflictErrors = checkTerminalAndNoneTerminalsNameSpace(
+    topLevels,
+    tokenTypes,
+    errMsgProvider,
+  );
+
+  const tooManyAltsErrors = flatMap(topLevels, (curRule) =>
+    validateTooManyAlts(curRule, errMsgProvider),
+  );
+
+  const duplicateRulesError = flatMap(topLevels, (curRule) =>
+    validateRuleDoesNotAlreadyExist(
+      curRule,
+      topLevels,
+      grammarName,
+      errMsgProvider,
+    ),
+  );
+
+  return duplicateErrors.concat(
+    termsNamespaceConflictErrors,
+    tooManyAltsErrors,
+    duplicateRulesError,
+  );
+}
+
+function validateDuplicateProductions(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserDuplicatesDefinitionError[] {
+  const collectorVisitor = new OccurrenceValidationCollector();
+  topLevelRule.accept(collectorVisitor);
+  const allRuleProductions = collectorVisitor.allProductions;
+
+  const productionGroups = groupBy(
+    allRuleProductions,
+    identifyProductionForDuplicates,
+  );
+
+  const duplicates: any = pickBy(productionGroups, (currGroup) => {
+    return currGroup.length > 1;
+  });
+
+  const errors = map(values(duplicates), (currDuplicates: any) => {
+    const firstProd: any = first(currDuplicates);
+    const msg = errMsgProvider.buildDuplicateFoundError(
+      topLevelRule,
+      currDuplicates,
+    );
+    const dslName = getProductionDslName(firstProd);
+    const defError: IParserDuplicatesDefinitionError = {
+      message: msg,
+      type: ParserDefinitionErrorType.DUPLICATE_PRODUCTIONS,
+      ruleName: topLevelRule.name,
+      dslName: dslName,
+      occurrence: firstProd.idx,
+    };
+
+    const param = getExtraProductionArgument(firstProd);
+    if (param) {
+      defError.parameter = param;
+    }
+
+    return defError;
+  });
+  return errors;
+}
+
+export function identifyProductionForDuplicates(
+  prod: IProductionWithOccurrence,
+): string {
+  return `${getProductionDslName(prod)}_#_${
+    prod.idx
+  }_#_${getExtraProductionArgument(prod)}`;
+}
+
+function getExtraProductionArgument(prod: IProductionWithOccurrence): string {
+  if (prod instanceof Terminal) {
+    return prod.terminalType.name;
+  } else if (prod instanceof NonTerminal) {
+    return prod.nonTerminalName;
+  } else {
+    return "";
+  }
+}
+
+export class OccurrenceValidationCollector extends GAstVisitor {
+  public allProductions: IProductionWithOccurrence[] = [];
+
+  public visitNonTerminal(subrule: NonTerminal): void {
+    this.allProductions.push(subrule);
+  }
+
+  public visitOption(option: Option): void {
+    this.allProductions.push(option);
+  }
+
+  public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
+    this.allProductions.push(manySep);
+  }
+
+  public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
+    this.allProductions.push(atLeastOne);
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    atLeastOneSep: RepetitionMandatoryWithSeparator,
+  ): void {
+    this.allProductions.push(atLeastOneSep);
+  }
+
+  public visitRepetition(many: Repetition): void {
+    this.allProductions.push(many);
+  }
+
+  public visitAlternation(or: Alternation): void {
+    this.allProductions.push(or);
+  }
+
+  public visitTerminal(terminal: Terminal): void {
+    this.allProductions.push(terminal);
+  }
+}
+
+export function validateRuleDoesNotAlreadyExist(
+  rule: Rule,
+  allRules: Rule[],
+  className: string,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserDefinitionError[] {
+  const errors = [];
+  const occurrences = reduce(
+    allRules,
+    (result, curRule) => {
+      if (curRule.name === rule.name) {
+        return result + 1;
+      }
+      return result;
+    },
+    0,
+  );
+  if (occurrences > 1) {
+    const errMsg = errMsgProvider.buildDuplicateRuleNameError({
+      topLevelRule: rule,
+      grammarName: className,
+    });
+    errors.push({
+      message: errMsg,
+      type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
+      ruleName: rule.name,
+    });
+  }
+
+  return errors;
+}
+
+// TODO: is there anyway to get only the rule names of rules inherited from the super grammars?
+// This is not part of the IGrammarErrorProvider because the validation cannot be performed on
+// The grammar structure, only at runtime.
+export function validateRuleIsOverridden(
+  ruleName: string,
+  definedRulesNames: string[],
+  className: string,
+): IParserDefinitionError[] {
+  const errors = [];
+  let errMsg;
+
+  if (!includes(definedRulesNames, ruleName)) {
+    errMsg =
+      `Invalid rule override, rule: ->${ruleName}<- cannot be overridden in the grammar: ->${className}<-` +
+      `as it is not defined in any of the super grammars `;
+    errors.push({
+      message: errMsg,
+      type: ParserDefinitionErrorType.INVALID_RULE_OVERRIDE,
+      ruleName: ruleName,
+    });
+  }
+
+  return errors;
+}
+
+export function validateNoLeftRecursion(
+  topRule: Rule,
+  currRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+  path: Rule[] = [],
+): IParserDefinitionError[] {
+  const errors: IParserDefinitionError[] = [];
+  const nextNonTerminals = getFirstNoneTerminal(currRule.definition);
+  if (isEmpty(nextNonTerminals)) {
+    return [];
+  } else {
+    const ruleName = topRule.name;
+    const foundLeftRecursion = includes(nextNonTerminals, topRule);
+    if (foundLeftRecursion) {
+      errors.push({
+        message: errMsgProvider.buildLeftRecursionError({
+          topLevelRule: topRule,
+          leftRecursionPath: path,
+        }),
+        type: ParserDefinitionErrorType.LEFT_RECURSION,
+        ruleName: ruleName,
+      });
+    }
+
+    // we are only looking for cyclic paths leading back to the specific topRule
+    // other cyclic paths are ignored, we still need this difference to avoid infinite loops...
+    const validNextSteps = difference(nextNonTerminals, path.concat([topRule]));
+    const errorsFromNextSteps = flatMap(validNextSteps, (currRefRule) => {
+      const newPath = clone(path);
+      newPath.push(currRefRule);
+      return validateNoLeftRecursion(
+        topRule,
+        currRefRule,
+        errMsgProvider,
+        newPath,
+      );
+    });
+
+    return errors.concat(errorsFromNextSteps);
+  }
+}
+
+export function getFirstNoneTerminal(definition: IProduction[]): Rule[] {
+  let result: Rule[] = [];
+  if (isEmpty(definition)) {
+    return result;
+  }
+  const firstProd = first(definition);
+
+  /* istanbul ignore else */
+  if (firstProd instanceof NonTerminal) {
+    result.push(firstProd.referencedRule);
+  } else if (
+    firstProd instanceof AlternativeGAST ||
+    firstProd instanceof Option ||
+    firstProd instanceof RepetitionMandatory ||
+    firstProd instanceof RepetitionMandatoryWithSeparator ||
+    firstProd instanceof RepetitionWithSeparator ||
+    firstProd instanceof Repetition
+  ) {
+    result = result.concat(
+      getFirstNoneTerminal(<IProduction[]>firstProd.definition),
+    );
+  } else if (firstProd instanceof Alternation) {
+    // each sub definition in alternation is a FLAT
+    result = flatten(
+      map(firstProd.definition, (currSubDef) =>
+        getFirstNoneTerminal((<AlternativeGAST>currSubDef).definition),
+      ),
+    );
+  } else if (firstProd instanceof Terminal) {
+    // nothing to see, move along
+  } else {
+    throw Error("non exhaustive match");
+  }
+
+  const isFirstOptional = isOptionalProd(firstProd);
+  const hasMore = definition.length > 1;
+  if (isFirstOptional && hasMore) {
+    const rest = drop(definition);
+    return result.concat(getFirstNoneTerminal(rest));
+  } else {
+    return result;
+  }
+}
+
+class OrCollector extends GAstVisitor {
+  public alternations: Alternation[] = [];
+
+  public visitAlternation(node: Alternation): void {
+    this.alternations.push(node);
+  }
+}
+
+export function validateEmptyOrAlternative(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserEmptyAlternativeDefinitionError[] {
+  const orCollector = new OrCollector();
+  topLevelRule.accept(orCollector);
+  const ors = orCollector.alternations;
+
+  const errors = flatMap<Alternation, IParserEmptyAlternativeDefinitionError>(
+    ors,
+    (currOr) => {
+      const exceptLast = dropRight(currOr.definition);
+      return flatMap(exceptLast, (currAlternative, currAltIdx) => {
+        const possibleFirstInAlt = nextPossibleTokensAfter(
+          [currAlternative],
+          [],
+          tokenStructuredMatcher,
+          1,
+        );
+        if (isEmpty(possibleFirstInAlt)) {
+          return [
+            {
+              message: errMsgProvider.buildEmptyAlternationError({
+                topLevelRule: topLevelRule,
+                alternation: currOr,
+                emptyChoiceIdx: currAltIdx,
+              }),
+              type: ParserDefinitionErrorType.NONE_LAST_EMPTY_ALT,
+              ruleName: topLevelRule.name,
+              occurrence: currOr.idx,
+              alternative: currAltIdx + 1,
+            },
+          ];
+        } else {
+          return [];
+        }
+      });
+    },
+  );
+
+  return errors;
+}
+
+export function validateAmbiguousAlternationAlternatives(
+  topLevelRule: Rule,
+  globalMaxLookahead: number,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserAmbiguousAlternativesDefinitionError[] {
+  const orCollector = new OrCollector();
+  topLevelRule.accept(orCollector);
+  let ors = orCollector.alternations;
+
+  // New Handling of ignoring ambiguities
+  // - https://github.com/chevrotain/chevrotain/issues/869
+  ors = reject(ors, (currOr) => currOr.ignoreAmbiguities === true);
+
+  const errors = flatMap(ors, (currOr: Alternation) => {
+    const currOccurrence = currOr.idx;
+    const actualMaxLookahead = currOr.maxLookahead || globalMaxLookahead;
+    const alternatives = getLookaheadPathsForOr(
+      currOccurrence,
+      topLevelRule,
+      actualMaxLookahead,
+      currOr,
+    );
+    const altsAmbiguityErrors = checkAlternativesAmbiguities(
+      alternatives,
+      currOr,
+      topLevelRule,
+      errMsgProvider,
+    );
+    const altsPrefixAmbiguityErrors = checkPrefixAlternativesAmbiguities(
+      alternatives,
+      currOr,
+      topLevelRule,
+      errMsgProvider,
+    );
+
+    return altsAmbiguityErrors.concat(altsPrefixAmbiguityErrors);
+  });
+
+  return errors;
+}
+
+export class RepetitionCollector extends GAstVisitor {
+  public allProductions: (IProductionWithOccurrence & {
+    maxLookahead?: number;
+  })[] = [];
+
+  public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void {
+    this.allProductions.push(manySep);
+  }
+
+  public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void {
+    this.allProductions.push(atLeastOne);
+  }
+
+  public visitRepetitionMandatoryWithSeparator(
+    atLeastOneSep: RepetitionMandatoryWithSeparator,
+  ): void {
+    this.allProductions.push(atLeastOneSep);
+  }
+
+  public visitRepetition(many: Repetition): void {
+    this.allProductions.push(many);
+  }
+}
+
+export function validateTooManyAlts(
+  topLevelRule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserDefinitionError[] {
+  const orCollector = new OrCollector();
+  topLevelRule.accept(orCollector);
+  const ors = orCollector.alternations;
+
+  const errors = flatMap(ors, (currOr) => {
+    if (currOr.definition.length > 255) {
+      return [
+        {
+          message: errMsgProvider.buildTooManyAlternativesError({
+            topLevelRule: topLevelRule,
+            alternation: currOr,
+          }),
+          type: ParserDefinitionErrorType.TOO_MANY_ALTS,
+          ruleName: topLevelRule.name,
+          occurrence: currOr.idx,
+        },
+      ];
+    } else {
+      return [];
+    }
+  });
+
+  return errors;
+}
+
+export function validateSomeNonEmptyLookaheadPath(
+  topLevelRules: Rule[],
+  maxLookahead: number,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserDefinitionError[] {
+  const errors: IParserDefinitionError[] = [];
+  forEach(topLevelRules, (currTopRule) => {
+    const collectorVisitor = new RepetitionCollector();
+    currTopRule.accept(collectorVisitor);
+    const allRuleProductions = collectorVisitor.allProductions;
+    forEach(allRuleProductions, (currProd) => {
+      const prodType = getProdType(currProd);
+      const actualMaxLookahead = currProd.maxLookahead || maxLookahead;
+      const currOccurrence = currProd.idx;
+      const paths = getLookaheadPathsForOptionalProd(
+        currOccurrence,
+        currTopRule,
+        prodType,
+        actualMaxLookahead,
+      );
+      const pathsInsideProduction = paths[0];
+      if (isEmpty(flatten(pathsInsideProduction))) {
+        const errMsg = errMsgProvider.buildEmptyRepetitionError({
+          topLevelRule: currTopRule,
+          repetition: currProd,
+        });
+        errors.push({
+          message: errMsg,
+          type: ParserDefinitionErrorType.NO_NON_EMPTY_LOOKAHEAD,
+          ruleName: currTopRule.name,
+        });
+      }
+    });
+  });
+
+  return errors;
+}
+
+export interface IAmbiguityDescriptor {
+  alts: number[];
+  path: TokenType[];
+}
+
+function checkAlternativesAmbiguities(
+  alternatives: Alternative[],
+  alternation: Alternation,
+  rule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserAmbiguousAlternativesDefinitionError[] {
+  const foundAmbiguousPaths: Alternative = [];
+  const identicalAmbiguities = reduce(
+    alternatives,
+    (result, currAlt, currAltIdx) => {
+      // ignore (skip) ambiguities with this alternative
+      if (alternation.definition[currAltIdx].ignoreAmbiguities === true) {
+        return result;
+      }
+
+      forEach(currAlt, (currPath) => {
+        const altsCurrPathAppearsIn = [currAltIdx];
+        forEach(alternatives, (currOtherAlt, currOtherAltIdx) => {
+          if (
+            currAltIdx !== currOtherAltIdx &&
+            containsPath(currOtherAlt, currPath) &&
+            // ignore (skip) ambiguities with this "other" alternative
+            alternation.definition[currOtherAltIdx].ignoreAmbiguities !== true
+          ) {
+            altsCurrPathAppearsIn.push(currOtherAltIdx);
+          }
+        });
+
+        if (
+          altsCurrPathAppearsIn.length > 1 &&
+          !containsPath(foundAmbiguousPaths, currPath)
+        ) {
+          foundAmbiguousPaths.push(currPath);
+          result.push({
+            alts: altsCurrPathAppearsIn,
+            path: currPath,
+          });
+        }
+      });
+      return result;
+    },
+    [] as { alts: number[]; path: TokenType[] }[],
+  );
+
+  const currErrors = map(identicalAmbiguities, (currAmbDescriptor) => {
+    const ambgIndices = map(
+      currAmbDescriptor.alts,
+      (currAltIdx) => currAltIdx + 1,
+    );
+
+    const currMessage = errMsgProvider.buildAlternationAmbiguityError({
+      topLevelRule: rule,
+      alternation: alternation,
+      ambiguityIndices: ambgIndices,
+      prefixPath: currAmbDescriptor.path,
+    });
+
+    return {
+      message: currMessage,
+      type: ParserDefinitionErrorType.AMBIGUOUS_ALTS,
+      ruleName: rule.name,
+      occurrence: alternation.idx,
+      alternatives: currAmbDescriptor.alts,
+    };
+  });
+
+  return currErrors;
+}
+
+export function checkPrefixAlternativesAmbiguities(
+  alternatives: Alternative[],
+  alternation: Alternation,
+  rule: Rule,
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserAmbiguousAlternativesDefinitionError[] {
+  // flatten
+  const pathsAndIndices = reduce(
+    alternatives,
+    (result, currAlt, idx) => {
+      const currPathsAndIdx = map(currAlt, (currPath) => {
+        return { idx: idx, path: currPath };
+      });
+      return result.concat(currPathsAndIdx);
+    },
+    [] as { idx: number; path: TokenType[] }[],
+  );
+
+  const errors = compact(
+    flatMap(pathsAndIndices, (currPathAndIdx) => {
+      const alternativeGast = alternation.definition[currPathAndIdx.idx];
+      // ignore (skip) ambiguities with this alternative
+      if (alternativeGast.ignoreAmbiguities === true) {
+        return [];
+      }
+      const targetIdx = currPathAndIdx.idx;
+      const targetPath = currPathAndIdx.path;
+
+      const prefixAmbiguitiesPathsAndIndices = filter(
+        pathsAndIndices,
+        (searchPathAndIdx) => {
+          // prefix ambiguity can only be created from lower idx (higher priority) path
+          return (
+            // ignore (skip) ambiguities with this "other" alternative
+            alternation.definition[searchPathAndIdx.idx].ignoreAmbiguities !==
+              true &&
+            searchPathAndIdx.idx < targetIdx &&
+            // checking for strict prefix because identical lookaheads
+            // will be be detected using a different validation.
+            isStrictPrefixOfPath(searchPathAndIdx.path, targetPath)
+          );
+        },
+      );
+
+      const currPathPrefixErrors = map(
+        prefixAmbiguitiesPathsAndIndices,
+        (currAmbPathAndIdx): IParserAmbiguousAlternativesDefinitionError => {
+          const ambgIndices = [currAmbPathAndIdx.idx + 1, targetIdx + 1];
+          const occurrence = alternation.idx === 0 ? "" : alternation.idx;
+
+          const message = errMsgProvider.buildAlternationPrefixAmbiguityError({
+            topLevelRule: rule,
+            alternation: alternation,
+            ambiguityIndices: ambgIndices,
+            prefixPath: currAmbPathAndIdx.path,
+          });
+          return {
+            message: message,
+            type: ParserDefinitionErrorType.AMBIGUOUS_PREFIX_ALTS,
+            ruleName: rule.name,
+            occurrence: occurrence,
+            alternatives: ambgIndices,
+          };
+        },
+      );
+
+      return currPathPrefixErrors;
+    }),
+  );
+
+  return errors;
+}
+
+function checkTerminalAndNoneTerminalsNameSpace(
+  topLevels: Rule[],
+  tokenTypes: TokenType[],
+  errMsgProvider: IGrammarValidatorErrorMessageProvider,
+): IParserDefinitionError[] {
+  const errors: IParserDefinitionError[] = [];
+
+  const tokenNames = map(tokenTypes, (currToken) => currToken.name);
+
+  forEach(topLevels, (currRule) => {
+    const currRuleName = currRule.name;
+    if (includes(tokenNames, currRuleName)) {
+      const errMsg = errMsgProvider.buildNamespaceConflictError(currRule);
+
+      errors.push({
+        message: errMsg,
+        type: ParserDefinitionErrorType.CONFLICT_TOKENS_RULES_NAMESPACE,
+        ruleName: currRuleName,
+      });
+    }
+  });
+
+  return errors;
+}