axyseo 2.0.0-alpha.0.0.20 → 2.0.0-alpha.0.0.4

package/src/languageProcessing/languages/en/helpers/internal/getAdjectiveStem.js

@@ -0,0 +1,162 @@
+import {languageProcessing} from '../../../index';
+import {countSyllablesInWord} from '../../../../helpers/syllables/countSyllables';
+import syllables from '../../config/syllables.json';
+const {buildFormRule, createRulesFromArrays} = languageProcessing;
+
+/**
+ * Further checks if the word can be a specific adjectival form.
+ *
+ * @param {string} word		The word to check.
+ * @param {string} endsWith	The ending of the form.
+ * @param {string[]} nonStemmingExceptions		The list of words ending with the form that are not this form.
+ * @param {string[]} forcedStemmingExceptions	The list of words ending with the form that are not caught by the regular rules.
+ *
+ * @returns {boolean} Whether the word can be a specific adjectival form.
+ */
+const refineAdjectiveCheck = (word, endsWith, nonStemmingExceptions, forcedStemmingExceptions) => {
+  const countSyllables = countSyllablesInWord(word, syllables);
+  /*
+   * This check is based on the regular comparative and superlative formation rules.
+   * One syllable adjectives receive -er and -est. The syllable count of the comparative/superlative form is 2.
+   * One/two syllable adjectives ending in -e receive -r and -st. The syllable count of the comparative/superlative form is 2.
+   * Two syllable adjectives ending in -y receive -ier and -iest. The syllable count of the comparative/superlative form is 3.
+   * Words ending in -er/-est that don't follow the above rules are not comparatives/superlatives, unless listed in the exception lists.
+   */
+  if (word.endsWith(endsWith)) {
+    // Words with one syllable or less can't be comparatives/superlatives.
+    if (countSyllables <= 1) {
+      return false;
+    }
+    // The suffix for adjectives ending in -y is -ier/-iest.
+    const adjectiveEndingInYSuffix = `i${endsWith}`;
+    // Checks if the word is an adjective ending in -y that can have suffixes -er/-est. For example, 'pretty'.
+    const isStemEndingInY = word.endsWith(adjectiveEndingInYSuffix);
+    // Checks if the word is an adjective longer than one syllable that can have suffixes -er/-est (e.g. 'shallow'), OR
+    // if the word is an adjective ending in -y containing more than two syllables that can have suffixes -ier/-iest (e.g. 'unlikely').
+    const isForcedStemmingException =
+      forcedStemmingExceptions.includes(word.slice(0, -adjectiveEndingInYSuffix.length)) ||
+      forcedStemmingExceptions.includes(word.slice(0, -endsWith.length));
+    // Check if the word ending in -er/-est is not a comparative/superlative (e.g. 'banner').
+    const isNonStemmingException = nonStemmingExceptions.includes(word);
+    // The maximum syllable count for a word to be a comparative/superlative is 2, unless it ends in -y, then it's 3.
+    const maxSyllables = isStemEndingInY ? 3 : 2;
+
+    if (isForcedStemmingException || (countSyllables <= maxSyllables && !isNonStemmingException)) {
+      return true;
+    }
+  }
+  return false;
+};
+
+/**
+ * Checks if the word ends with a specific form and is not in the exceptions list.
+ *
+ * @param {string} word		The word to check.
+ * @param {string} endsWith	The ending of the form.
+ * @param {string[]} nonStemmingExceptions 	The list of words ending with the form that are not this form.
+ * @returns {boolean} Whether the word ends with the form and is not in the exceptions list.
+ */
+const endsWithAndNoException = (word, endsWith, nonStemmingExceptions) =>
+  word.endsWith(endsWith) && !nonStemmingExceptions.includes(word);
+
+/**
+ * Constructs a function that checks if the input word can be a specific adjectival form.
+ *
+ * @param {string}      endsWith            How the form ends.
+ * @param {int}         minimumWordLength   How long the word should be to classify for this form.
+ * @param {string[]}    [nonStemmingExceptions=[]]		The list of words with that ending (endsWith) which are not this form.
+ * @param {string[]}	[forcedStemmingExceptions=[]]	The list of words with that ending (endsWith) which are not caught by the regular rules.
+ * @param {Function}    [checkFunction=endsWithAndNoException]	An extra check to determine if the word is this form.
+ *
+ * @returns {Function} A function that checks if the input word can be a specific adjectival form.
+ */
+const constructCanBeFunction = function(
+  endsWith,
+  minimumWordLength,
+  nonStemmingExceptions = [],
+  forcedStemmingExceptions = [],
+  checkFunction = endsWithAndNoException
+) {
+  return word => {
+    const wordLength = word.length;
+
+    if (wordLength < minimumWordLength) {
+      return false;
+    }
+    return checkFunction(word, endsWith, nonStemmingExceptions, forcedStemmingExceptions);
+  };
+};
+
+/**
+ * Forms the base form from an input word.
+ *
+ * @param {string}   word                                  The word to build the base form for.
+ * @param {Object}   regexAdjective                        The lists of regexes to apply to stem adjectives.
+ * @param {Array}    regexAdjective.comparativeToBaseRegex The Array of regex-based rules to bring comparatives to base.
+ * @param {Array}    regexAdjective.superlativeToBaseRegex The Array of regex-based rules to bring superlatives to base.
+ * @param {Array}    regexAdjective.adverbToBaseRegex      The Array of regex-based rules to bring adverbs to base.
+ * @param {Object}   stopAdjectives                        The lists of words that are not adverbs.
+ * @param {string[]} stopAdjectives.erExceptions           The list of words that end with -er and are not comparatives.
+ * @param {string[]} stopAdjectives.estExceptions          The list of words that end with -est and are not superlatives.
+ * @param {string[]} stopAdjectives.lyExceptions           The list of words that end with -ly and are not adverbs.
+ * @param {string[]} multiSyllableAdjWithSuffixes          The list of adjectives containing more than 2 syllables that can have suffixes.
+ *
+ * @returns {Object} The base form of the input word.
+ */
+export default function(word, regexAdjective, stopAdjectives, multiSyllableAdjWithSuffixes) {
+  /*
+   * Check comparatives: Consider only words of four letters or more (otherwise, words like "per" are being treated
+   * as comparatives).
+   */
+  const canBeComparative = constructCanBeFunction(
+    'er',
+    4,
+    stopAdjectives.erExceptions,
+    multiSyllableAdjWithSuffixes,
+    refineAdjectiveCheck
+  );
+  if (canBeComparative(word)) {
+    const comparativeToBaseRegex = createRulesFromArrays(regexAdjective.comparativeToBase);
+    return {
+      base: buildFormRule(word, comparativeToBaseRegex) || word,
+      guessedForm: 'er'
+    };
+  }
+
+  /*
+   * Check superlatives: Consider only words of five letters or more (otherwise, words like "test" are being treated
+   * as superlatives).
+   */
+  const canBeSuperlative = constructCanBeFunction(
+    'est',
+    5,
+    stopAdjectives.estExceptions,
+    multiSyllableAdjWithSuffixes,
+    refineAdjectiveCheck
+  );
+  if (canBeSuperlative(word)) {
+    const superlativeToBaseRegex = createRulesFromArrays(regexAdjective.superlativeToBase);
+    return {
+      base: buildFormRule(word, superlativeToBaseRegex) || word,
+      guessedForm: 'est'
+    };
+  }
+
+  /*
+   * Check ly-adverbs: Consider only words of five letters or more (otherwise, words like "lily" are being treated
+   * as ly-adverbs).
+   */
+  const canBeLyAdverb = constructCanBeFunction('ly', 5, stopAdjectives.lyExceptions);
+  if (canBeLyAdverb(word)) {
+    const adverbToBaseRegex = createRulesFromArrays(regexAdjective.adverbToBase);
+    return {
+      base: buildFormRule(word, adverbToBaseRegex),
+      guessedForm: 'ly'
+    };
+  }
+
+  return {
+    base: word,
+    guessedForm: 'base'
+  };
+}

package/src/languageProcessing/languages/en/helpers/internal/getParticiples.js

@@ -0,0 +1,25 @@
+import {forEach, includes} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {matchRegularParticiples, getWords} = languageProcessing;
+
+import irregularParticiples from '../../config/internal/passiveVoiceIrregulars';
+
+/**
+ * Creates participle array for the participles found in a clause.
+ *
+ * @param {string} clauseText The clause to find participles in
+ *
+ * @returns {Array} The list with participles.
+ */
+export default function getParticiples(clauseText) {
+  const words = getWords(clauseText);
+  const foundParticiples = [];
+
+  forEach(words, function(word) {
+    const regex = [/\w+ed($|[ \n\r\t.,'()"+\-;!?:/»«‹›<>])/gi];
+    if (matchRegularParticiples(word, regex).length !== 0 || includes(irregularParticiples, word)) {
+      foundParticiples.push(word);
+    }
+  });
+  return foundParticiples;
+}

package/src/languageProcessing/languages/en/helpers/internal/getVerbStem.js

@@ -0,0 +1,237 @@
+// "use strict";
+import {isUndefined} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {buildFormRule, createRulesFromArrays} = languageProcessing;
+
+const vowelRegex = /([aeiouy])/g;
+
+/**
+ * Checks if the input word has one of the standard verb prefixes and if so returns a prefix and a de-prefixed verb to be
+ * further used to compare with the list of irregular verbs.
+ *
+ * @param {string} word             The word for which to determine if it has one of the standard verb prefixes.
+ * @param {Object} verbPrefixes     The collection of verb prefixes to be used for normalization
+ *
+ * @returns {Array} Array of word forms from the exception list.
+ */
+const normalizePrefixed = function(word, verbPrefixes) {
+  for (const property in verbPrefixes) {
+    if (verbPrefixes.hasOwnProperty) {
+      verbPrefixes[property] = new RegExp(verbPrefixes[property], 'i');
+    }
+  }
+
+  if (verbPrefixes.sevenLetterHyphenPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.sevenLetterHyphenPrefixes, ''),
+      prefix: word.substring(0, 8)
+    };
+  }
+
+  if (verbPrefixes.sevenLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.sevenLetterPrefixes, ''),
+      prefix: word.substring(0, 7)
+    };
+  }
+
+  if (verbPrefixes.fiveLetterHyphenPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.fiveLetterHyphenPrefixes, ''),
+      prefix: word.substring(0, 6)
+    };
+  }
+
+  if (verbPrefixes.fiveLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.fiveLetterPrefixes, ''),
+      prefix: word.substring(0, 5)
+    };
+  }
+
+  if (verbPrefixes.fourLetterHyphenPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.fourLetterHyphenPrefixes, ''),
+      prefix: word.substring(0, 5)
+    };
+  }
+
+  if (verbPrefixes.fourLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.fourLetterPrefixes, ''),
+      prefix: word.substring(0, 4)
+    };
+  }
+
+  if (verbPrefixes.threeLetterHyphenPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.threeLetterHyphenPrefixes, ''),
+      prefix: word.substring(0, 4)
+    };
+  }
+
+  if (verbPrefixes.threeLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.threeLetterPrefixes, ''),
+      prefix: word.substring(0, 3)
+    };
+  }
+
+  if (verbPrefixes.twoLetterHyphenPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.twoLetterHyphenPrefixes, ''),
+      prefix: word.substring(0, 3)
+    };
+  }
+
+  if (verbPrefixes.twoLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.twoLetterPrefixes, ''),
+      prefix: word.substring(0, 2)
+    };
+  }
+
+  if (verbPrefixes.oneLetterPrefixes.test(word) === true) {
+    return {
+      normalizedWord: word.replace(verbPrefixes.oneLetterPrefixes, ''),
+      prefix: word.substring(0, 1)
+    };
+  }
+};
+
+/**
+ * Checks if the input word occurs in the list of exception verbs and if so returns all its irregular forms.
+ * If not checks if it is an irregular verb with one of the standard verb prefixes, if so returns all irregular prefixed forms.
+ *
+ * @param {string} word             The word for which to determine its irregular forms.
+ * @param {Array} irregularVerbs    The array of irregular verbs available for this language.
+ * @param {Object} verbPrefixes     The collection of verb prefixes to be used for normalization of irregular verbs.
+ *
+ * @returns {Array} Array of word forms from the exception list.
+ */
+const checkIrregulars = function(word, irregularVerbs, verbPrefixes) {
+  let irregulars;
+
+  irregularVerbs.forEach(function(paradigm) {
+    paradigm.forEach(function(wordInParadigm) {
+      if (wordInParadigm === word) {
+        irregulars = paradigm;
+      }
+    });
+  });
+
+  if (isUndefined(irregulars)) {
+    const normalizedIrregular = normalizePrefixed(word, verbPrefixes);
+
+    if (!isUndefined(normalizedIrregular)) {
+      irregularVerbs.forEach(function(paradigm) {
+        paradigm.forEach(function(wordInParadigm) {
+          if (wordInParadigm === normalizedIrregular.normalizedWord) {
+            irregulars = paradigm.map(function(verb) {
+              return normalizedIrregular.prefix.concat(verb);
+            });
+          }
+        });
+      });
+    }
+  }
+
+  return irregulars;
+};
+
+/**
+ * Checks if the input word ends with "s".
+ *
+ * @param {string} word The word to check.
+ *
+ * @returns {boolean} True if the word ends with "s".
+ */
+const endsWithS = function(word) {
+  const wordLength = word.length;
+  // Consider only words of four letters or more to be s-forms (otherwise, words like "its" are being treated as verb forms).
+  if (wordLength > 3) {
+    return word[word.length - 1] === 's';
+  }
+  return false;
+};
+
+/**
+ * Checks if the input word ends with "ing".
+ *
+ * @param {string} word The word to check.
+ *
+ * @returns {boolean} True if the word ends with "ing".
+ */
+const endsWithIng = function(word) {
+  const vowelCount = (word.match(vowelRegex) || []).length;
+
+  // Consider only words that have at least one more vowel besides "i" in "ing" (otherwise, words like "ping" are being treated as verb forms).
+  if (vowelCount > 1 && word.length > 4) {
+    return word.substring(word.length - 3, word.length) === 'ing';
+  }
+  return false;
+};
+
+/**
+ * Checks if the input word ends with "ed".
+ *
+ * @param {string} word The word to check.
+ *
+ * @returns {boolean} True if the word ends with "ed".
+ */
+const endsWithEd = function(word) {
+  const vowelCount = (word.match(vowelRegex) || []).length;
+
+  // Consider only words that have at least one more vowel besides "e" in "ed" (otherwise, words like "red" are being treated as verb forms).
+  if (
+    vowelCount > 1 ||
+    (vowelCount === 1 && word.substring(word.length - 3, word.length - 2) !== 'e')
+  ) {
+    return word.substring(word.length - 2, word.length) === 'ed';
+  }
+  return false;
+};
+
+/**
+ * Forms the infinitive from an input word.
+ *
+ * @param {string} word                               The word to build the infinitive for.
+ * @param {Object} regexVerb                          The list of regex rules used to bring verb forms to infinitive.
+ * @param {Array}  regexVerb.sFormToInfinitiveRegex   The array of regex-based rules used to bring -s forms to infinitive.
+ * @param {Array}  regexVerb.ingFormToInfinitiveRegex The array of regex-based rules used to bring -ing forms to infinitive.
+ * @param {Array}  regexVerb.edFormToInfinitiveRegex  The array of regex-based rules used to bring -ed forms to infinitive.
+ *
+ * @returns {Object} The infinitive of the input word.
+ */
+const getInfinitive = function(word, regexVerb) {
+  const sFormToInfinitiveRegex = createRulesFromArrays(regexVerb.sFormToInfinitive);
+  const ingFormToInfinitiveRegex = createRulesFromArrays(regexVerb.ingFormToInfinitive);
+  const edFormToInfinitiveRegex = createRulesFromArrays(regexVerb.edFormToInfinitive);
+
+  if (endsWithS(word)) {
+    return {
+      infinitive: buildFormRule(word, sFormToInfinitiveRegex),
+      guessedForm: 's'
+    };
+  }
+
+  if (endsWithIng(word)) {
+    return {
+      infinitive: buildFormRule(word, ingFormToInfinitiveRegex),
+      guessedForm: 'ing'
+    };
+  }
+
+  if (endsWithEd(word)) {
+    return {
+      infinitive: buildFormRule(word, edFormToInfinitiveRegex) || word,
+      guessedForm: 'ed'
+    };
+  }
+  return {
+    infinitive: word,
+    guessedForm: 'inf'
+  };
+};
+
+export {getInfinitive, checkIrregulars, endsWithIng, normalizePrefixed};

package/src/languageProcessing/languages/en/values/Clause.js

@@ -0,0 +1,68 @@
+import {includes, intersection, isEmpty} from 'lodash';
+import {
+  cannotBeBetweenPassiveAuxiliaryAndParticiple,
+  cannotDirectlyPrecedePassiveParticiple
+} from '../config/functionWords';
+import nonVerbsEndingEd from '../config/internal/passiveVoiceNonVerbEndingEd';
+import getParticiples from '../helpers/internal/getParticiples';
+import {languageProcessing} from '../../../index';
+const {precedenceException, directPrecedenceException, values} = languageProcessing;
+const Clause = values.Clause;
+
+/**
+ * Creates a Clause object for the English language.
+ */
+class EnglishClause extends Clause {
+  /**
+   * Constructor.
+   *
+   * @param {string} clauseText   The text of the clause.
+   * @param {Array} auxiliaries   The auxiliaries.
+   *
+   * @constructor
+   */
+  constructor(clauseText, auxiliaries) {
+    super(clauseText, auxiliaries);
+    this._participles = getParticiples(this.getClauseText());
+    this.checkParticiples();
+  }
+
+  /**
+   * Checks if any exceptions are applicable to this participle that would result in the clause not being passive.
+   * If no exceptions are found, the clause is passive.
+   *
+   * @returns {boolean} Returns true if no exception is found.
+   */
+  checkParticiples() {
+    const clause = this.getClauseText();
+
+    const passiveParticiples = this.getParticiples().filter(
+      participle =>
+        !includes(nonVerbsEndingEd, participle) &&
+        !this.hasRidException(participle) &&
+        !directPrecedenceException(clause, participle, cannotDirectlyPrecedePassiveParticiple) &&
+        !precedenceException(clause, participle, cannotBeBetweenPassiveAuxiliaryAndParticiple)
+    );
+
+    this.setPassive(passiveParticiples.length > 0);
+  }
+
+  /**
+   * Checks whether the participle is 'rid' in combination with 'get', 'gets', 'getting', 'got' or 'gotten'.
+   * If this is true, the participle is not passive.
+   *
+   * @param {string} participle   The participle
+   *
+   * @returns {boolean} Returns true if 'rid' is found in combination with a form of 'get'
+   * otherwise returns false.
+   */
+  hasRidException(participle) {
+    if (participle === 'rid') {
+      const irregularExclusionArray = ['get', 'gets', 'getting', 'got', 'gotten'];
+      return !isEmpty(intersection(irregularExclusionArray, this.getAuxiliaries()));
+    }
+    return false;
+  }
+}
+
+export default EnglishClause;

package/src/languageProcessing/languages/es/Researcher.js

@@ -0,0 +1,48 @@
+import {languageProcessing} from '../../../index';
+const {AbstractResearcher} = languageProcessing;
+
+// All config
+import firstWordExceptions from './config/firstWordExceptions';
+import {all as functionWords} from './config/functionWords';
+import stopWords from './config/stopWords';
+import transitionWords from './config/transitionWords';
+import twoPartTransitionWords from './config/twoPartTransitionWords';
+import syllables from './config/syllables.json';
+import sentenceLength from './config/sentenceLength';
+
+// All helpers
+import getClauses from './helpers/getClauses';
+import getStemmer from './helpers/getStemmer';
+import fleschReadingScore from './helpers/calculateFleschReadingScore';
+
+/**
+ * The researcher contains all the researches, helpers, data, and config.
+ */
+export default class Researcher extends AbstractResearcher {
+  /**
+   * Constructor
+   * @param {Paper} paper The Paper object that is needed within the researches.
+   * @constructor
+   */
+  constructor(paper) {
+    super(paper);
+
+    Object.assign(this.config, {
+      language: 'es',
+      passiveConstructionType: 'periphrastic',
+      firstWordExceptions,
+      functionWords,
+      stopWords,
+      transitionWords,
+      twoPartTransitionWords,
+      syllables,
+      sentenceLength
+    });
+
+    Object.assign(this.helpers, {
+      getClauses,
+      getStemmer,
+      fleschReadingScore
+    });
+  }
+}

package/src/languageProcessing/languages/es/config/firstWordExceptions.js

@@ -0,0 +1,16 @@
+/**
+ * Returns an array with exceptions for the sentence beginning researcher.
+ * @returns {Array} The array filled with exceptions.
+ */
+export default [
+	// Definite articles:
+	"el", "los", "la", "las",
+	// Indefinite articles:
+	"un", "una", "unas", "unos",
+	// Numbers 1-10:
+	"uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez",
+	// Demonstrative pronouns:
+	"este", "estos", "esta", "estas", "ese", "esos", "esa", "esas", "aquel",
+	"aquellos", "aquella", "aquellas", "esto", "eso", "aquello" ];
+
+