axyseo 2.0.0-alpha.0.0.20 → 2.0.0-alpha.0.0.4

package/src/languageProcessing/languages/en/config/regularParticiplesRegex.js

@@ -0,0 +1,5 @@
+/**
+ * A Regex that recognizes regular participles. NOTE: this often coincides with simple past forms.
+ * @type {RegExp}
+ */
+export const regularParticiplesRegex = /\w+ed($|[ \n\r\t.,'()"+\-;!?:/»«‹›<>])/ig;

package/src/languageProcessing/languages/en/config/stopWords.js

@@ -0,0 +1,52 @@
+/**
+ * Returns a list with stopwords for the English passive voice assessment.
+ * @type {Array} The list with stopwords.
+ */
+export default [
+	"to",
+	"which",
+	"who",
+	"whom",
+	"that",
+	"whose",
+	"after",
+	"although",
+	"as",
+	"because",
+	"before",
+	"even if",
+	"even though",
+	"if",
+	"in order that",
+	"inasmuch",
+	"lest",
+	"once",
+	"provided",
+	"since",
+	"so that",
+	"than",
+	"though",
+	"till",
+	"unless",
+	"until",
+	"when",
+	"whenever",
+	"where",
+	"whereas",
+	"wherever",
+	"whether",
+	"while",
+	"why",
+	"by the time",
+	"supposing",
+	"no matter",
+	"how",
+	"what",
+	"won't",
+	"do",
+	"does",
+	"–",
+	"and",
+	"but",
+	"or",
+];

package/src/languageProcessing/languages/en/config/syllables.json

@@ -0,0 +1,86 @@
+{
+	"vowels": "aeiouy",
+	"deviations": {
+		"vowels": [
+			{
+				"fragments": [ "cial", "tia", "cius", "giu", "ion",
+					"[^bdnprv]iou", "sia$", "[^aeiuot]{2,}ed$", "[aeiouy][^aeiuoyts]{1,}e$",
+					"[a-z]ely$", "[cgy]ed$", "rved$", "[aeiouy][dt]es?$", "eau", "ieu",
+					"oeu", "[aeiouy][^aeiouydt]e[sd]?$", "[aeouy]rse$", "^eye" ],
+				"countModifier": -1
+			},
+			{
+				"fragments": [ "ia", "iu", "ii", "io", "[aeio][aeiou]{2}", "[aeiou]ing", "[^aeiou]ying", "ui[aeou]" ],
+				"countModifier": 1
+			},
+			{
+				"fragments": [ "^ree[jmnpqrsx]", "^reele", "^reeva", "riet",
+					"dien", "[aeiouym][bdp]le$", "uei", "uou",
+					"^mc", "ism$", "[^l]lien", "^coa[dglx].",
+					"[^gqauieo]ua[^auieo]", "dn't$", "uity$", "ie(r|st)",
+					"[aeiouw]y[aeiou]", "[^ao]ire[ds]", "[^ao]ire$" ],
+				"countModifier": 1
+			},
+			{
+				"fragments": [ "eoa", "eoo", "ioa", "ioe", "ioo" ],
+				"countModifier": 1
+			}
+		],
+		"words": {
+			"full": [
+				{
+					"word": "business",
+					"syllables": 2
+				},
+				{
+					"word": "coheiress",
+					"syllables": 3
+				},
+				{
+					"word": "colonel",
+					"syllables": 2
+				},
+				{
+					"word": "heiress",
+					"syllables": 2
+				},
+				{
+					"word": "i.e",
+					"syllables": 2
+				},
+				{
+					"word": "shoreline",
+					"syllables": 2
+				},
+				{
+					"word": "simile",
+					"syllables": 3
+				},
+				{
+					"word": "unheired",
+					"syllables": 2
+				},
+				{
+					"word": "wednesday",
+					"syllables": 2
+				}
+			],
+			"fragments": {
+				"global": [
+					{
+						"word": "coyote",
+						"syllables": 3
+					},
+					{
+						"word": "graveyard",
+						"syllables": 2
+					},
+					{
+						"word": "lawyer",
+						"syllables": 2
+					}
+				]
+			}
+		}
+	}
+}

package/src/languageProcessing/languages/en/config/transitionWords.js

@@ -0,0 +1,48 @@
+/** @module config/transitionWords */
+
+export const singleWords = [ "accordingly", "additionally", "afterward", "afterwards", "albeit", "also", "although", "altogether",
+	"another", "basically", "because", "before", "besides", "but", "certainly", "chiefly", "comparatively",
+	"concurrently", "consequently", "contrarily", "conversely", "correspondingly", "despite", "doubtedly", "during",
+	"e.g.", "earlier", "emphatically", "equally", "especially", "eventually", "evidently", "explicitly", "finally",
+	"firstly", "following", "formerly", "forthwith", "fourthly", "further", "furthermore", "generally", "hence",
+	"henceforth", "however", "i.e.", "identically", "indeed", "initially", "instead", "last", "lastly", "later", "lest",
+	"likewise", "markedly", "meanwhile", "moreover", "nevertheless", "nonetheless", "nor",  "notwithstanding", "obviously",
+	"occasionally", "otherwise", "overall", "particularly", "presently", "previously", "rather", "regardless", "secondly",
+	"shortly", "significantly", "similarly", "simultaneously", "since", "so", "soon", "specifically", "still", "straightaway",
+	"subsequently", "surely", "surprisingly", "than", "then", "thereafter", "therefore", "thereupon", "thirdly", "though",
+	"thus", "till", "undeniably", "undoubtedly", "unless", "unlike", "unquestionably", "until", "when", "whenever",
+	"whereas", "while", "whether", "if", "actually", "anyway", "anyways", "anyhow", "mostly", "namely", "including", "suddenly" ];
+export const multipleWords = [ "above all", "after all", "after that", "all in all", "all of a sudden", "all things considered",
+	"analogous to", "although this may be true", "analogous to", "another key point", "as a matter of fact", "as a result",
+	"as an illustration", "as can be seen", "as has been noted", "as I have noted", "as I have said", "as I have shown",
+	"as long as", "as much as", "as opposed to", "as shown above", "as soon as", "as well as", "at any rate", "at first", "at last",
+	"at least", "at length", "at the present time", "at the same time", "at this instant", "at this point", "at this time",
+	"balanced against", "being that", "by all means", "by and large", "by comparison", "by the same token", "by the time",
+	"compared to", "be that as it may", "coupled with", "different from", "due to", "equally important", "even if",
+	"even more", "even so", "even though", "first thing to remember", "for example", "for fear that", "for instance",
+	"for one thing", "for that reason", "for the most part", "for the purpose of", "for the same reason", "for this purpose",
+	"for this reason", "from time to time", "given that", "given these points", "important to realize", "in a word", "in addition",
+	"in another case", "in any case", "in any event", "in brief", "in case", "in conclusion", "in contrast",
+	"in detail", "in due time", "in effect", "in either case", "either way", "in essence", "in fact", "in general", "in light of",
+	"in like fashion", "in like manner", "in order that", "in order to", "in other words", "in particular", "in reality",
+	"in short", "in similar fashion", "in spite of", "in sum", "in summary", "in that case", "in the event that",
+	"in the final analysis", "in the first place", "in the fourth place", "in the hope that", "in the light of",
+	"in the long run", "in the meantime", "in the same fashion", "in the same way", "in the second place",
+	"in the third place", "in this case", "in this situation", "in time", "in truth", "in view of", "inasmuch as",
+	"most compelling evidence", "most important", "must be remembered", "not only", "not to mention", "note that",
+	"now that", "of course", "on account of", "on balance", "on condition that", "on one hand", "on the condition that", "on the contrary",
+	"on the negative side", "on the other hand", "on the positive side", "on the whole", "on this occasion", "all at once",
+	"once in a while", 	"only if", "owing to", "point often overlooked", "prior to", "provided that", "seeing that",
+	"so as to", "so far", "so long as", "so that", "sooner or later", "such as", "summing up", "take the case of",
+	"that is", "that is to say", "then again", "this time", "to be sure", "to begin with", "to clarify", "to conclude",
+	"to demonstrate", "to emphasize", "to enumerate", "to explain", "to illustrate", "to list", "to point out",
+	"to put it another way", "to put it differently", "to repeat", "to rephrase it", "to say nothing of", "to sum up",
+	"to summarize", "to that end", "to the end that", "to this end", "together with", "under those circumstances", "until now",
+	"up against", "up to the present time", "vis a vis", "what's more", "what is more", "while it may be true", "while this may be true",
+	"with attention to", "with the result that", "with this in mind", "with this intention", "with this purpose in mind",
+	"without a doubt", "without delay", "without doubt", "without reservation", "according to", "no sooner", "at most", "at the most",
+	"from now on" ];
+
+export const allWords = singleWords.concat( multipleWords );
+
+export default allWords;

package/src/languageProcessing/languages/en/config/twoPartTransitionWords.js

@@ -0,0 +1,7 @@
+/** @module config/twoPartTransitionWords */
+
+/**
+ * Returns an array with two-part transition words to be used by the assessments.
+ * @type {Array} The array filled with two-part transition words.
+ */
+export default [ [ "both", "and" ], [ "not only", "but also" ], [ "neither", "nor" ], [ "either", "or" ], [ "not", "but" ] ];

package/src/languageProcessing/languages/en/config/wordComplexity.js

@@ -0,0 +1,5 @@
+// This is a config for the Word Complexity assessment. As such, this helper is not bundled in Yoast SEO.
+export default {
+	wordLength: 7,
+	doesUpperCaseDecreaseComplexity: true,
+};

package/src/languageProcessing/languages/en/helpers/calculateFleschReadingScore.js

@@ -0,0 +1,18 @@
+import {helpers} from '../../../index';
+const {formatNumber} = helpers;
+
+/**
+ * Returns the Flesch reading score for English.
+ *
+ * @param {Object} statistics The Flesch reading statistics.
+ *
+ * @returns {number} The Flesch reading score for English.
+ */
+export default function calculateScore(statistics) {
+  const score =
+    206.835 -
+    1.015 * statistics.averageWordsPerSentence -
+    84.6 * (statistics.numberOfSyllables / statistics.numberOfWords);
+
+  return formatNumber(score);
+}

package/src/languageProcessing/languages/en/helpers/checkIfWordIsComplex.js

@@ -0,0 +1,43 @@
+import buildFormRule from "../../../helpers/morphology/buildFormRule";
+import createRulesFromArrays from "../../../helpers/morphology/createRulesFromArrays";
+
+/**
+ * Checks if a word is complex.
+ * This is a helper for the Word Complexity assessment. As such, this helper is not bundled in Yoast SEO.
+ *
+ * @param {object} config The configuration needed for assessing the word's complexity, e.g., the frequency list.
+ * @param {string} word The word to check.
+ * @param {object}	premiumData The object that contains data for the assessment including the frequency list.
+ *
+ * @returns {boolean} Whether or not a word is complex.
+ */
+export default function checkIfWordIsComplex( config, word, premiumData ) {
+	const lengthLimit = config.wordLength;
+	const frequencyList = premiumData.frequencyList.list;
+
+	// Whether uppercased beginning of a word decreases its complexity.
+	const doesUpperCaseDecreaseComplexity = config.doesUpperCaseDecreaseComplexity;
+
+	// The word is not complex if it's less than or the same as the length limit, i.e. 7 characters for English.
+	if ( word.length <= lengthLimit ) {
+		return false;
+	}
+
+	// The word is not complex if it starts with a capital and thus is assumed to be a named entity.
+	if ( doesUpperCaseDecreaseComplexity && word[ 0 ].toLowerCase() !== word[ 0 ] ) {
+		return false;
+	}
+
+	// The word is not complex if it's in the frequency list.
+	if ( frequencyList.includes( word ) ) {
+		return false;
+	}
+
+	// The word is not complex if its singular form is in the frequency list.
+	if ( premiumData ) {
+		const singular = buildFormRule( word, createRulesFromArrays( premiumData.nouns.regexNoun.singularize ) );
+		return ! frequencyList.includes( singular );
+	}
+
+	return true;
+}

package/src/languageProcessing/languages/en/helpers/getClauses.js

@@ -0,0 +1,49 @@
+import {languageProcessing} from '../../../index';
+const {createRegexFromArray, getClauses} = languageProcessing;
+
+import Clause from '../values/Clause';
+import auxiliaries from '../config/internal/passiveVoiceAuxiliaries.js';
+import stopwords from '../config/stopWords.js';
+import {includes} from 'lodash';
+import stripSpaces from '../../../helpers/sanitize/stripSpaces';
+import {getIndicesByWordList as getIndicesOfList} from '../../../helpers/word/indices';
+
+const options = {
+  Clause,
+  stopwords,
+  auxiliaries: auxiliaries,
+  ingExclusions: ['king', 'cling', 'ring', 'being', 'thing', 'something', 'anything'],
+  regexes: {
+    auxiliaryRegex: createRegexFromArray(auxiliaries),
+    stopCharacterRegex: /([:,]|('ll)|('ve))(?=[ \n\r\t'"+\-»«‹›<>])/gi,
+    verbEndingInIngRegex: /\w+ing(?=$|[ \n\r\t.,'()"+\-;!?:/»«‹›<>])/gi
+  },
+  otherStopWordIndices: []
+};
+
+/**
+ * Gets active verbs (ending in ing) to determine sentence breakers in English.
+ *
+ * @param {string} sentence The sentence to get the active verbs from.
+ *
+ * @returns {Array} The array with valid matches.
+ */
+const getVerbsEndingInIngIndices = function(sentence) {
+  // Matches the sentences with words ending in ing.
+  let matches = sentence.match(options.regexes.verbEndingInIngRegex) || [];
+  // Filters out words ending in -ing that aren't verbs.
+  matches = matches.filter(match => !includes(options.ingExclusions, stripSpaces(match)));
+  return getIndicesOfList(matches, sentence);
+};
+
+/**
+ * Gets the clauses from a sentence by determining sentence breakers.
+ *
+ * @param {string} sentence The sentence to split up in clauses.
+ *
+ * @returns {Array} The array with all clauses that have an auxiliary.
+ */
+export default function getEnglishClauses(sentence) {
+  options.otherStopWordIndices = getVerbsEndingInIngIndices(sentence);
+  return getClauses(sentence, options);
+}

package/src/languageProcessing/languages/en/helpers/getStemmer.js

@@ -0,0 +1,22 @@
+import {get} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {baseStemmer} = languageProcessing;
+
+import determineStem from './internal/determineStem';
+
+/**
+ * Returns the stemmer for a researcher.
+ *
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Function} The stemmer.
+ */
+export default function getStemmer(researcher) {
+  const morphologyData = get(researcher.getData('morphology'), 'en', false);
+
+  if (morphologyData) {
+    return word => determineStem(word, morphologyData);
+  }
+
+  return baseStemmer;
+}

package/src/languageProcessing/languages/en/helpers/internal/determineStem.js

@@ -0,0 +1,178 @@
+import {isUndefined, flatten} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {buildFormRule, createRulesFromArrays} = languageProcessing;
+
+import getAdjectiveStem from './getAdjectiveStem';
+import {
+  getInfinitive,
+  checkIrregulars as getIrregularVerbParadigm,
+  endsWithIng
+} from './getVerbStem.js';
+
+/**
+ * Gets the shortest of the alphabetically ordered strings from an array.
+ *
+ * @param {string[]} array  The array of strings.
+ *
+ * @returns {string|undefined}  The shortest of the alphabetically ordered strings from the input array;
+ *                              undefined if the input array is empty.
+ */
+export function findShortestAndAlphabeticallyFirst(array) {
+  const strings = flatten(array);
+  let result = strings.pop();
+
+  strings.forEach(str => {
+    const lengthDifference = str.length - result.length;
+    if (lengthDifference === 0) {
+      if (str.localeCompare(result) < 0) {
+        result = str;
+      }
+    } else if (lengthDifference < 0) {
+      result = str;
+    }
+  });
+
+  return result;
+}
+
+/**
+ * Checks if the input word occurs in the list of exceptions and if so returns the first form of the paradigm, which is
+ * always the base.
+ *
+ * @param {string}		word		The word for which to determine its base.
+ * @param {string[]}	irregulars	An array of irregular nouns and adjectives.
+ *
+ * @returns {string|null} The base form of the irregular word; null if no irregular stem was found.
+ */
+export function determineIrregularStem(word, irregulars) {
+  for (const paradigm of irregulars) {
+    if (paradigm.includes(word)) {
+      return paradigm[0];
+    }
+  }
+  return null;
+}
+
+/**
+ * Checks if the input word occurs in the list of exception verbs and if so returns the first form
+ * of the paradigm, which is always the base. Contrary to nouns and adjectives, irregular verbs can have different prefixes
+ * which are not included in the list of exceptions and have to be processed separately.
+ *
+ * @param {string}	word            The word for which to determine its base.
+ * @param {Object}	verbMorphology  Regexes and irregulars for verb morphology, False if verb rules should not be applied.
+ *
+ * @returns {string|null} The base form of the irregular word; null if no irregular stem was found.
+ */
+export function determineIrregularVerbStem(word, verbMorphology) {
+  const paradigmIfIrregularVerb = getIrregularVerbParadigm(
+    word,
+    verbMorphology.irregularVerbs,
+    verbMorphology.regexVerb.verbPrefixes
+  );
+  if (!isUndefined(paradigmIfIrregularVerb)) {
+    return paradigmIfIrregularVerb[0];
+  }
+  return null;
+}
+
+/**
+ * Gets possible stems as a regular noun, adjective and verb.
+ *
+ * @param {string} word              The word for which to determine its base.
+ * @param {Object} morphologyData    The morphology data for the language.
+ *
+ * @returns {string} The shortest and the alphabetically-first of possible noun-like, verb-like and adjective-like bases.
+ */
+export function determineRegularStem(word, morphologyData) {
+  // Try to singularize as a noun.
+  const regexVerb = morphologyData.verbs.regexVerb;
+  const baseIfPluralNoun = buildFormRule(
+    word,
+    createRulesFromArrays(morphologyData.nouns.regexNoun.singularize)
+  );
+  if (!isUndefined(baseIfPluralNoun)) {
+    // Bring ing-nouns to base forms ("blessings" -> "bless").
+    if (endsWithIng(baseIfPluralNoun)) {
+      return buildFormRule(baseIfPluralNoun, createRulesFromArrays(regexVerb.ingFormToInfinitive));
+    }
+    return baseIfPluralNoun;
+  }
+
+  // Check if the word ends with "ic", "ical" or "ically". Return the "ical" form for consistency.
+  const regexAdjective = morphologyData.adjectives.regexAdjective;
+  const baseIfIcally = buildFormRule(word, createRulesFromArrays(regexAdjective.icallyToBase));
+  if (!isUndefined(baseIfIcally)) {
+    return baseIfIcally;
+  }
+
+  // No more quick guesses, we have to determine a verbal infinitive and an adjectival base.
+  const possibleRegularBases = [];
+
+  // Verbal infinitive.
+  const baseIfVerb = getInfinitive(word, regexVerb).infinitive;
+
+  possibleRegularBases.push(baseIfVerb);
+
+  // Adjectival base.
+  const stopAdjectives = morphologyData.adjectives.stopAdjectives;
+  /*
+	 The following list of words are the stem of adjectives that have two syllables or more that receive -er and -est in their comparative and superlative forms, e.g 'shallow'.
+	 Please note that if the adjective ends in -y and -e, the ending is removed, e.g. 'acute' -> 'acut'.
+	 The list is not exhaustive and can be expanded if needed. Oxford dictionary was used to check if the forms indeed exist, e.g. "acuter".
+	 */
+  const multiSyllableAdjWithSuffixes = morphologyData.adjectives.multiSyllableAdjectives.list;
+
+  const baseIfAdjective = getAdjectiveStem(
+    word,
+    regexAdjective,
+    stopAdjectives,
+    multiSyllableAdjWithSuffixes
+  ).base;
+  possibleRegularBases.push(baseIfAdjective);
+
+  return findShortestAndAlphabeticallyFirst(possibleRegularBases);
+}
+
+/**
+ * Returns the stem of the input word using the morphologyData (language-specific).
+ *
+ * @param   {string} word           The word to get the stem for.
+ * @param   {Object} morphologyData The available morphology data per language (false if unavailable).
+ *
+ * @returns {string} Stemmed (or base) form of the word.
+ */
+export function determineStem(word, morphologyData) {
+  const nounMorphology = morphologyData.nouns;
+
+  const baseIfPossessive = buildFormRule(
+    word,
+    createRulesFromArrays(nounMorphology.regexNoun.possessiveToBase)
+  );
+
+  let stem;
+  let irregular;
+
+  // Determine if this is an irregular noun, adjective or verb.
+  if (isUndefined(baseIfPossessive)) {
+    stem = word;
+    // Word can be a noun, adjective or verb.
+    irregular =
+      determineIrregularStem(word, nounMorphology.irregularNouns) ||
+      determineIrregularStem(word, morphologyData.adjectives.irregularAdjectives) ||
+      determineIrregularVerbStem(word, morphologyData.verbs);
+  } else {
+    stem = baseIfPossessive;
+    // The word is a possessive, it can only be a noun or an ing-noun;
+    irregular = determineIrregularStem(baseIfPossessive, nounMorphology.irregularNouns);
+  }
+
+  if (irregular) {
+    // Found the stem of an irregular word! Return it.
+    return irregular;
+  }
+
+  // Treat the word as a regular noun, adjective or verb.
+  return determineRegularStem(stem, morphologyData);
+}
+
+export default determineStem;