axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/helpers/sentence/getSentences.js

@@ -0,0 +1,65 @@
+// Lodash imports.
+import { filter, flatMap, isEmpty, negate } from "lodash";
+
+// Internal dependencies.
+import { getBlocks } from "../html/html.js";
+import { imageRegex } from "../image/imageInText";
+import { stripBlockTagsAtStartEnd } from "../sanitize/stripHTMLTags";
+import { unifyNonBreakingSpace } from "../sanitize/unifyWhitespace";
+import defaultSentenceTokenizer from "./memoizedSentenceTokenizer";
+
+// Character classes.
+const newLines = "\n\r|\n|\r";
+
+// Regular expressions.
+const newLineRegex = new RegExp( newLines );
+const paragraphTagsRegex = new RegExp( "^(<p>|</p>)$" );
+
+/**
+ * Returns sentences in a string.
+ *
+ * @param {String}      text                The string to count sentences in.
+ * @param {function}    memoizedTokenizer   The memoized sentence tokenizer.
+ *
+ * @returns {Array} Sentences found in the text.
+ */
+export default function( text, memoizedTokenizer = defaultSentenceTokenizer ) {
+	// We don't remove the other HTML tags here since removing them might lead to incorrect results when running the sentence tokenizer.
+	// Unify only non-breaking spaces and not the other whitespaces since a whitespace could signify a sentence break or a new line.
+	text = unifyNonBreakingSpace( text );
+	/*
+	 * Remove images from text before tokenizing it into sentences.
+	 * This is necessary since the highlighting feature doesn't work if the yoastmark tags are enclosing a sentence starting with an image.
+	 * This step is done here so that applying highlight in captions is possible for all assessments that use this helper.
+	 */
+	text = text.replace( imageRegex, "" );
+
+	let blocks = getBlocks( text );
+
+	// Split each block on newlines.
+	blocks = flatMap( blocks, function( block ) {
+		return block.split( newLineRegex );
+	} );
+
+	/*
+	 * Filter blocks that contain only paragraph tags. This step is necessary
+	 * since switching between editors might add extra paragraph tags with a new line tag in the end
+	 * that are incorrectly converted into separate blocks.
+	 */
+	blocks = blocks.filter( block => ! paragraphTagsRegex.test( block ) );
+
+	/*
+	 * We use the `map` method followed by `flat` instead of `flatMap` because `flatMap` would override the second
+	 * argument of the memoizedTokenizer with the index of the iteratee.
+	 */
+	let sentences = blocks.map( block => memoizedTokenizer( block ) ).flat();
+
+	/*
+	 * Strip block tags from the start and/or the end of each sentence and whitespaces if present.
+	 * After tokenized, sometimes there are still block tags present in the beginning/end of a sentence.
+	 * Unstripped, these tags could potentially break the highlighting functionality.
+	 */
+	sentences = sentences.map( sentence => stripBlockTagsAtStartEnd( sentence ).trim() );
+
+	return filter( sentences, negate( isEmpty ) );
+}

package/src/languageProcessing/helpers/sentence/getSentencesFromTree.js

@@ -0,0 +1,55 @@
+/**
+ * Retrieves the start offset for a given node.
+ * @param {Node} node The current node.
+ * @returns {number} The start offset.
+ */
+function getStartOffset( node ) {
+	return node.sourceCodeLocation &&
+		( ( node.sourceCodeLocation.startTag && node.sourceCodeLocation.startTag.endOffset ) || node.sourceCodeLocation.startOffset ) || 0;
+}
+
+/**
+ * Retrieves the parent node for a given node.
+ * @param {Paper} 	paper 	The current paper.
+ * @param {Node} 	node 	The current node.
+ * @returns {Node} The parent node.
+ */
+export function getParentNode( paper, node ) {
+	// Includes a fallback so that if a parent node cannot be found for an implicit paragraph, we use the current node as the parent node.
+	return paper.getTree().findAll( treeNode => treeNode.childNodes && treeNode.childNodes.includes( node ) )[ 0 ] || node;
+}
+
+/**
+ * Gets all the sentences from paragraph and heading nodes.
+ * These two node types are the nodes that should contain sentences for the analysis.
+ *
+ * @param {Paper} paper The paper to get the sentences from.
+ *
+ * @returns {Sentence[]} The array of sentences retrieved from paragraph and heading nodes plus sourceCodeLocation of the parent node.
+ */
+export default function( paper ) {
+	// Get all nodes that have a sentence property which is not an empty array.
+	const tree = paper.getTree().findAll( treeNode => !! treeNode.sentences );
+
+	return tree.flatMap( node => node.sentences.map( sentence => {
+		let parentNode = node;
+
+		// For implicit paragraphs, base the details on the parent of this node.
+		if ( node.isImplicit ) {
+			parentNode = getParentNode( paper, node );
+		}
+
+		return {
+			...sentence,
+			// The parent node's start offset is the start offset of the parent node if it doesn't have a `startTag` property.
+			parentStartOffset: getStartOffset( parentNode ),
+			// The block client id of the parent node.
+			parentClientId: parentNode.clientId || "",
+			// The attribute id of the parent node, if available, otherwise an empty string.
+			// Only used for position-based highlighting in sub-blocks of Yoast blocks.
+			parentAttributeId: node.attributeId || "",
+			// Whether the parent node is the first section of Yoast sub-blocks. Only used for position-based highlighting.
+			isParentFirstSectionOfBlock: node.isFirstSection || false,
+		};
+	} ) );
+}

package/src/languageProcessing/helpers/sentence/memoizedSentenceTokenizer.js

@@ -0,0 +1,28 @@
+import SentenceTokenizer from "./SentenceTokenizer";
+import { memoize } from "lodash";
+
+/**
+ * Returns the sentences from a certain text.
+ *
+ * @param {string} text 					The text to retrieve sentences from.
+ * @param {boolean} [trimSentences=true] 	Whether to trim whitespace from the beginning and end of the sentences or not.
+ *
+ * @returns {Array<string>} The list of sentences in the text.
+ */
+function getSentenceTokenizer( text, trimSentences = true ) {
+	const sentenceTokenizer = new SentenceTokenizer();
+	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
+	sentenceTokenizer.tokenize( tokenizer, text );
+
+	return ( tokens.length === 0 ? [] : sentenceTokenizer.getSentencesFromTokens( tokens, trimSentences ) );
+}
+
+/*
+ * The second argument to the memoize function is a so-called resolver function.
+ * It creates a cache key consisting of a combination of all arguments to a function.
+ * This is needed because by default, only the first argument to a function is used as the map cache key by the memoize function.
+ * This means that a function is only re-run if the value of the first argument changes.
+ * We want to re-run the getSentenceTokenizer function also when only the second argument changes to prevent cache collisions.
+ * @see https://lodash.com/docs/4.17.15#memoize
+ */
+export default memoize( getSentenceTokenizer, ( ...args ) => JSON.stringify( args ) );

package/src/languageProcessing/helpers/sentence/sentencesLength.js

@@ -0,0 +1,31 @@
+import wordCount from "../word/countWords.js";
+import { forEach } from "lodash";
+import { stripFullTags as stripHTMLTags } from "../sanitize/stripHTMLTags.js";
+
+/**
+ * Returns an array with the length of each sentence.
+ *
+ * @param {Array} sentences Array with sentences from text.
+ * @param {Researcher} 	researcher 	The researcher to use for analysis.
+ *
+ * @returns {Array} Array with the length of each sentence.
+ */
+export default function( sentences, researcher ) {
+	const sentencesWordCount = [];
+	forEach( sentences, function( sentence ) {
+		// For counting words we want to omit the HTMLtags.
+		const strippedSentence = stripHTMLTags( sentence );
+		// A helper to count characters for languages that don't count number of words for text length.
+		const countCharacters = researcher.getHelper( "customCountLength" );
+		const length = countCharacters ? countCharacters( strippedSentence ) : wordCount( strippedSentence );
+		if ( length <= 0 ) {
+			return;
+		}
+
+		sentencesWordCount.push( {
+			sentence: sentence,
+			sentenceLength: length,
+		} );
+	} );
+	return sentencesWordCount;
+}

package/src/languageProcessing/helpers/syllables/DeviationFragment.js

@@ -0,0 +1,112 @@
+import { isUndefined, pick } from "lodash";
+
+/**
+ * A DeviationFragment represents a partial deviation when counting syllables.
+ */
+export default class DeviationFragment {
+	/**
+	 * Constructs a new DeviationFragment.
+	 *
+	 * @param {Object} options Extra options that are used to match this deviation fragment.
+	 * @param {string} options.location The location in the word where this deviation can occur.
+	 * @param {string} options.word The actual string that should be counted differently.
+	 * @param {number} options.syllables The amount of syllables this fragment has.
+	 * @param {string[]} [options.notFollowedBy] A list of characters that this fragment shouldn't be followed with.
+	 * @param {string[]} [options.alsoFollowedBy] A list of characters that this fragment could be followed with.
+	 *
+	 * @constructor
+	 */
+	constructor( options ) {
+		this._location = options.location;
+		this._fragment = options.word;
+		this._syllables = options.syllables;
+		this._regex = null;
+
+		this._options = pick( options, [ "notFollowedBy", "alsoFollowedBy" ] );
+	}
+
+	/**
+	 * Creates a regex that matches this fragment inside a word.
+	 *
+	 * @returns {void}
+	 */
+	createRegex() {
+		const options = this._options;
+
+		let fragment = this._fragment;
+
+		if ( ! isUndefined( options.notFollowedBy ) ) {
+			fragment += "(?![" + options.notFollowedBy.join( "" ) + "])";
+		}
+
+		if ( ! isUndefined( options.alsoFollowedBy ) ) {
+			fragment += "[" + options.alsoFollowedBy.join( "" ) + "]?";
+		}
+
+		let regexString;
+		switch ( this._location ) {
+			case "atBeginning":
+				regexString = "^" + fragment;
+				break;
+
+			case "atEnd":
+				regexString = fragment + "$";
+				break;
+
+			case "atBeginningOrEnd":
+				regexString = "(^" + fragment + ")|(" + fragment + "$)";
+				break;
+
+			default:
+				regexString = fragment;
+				break;
+		}
+
+		this._regex = new RegExp( regexString );
+	}
+
+	/**
+	 * Returns the regex that matches this fragment inside a word.
+	 *
+	 * @returns {RegExp} The regex that matches this fragment.
+	 */
+	getRegex() {
+		if ( null === this._regex ) {
+			this.createRegex();
+		}
+
+		return this._regex;
+	}
+
+	/**
+	 * Returns whether this fragment occurs in a word.
+	 *
+	 * @param {string} word The word to match the fragment in.
+	 * @returns {boolean} Whether or not this fragment occurs in a word.
+	 */
+	occursIn( word ) {
+		const regex = this.getRegex();
+
+		return regex.test( word );
+	}
+
+	/**
+	 * Removes this fragment from the given word.
+	 *
+	 * @param {string} word The word to remove this fragment from.
+	 * @returns {string} The modified word.
+	 */
+	removeFrom( word ) {
+		// Replace by a space to keep the remaining parts separated.
+		return word.replace( this._fragment, " " );
+	}
+
+	/**
+	 * Returns the amount of syllables for this fragment.
+	 *
+	 * @returns {number} The amount of syllables for this fragment.
+	 */
+	getSyllables() {
+		return this._syllables;
+	}
+}

package/src/languageProcessing/helpers/syllables/countSyllables.js

@@ -0,0 +1,182 @@
+/** @module stringProcessing/countSyllables */
+
+import getWords from "../word/getWords.js";
+
+import { filter, find, flatMap, forEach, isUndefined, map, memoize, sum } from "lodash";
+
+import SyllableCountIterator from "./syllableCountIterator.js";
+import DeviationFragment from "./DeviationFragment";
+
+/**
+ * Counts vowel groups inside a word.
+ *
+ * @param {string} word         A text with words to count syllables.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {number} the syllable count.
+ */
+const countVowelGroups = function( word, syllables ) {
+	let numberOfSyllables = 0;
+	const vowelRegex = new RegExp( "[^" + syllables.vowels + "]", "ig" );
+	const foundVowels = word.split( vowelRegex );
+	const filteredWords = filter( foundVowels, function( vowel ) {
+		return vowel !== "";
+	} );
+	numberOfSyllables += filteredWords.length;
+
+	return numberOfSyllables;
+};
+
+/**
+ * Counts the syllables using vowel exclusions. These are used for groups of vowels that are more or less
+ * than 1 syllable.
+ *
+ * @param {String} word         The word to count syllables of.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {number} The number of syllables found in the given word.
+ */
+const countVowelDeviations = function( word, syllables ) {
+	const syllableCountIterator = new SyllableCountIterator( syllables );
+	return syllableCountIterator.countSyllables( word );
+};
+
+/**
+ * Returns the number of syllables for the word if it is in the list of full word deviations.
+ *
+ * @param {String} word         The word to retrieve the syllables for.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {number} The number of syllables found.
+ */
+const countFullWordDeviations = function( word, syllables ) {
+	const fullWordDeviations = syllables.deviations.words.full;
+
+	const deviation = find( fullWordDeviations, function( fullWordDeviation ) {
+		return fullWordDeviation.word === word;
+	} );
+
+	if ( ! isUndefined( deviation ) ) {
+		return deviation.syllables;
+	}
+
+	return 0;
+};
+
+/**
+ * Creates an array of deviation fragments for a certain locale.
+ *
+ * @param {Object} syllableConfig Syllable config for a certain locale.
+ * @returns {DeviationFragment[]} A list of deviation fragments
+ */
+function createDeviationFragments( syllableConfig ) {
+	let deviationFragments = [];
+
+	const deviations = syllableConfig.deviations;
+
+	deviationFragments = flatMap( deviations.words.fragments, function( fragments, fragmentLocation ) {
+		return map( fragments, function( fragment ) {
+			fragment.location = fragmentLocation;
+
+			return new DeviationFragment( fragment );
+		} );
+	} );
+
+	return deviationFragments;
+}
+
+const createDeviationFragmentsMemoized = memoize( createDeviationFragments );
+
+/**
+ * Counts syllables in partial exclusions. If these are found, returns the number of syllables found, and the modified word.
+ * The word is modified so the excluded part isn't counted by the normal syllable counter.
+ *
+ * @param {String} word 		The word to count syllables of.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {object} The number of syllables found and the modified word.
+ */
+const countPartialWordDeviations = function( word, syllables ) {
+	const deviationFragments = createDeviationFragmentsMemoized( syllables );
+	let remainingParts = word;
+	let syllableCount = 0;
+
+	forEach( deviationFragments, function( deviationFragment ) {
+		if ( deviationFragment.occursIn( remainingParts ) ) {
+			remainingParts = deviationFragment.removeFrom( remainingParts );
+			syllableCount += deviationFragment.getSyllables();
+		}
+	} );
+
+	return { word: remainingParts, syllableCount: syllableCount };
+};
+
+/**
+ * Count the number of syllables in a word, using vowels and exceptions.
+ *
+ * @param {String} word         The word to count the number of syllables of.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {number} The number of syllables found in a word.
+ */
+const countUsingVowels = function( word, syllables ) {
+	let syllableCount = 0;
+
+	syllableCount += countVowelGroups( word, syllables );
+	if ( ! isUndefined( syllables.deviations ) && ! isUndefined( syllables.deviations.vowels ) ) {
+		syllableCount += countVowelDeviations( word, syllables );
+	}
+	return syllableCount;
+};
+
+/**
+ * Counts the number of syllables in a word.
+ *
+ * @param {string} word         The word to count syllables of.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {number} The syllable count for the word.
+ */
+export const countSyllablesInWord = function( word, syllables ) {
+	let syllableCount = 0;
+
+	if ( ! isUndefined( syllables.deviations ) && ! isUndefined( syllables.deviations.words ) ) {
+		if ( ! isUndefined( syllables.deviations.words.full ) ) {
+			const fullWordExclusion = countFullWordDeviations( word, syllables );
+			if ( fullWordExclusion !== 0 ) {
+				return fullWordExclusion;
+			}
+		}
+
+		if ( ! isUndefined( syllables.deviations.words.fragments ) ) {
+			const partialExclusions = countPartialWordDeviations( word, syllables );
+			word = partialExclusions.word;
+			syllableCount += partialExclusions.syllableCount;
+		}
+	}
+	syllableCount += countUsingVowels( word, syllables );
+
+	return syllableCount;
+};
+
+/**
+ * Counts the number of syllables in a text per word based on vowels.
+ * Uses exclusion words for words that cannot be matched with vowel matching.
+ *
+ * @param {String} text         The text to count the syllables of.
+ * @param {Object} syllables    The syllables data for the specific language.
+ *
+ * @returns {int} The total number of syllables found in the text.
+ */
+const countSyllablesInText = function( text, syllables ) {
+	text = text.toLocaleLowerCase();
+	const words = getWords( text );
+
+	const syllableCounts = map( words,  function( word ) {
+		return countSyllablesInWord( word, syllables );
+	} );
+
+	return sum( syllableCounts );
+};
+
+export default countSyllablesInText;

package/src/languageProcessing/helpers/syllables/syllableCountIterator.js

@@ -0,0 +1,56 @@
+import SyllableCountStep from "./syllableCountStep.js";
+
+import { forEach, isUndefined } from "lodash";
+
+/**
+ * A SyllableCountIterator contains individual SyllableCountSteps.
+ */
+export default class SyllableCountIterator {
+	/**
+	 * Creates a syllable count iterator.
+	 *
+	 * @param {object} config The config object containing an array with syllable exclusions.
+	 * @constructor
+	 */
+	constructor( config ) {
+		this.countSteps = [];
+		if ( ! isUndefined( config ) ) {
+			this.createSyllableCountSteps( config.deviations.vowels );
+		}
+	}
+
+	/**
+	 * Creates a syllable count step object for each exclusion.
+	 *
+	 * @param {object} syllableCounts The object containing all exclusion syllables including the multipliers.
+	 * @returns {void}
+	 */
+	createSyllableCountSteps( syllableCounts ) {
+		forEach( syllableCounts, function( syllableCountStep ) {
+			this.countSteps.push( new SyllableCountStep( syllableCountStep ) );
+		}.bind( this ) );
+	}
+
+	/**
+	 * Returns all available count steps.
+	 *
+	 * @returns {Array} All available count steps.
+	 */
+	getAvailableSyllableCountSteps() {
+		return this.countSteps;
+	}
+
+	/**
+	 * Counts the syllables for all the steps and returns the total syllable count.
+	 *
+	 * @param {String} word The word to count syllables in.
+	 * @returns {number} The number of syllables found based on exclusions.
+	 */
+	countSyllables( word ) {
+		let syllableCount = 0;
+		forEach( this.countSteps, function( step ) {
+			syllableCount += step.countSyllables( word );
+		} );
+		return syllableCount;
+	}
+}

package/src/languageProcessing/helpers/syllables/syllableCountStep.js

@@ -0,0 +1,68 @@
+import { isUndefined } from "lodash";
+
+import arrayToRegex from "../regex/createRegexFromArray.js";
+
+/**
+ * A SyllableCountStep is an individual step in a SyllableCountIterator.
+ */
+export default class SyllableCountStep {
+	/**
+	 * Constructs a language syllable regex that contains a regex for matching syllable exclusion.
+	 *
+	 * @param {object} syllableRegex The object containing the syllable exclusions.
+	 * @constructor
+	 */
+	constructor( syllableRegex ) {
+		this._hasRegex = false;
+		this._regex = "";
+		this._multiplier = "";
+		this.createRegex( syllableRegex );
+	}
+
+	/**
+	 * Checks whether a valid regex has been set.
+	 *
+	 * @returns {boolean} True if a regex has been set, false if not.
+	 */
+	hasRegex() {
+		return this._hasRegex;
+	}
+
+	/**
+	 * Creates a regex based on the given syllable exclusions, and sets the multiplier to use.
+	 *
+	 * @param {object} syllableRegex The object containing the syllable exclusions and multiplier.
+	 * @returns {void}
+	 */
+	createRegex( syllableRegex ) {
+		if ( ! isUndefined( syllableRegex ) && ! isUndefined( syllableRegex.fragments ) ) {
+			this._hasRegex = true;
+			this._regex = arrayToRegex( syllableRegex.fragments, true );
+			this._multiplier = syllableRegex.countModifier;
+		}
+	}
+
+	/**
+	 * Returns the stored regular expression.
+	 *
+	 * @returns {RegExp} The stored regular expression.
+	 */
+	getRegex() {
+		return this._regex;
+	}
+
+	/**
+	 * Matches syllable exclusions in a given word and returns the number found multiplied by the
+	 * given multiplier. The result of this multiplication is the syllable count.
+	 *
+	 * @param {String} word The word to match for syllable exclusions.
+	 * @returns {number} The amount of syllables found.
+	 */
+	countSyllables( word ) {
+		if ( this._hasRegex ) {
+			const match = word.match( this._regex ) || [];
+			return match.length * this._multiplier;
+		}
+		return 0;
+	}
+}

package/src/languageProcessing/helpers/transform/transformWordsWithHyphens.js

@@ -0,0 +1,17 @@
+/**
+ * Splits words from an array that contain hyphens and adds the elements to the array if they are not yet there.
+ *
+ * @param {Array} array The array to check and modify.
+ *
+ * @returns {Array} A new array with the elements containing hyphens split.
+ */
+export default function( array ) {
+	let newArray = array;
+	array.forEach( element => {
+		element = element.split( "-" );
+		if ( element.length > 0 && element.filter( unit => ! array.includes( unit ) ).length > 0 ) {
+			newArray = newArray.concat( element );
+		}
+	} );
+	return newArray;
+}

package/src/languageProcessing/helpers/transliterate/replaceDiacritics.js

@@ -0,0 +1,22 @@
+/** @module stringProcessing/replaceDiacritics */
+
+import diacriticsRemovalMap from "../../../config/diacritics.js";
+
+/**
+ * Replaces all diacritics from the text based on the diacritics removal map.
+ *
+ * @param {string} text The text to remove diacritics from.
+ *
+ * @returns {string} The text with all diacritics replaced.
+ */
+export default function( text ) {
+	const map = diacriticsRemovalMap();
+
+	for ( let i = 0; i < map.length; i++ ) {
+		text = text.replace(
+			map[ i ].letters,
+			map[ i ].base
+		);
+	}
+	return text;
+}