axyseo 2.0.0-alpha.0.0.4 → 2.0.0-alpha.0.0.40

package/src/languageProcessing/helpers/sentence/getSentences.js

@@ -1,65 +0,0 @@
-// Lodash imports.
-import { filter, flatMap, isEmpty, negate } from "lodash";
-
-// Internal dependencies.
-import { getBlocks } from "../html/html.js";
-import { imageRegex } from "../image/imageInText";
-import { stripBlockTagsAtStartEnd } from "../sanitize/stripHTMLTags";
-import { unifyNonBreakingSpace } from "../sanitize/unifyWhitespace";
-import defaultSentenceTokenizer from "./memoizedSentenceTokenizer";
-
-// Character classes.
-const newLines = "\n\r|\n|\r";
-
-// Regular expressions.
-const newLineRegex = new RegExp( newLines );
-const paragraphTagsRegex = new RegExp( "^(<p>|</p>)$" );
-
-/**
- * Returns sentences in a string.
- *
- * @param {String}      text                The string to count sentences in.
- * @param {function}    memoizedTokenizer   The memoized sentence tokenizer.
- *
- * @returns {Array} Sentences found in the text.
- */
-export default function( text, memoizedTokenizer = defaultSentenceTokenizer ) {
-	// We don't remove the other HTML tags here since removing them might lead to incorrect results when running the sentence tokenizer.
-	// Unify only non-breaking spaces and not the other whitespaces since a whitespace could signify a sentence break or a new line.
-	text = unifyNonBreakingSpace( text );
-	/*
-	 * Remove images from text before tokenizing it into sentences.
-	 * This is necessary since the highlighting feature doesn't work if the yoastmark tags are enclosing a sentence starting with an image.
-	 * This step is done here so that applying highlight in captions is possible for all assessments that use this helper.
-	 */
-	text = text.replace( imageRegex, "" );
-
-	let blocks = getBlocks( text );
-
-	// Split each block on newlines.
-	blocks = flatMap( blocks, function( block ) {
-		return block.split( newLineRegex );
-	} );
-
-	/*
-	 * Filter blocks that contain only paragraph tags. This step is necessary
-	 * since switching between editors might add extra paragraph tags with a new line tag in the end
-	 * that are incorrectly converted into separate blocks.
-	 */
-	blocks = blocks.filter( block => ! paragraphTagsRegex.test( block ) );
-
-	/*
-	 * We use the `map` method followed by `flat` instead of `flatMap` because `flatMap` would override the second
-	 * argument of the memoizedTokenizer with the index of the iteratee.
-	 */
-	let sentences = blocks.map( block => memoizedTokenizer( block ) ).flat();
-
-	/*
-	 * Strip block tags from the start and/or the end of each sentence and whitespaces if present.
-	 * After tokenized, sometimes there are still block tags present in the beginning/end of a sentence.
-	 * Unstripped, these tags could potentially break the highlighting functionality.
-	 */
-	sentences = sentences.map( sentence => stripBlockTagsAtStartEnd( sentence ).trim() );
-
-	return filter( sentences, negate( isEmpty ) );
-}

package/src/languageProcessing/helpers/sentence/getSentencesFromTree.js

@@ -1,55 +0,0 @@
-/**
- * Retrieves the start offset for a given node.
- * @param {Node} node The current node.
- * @returns {number} The start offset.
- */
-function getStartOffset( node ) {
-	return node.sourceCodeLocation &&
-		( ( node.sourceCodeLocation.startTag && node.sourceCodeLocation.startTag.endOffset ) || node.sourceCodeLocation.startOffset ) || 0;
-}
-
-/**
- * Retrieves the parent node for a given node.
- * @param {Paper} 	paper 	The current paper.
- * @param {Node} 	node 	The current node.
- * @returns {Node} The parent node.
- */
-export function getParentNode( paper, node ) {
-	// Includes a fallback so that if a parent node cannot be found for an implicit paragraph, we use the current node as the parent node.
-	return paper.getTree().findAll( treeNode => treeNode.childNodes && treeNode.childNodes.includes( node ) )[ 0 ] || node;
-}
-
-/**
- * Gets all the sentences from paragraph and heading nodes.
- * These two node types are the nodes that should contain sentences for the analysis.
- *
- * @param {Paper} paper The paper to get the sentences from.
- *
- * @returns {Sentence[]} The array of sentences retrieved from paragraph and heading nodes plus sourceCodeLocation of the parent node.
- */
-export default function( paper ) {
-	// Get all nodes that have a sentence property which is not an empty array.
-	const tree = paper.getTree().findAll( treeNode => !! treeNode.sentences );
-
-	return tree.flatMap( node => node.sentences.map( sentence => {
-		let parentNode = node;
-
-		// For implicit paragraphs, base the details on the parent of this node.
-		if ( node.isImplicit ) {
-			parentNode = getParentNode( paper, node );
-		}
-
-		return {
-			...sentence,
-			// The parent node's start offset is the start offset of the parent node if it doesn't have a `startTag` property.
-			parentStartOffset: getStartOffset( parentNode ),
-			// The block client id of the parent node.
-			parentClientId: parentNode.clientId || "",
-			// The attribute id of the parent node, if available, otherwise an empty string.
-			// Only used for position-based highlighting in sub-blocks of Yoast blocks.
-			parentAttributeId: node.attributeId || "",
-			// Whether the parent node is the first section of Yoast sub-blocks. Only used for position-based highlighting.
-			isParentFirstSectionOfBlock: node.isFirstSection || false,
-		};
-	} ) );
-}

package/src/languageProcessing/helpers/sentence/memoizedSentenceTokenizer.js

@@ -1,28 +0,0 @@
-import SentenceTokenizer from "./SentenceTokenizer";
-import { memoize } from "lodash";
-
-/**
- * Returns the sentences from a certain text.
- *
- * @param {string} text 					The text to retrieve sentences from.
- * @param {boolean} [trimSentences=true] 	Whether to trim whitespace from the beginning and end of the sentences or not.
- *
- * @returns {Array<string>} The list of sentences in the text.
- */
-function getSentenceTokenizer( text, trimSentences = true ) {
-	const sentenceTokenizer = new SentenceTokenizer();
-	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
-	sentenceTokenizer.tokenize( tokenizer, text );
-
-	return ( tokens.length === 0 ? [] : sentenceTokenizer.getSentencesFromTokens( tokens, trimSentences ) );
-}
-
-/*
- * The second argument to the memoize function is a so-called resolver function.
- * It creates a cache key consisting of a combination of all arguments to a function.
- * This is needed because by default, only the first argument to a function is used as the map cache key by the memoize function.
- * This means that a function is only re-run if the value of the first argument changes.
- * We want to re-run the getSentenceTokenizer function also when only the second argument changes to prevent cache collisions.
- * @see https://lodash.com/docs/4.17.15#memoize
- */
-export default memoize( getSentenceTokenizer, ( ...args ) => JSON.stringify( args ) );

package/src/languageProcessing/helpers/sentence/sentencesLength.js

@@ -1,31 +0,0 @@
-import wordCount from "../word/countWords.js";
-import { forEach } from "lodash";
-import { stripFullTags as stripHTMLTags } from "../sanitize/stripHTMLTags.js";
-
-/**
- * Returns an array with the length of each sentence.
- *
- * @param {Array} sentences Array with sentences from text.
- * @param {Researcher} 	researcher 	The researcher to use for analysis.
- *
- * @returns {Array} Array with the length of each sentence.
- */
-export default function( sentences, researcher ) {
-	const sentencesWordCount = [];
-	forEach( sentences, function( sentence ) {
-		// For counting words we want to omit the HTMLtags.
-		const strippedSentence = stripHTMLTags( sentence );
-		// A helper to count characters for languages that don't count number of words for text length.
-		const countCharacters = researcher.getHelper( "customCountLength" );
-		const length = countCharacters ? countCharacters( strippedSentence ) : wordCount( strippedSentence );
-		if ( length <= 0 ) {
-			return;
-		}
-
-		sentencesWordCount.push( {
-			sentence: sentence,
-			sentenceLength: length,
-		} );
-	} );
-	return sentencesWordCount;
-}

package/src/languageProcessing/helpers/syllables/DeviationFragment.js

@@ -1,112 +0,0 @@
-import { isUndefined, pick } from "lodash";
-
-/**
- * A DeviationFragment represents a partial deviation when counting syllables.
- */
-export default class DeviationFragment {
-	/**
-	 * Constructs a new DeviationFragment.
-	 *
-	 * @param {Object} options Extra options that are used to match this deviation fragment.
-	 * @param {string} options.location The location in the word where this deviation can occur.
-	 * @param {string} options.word The actual string that should be counted differently.
-	 * @param {number} options.syllables The amount of syllables this fragment has.
-	 * @param {string[]} [options.notFollowedBy] A list of characters that this fragment shouldn't be followed with.
-	 * @param {string[]} [options.alsoFollowedBy] A list of characters that this fragment could be followed with.
-	 *
-	 * @constructor
-	 */
-	constructor( options ) {
-		this._location = options.location;
-		this._fragment = options.word;
-		this._syllables = options.syllables;
-		this._regex = null;
-
-		this._options = pick( options, [ "notFollowedBy", "alsoFollowedBy" ] );
-	}
-
-	/**
-	 * Creates a regex that matches this fragment inside a word.
-	 *
-	 * @returns {void}
-	 */
-	createRegex() {
-		const options = this._options;
-
-		let fragment = this._fragment;
-
-		if ( ! isUndefined( options.notFollowedBy ) ) {
-			fragment += "(?![" + options.notFollowedBy.join( "" ) + "])";
-		}
-
-		if ( ! isUndefined( options.alsoFollowedBy ) ) {
-			fragment += "[" + options.alsoFollowedBy.join( "" ) + "]?";
-		}
-
-		let regexString;
-		switch ( this._location ) {
-			case "atBeginning":
-				regexString = "^" + fragment;
-				break;
-
-			case "atEnd":
-				regexString = fragment + "$";
-				break;
-
-			case "atBeginningOrEnd":
-				regexString = "(^" + fragment + ")|(" + fragment + "$)";
-				break;
-
-			default:
-				regexString = fragment;
-				break;
-		}
-
-		this._regex = new RegExp( regexString );
-	}
-
-	/**
-	 * Returns the regex that matches this fragment inside a word.
-	 *
-	 * @returns {RegExp} The regex that matches this fragment.
-	 */
-	getRegex() {
-		if ( null === this._regex ) {
-			this.createRegex();
-		}
-
-		return this._regex;
-	}
-
-	/**
-	 * Returns whether this fragment occurs in a word.
-	 *
-	 * @param {string} word The word to match the fragment in.
-	 * @returns {boolean} Whether or not this fragment occurs in a word.
-	 */
-	occursIn( word ) {
-		const regex = this.getRegex();
-
-		return regex.test( word );
-	}
-
-	/**
-	 * Removes this fragment from the given word.
-	 *
-	 * @param {string} word The word to remove this fragment from.
-	 * @returns {string} The modified word.
-	 */
-	removeFrom( word ) {
-		// Replace by a space to keep the remaining parts separated.
-		return word.replace( this._fragment, " " );
-	}
-
-	/**
-	 * Returns the amount of syllables for this fragment.
-	 *
-	 * @returns {number} The amount of syllables for this fragment.
-	 */
-	getSyllables() {
-		return this._syllables;
-	}
-}

package/src/languageProcessing/helpers/syllables/countSyllables.js

@@ -1,182 +0,0 @@
-/** @module stringProcessing/countSyllables */
-
-import getWords from "../word/getWords.js";
-
-import { filter, find, flatMap, forEach, isUndefined, map, memoize, sum } from "lodash";
-
-import SyllableCountIterator from "./syllableCountIterator.js";
-import DeviationFragment from "./DeviationFragment";
-
-/**
- * Counts vowel groups inside a word.
- *
- * @param {string} word         A text with words to count syllables.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {number} the syllable count.
- */
-const countVowelGroups = function( word, syllables ) {
-	let numberOfSyllables = 0;
-	const vowelRegex = new RegExp( "[^" + syllables.vowels + "]", "ig" );
-	const foundVowels = word.split( vowelRegex );
-	const filteredWords = filter( foundVowels, function( vowel ) {
-		return vowel !== "";
-	} );
-	numberOfSyllables += filteredWords.length;
-
-	return numberOfSyllables;
-};
-
-/**
- * Counts the syllables using vowel exclusions. These are used for groups of vowels that are more or less
- * than 1 syllable.
- *
- * @param {String} word         The word to count syllables of.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {number} The number of syllables found in the given word.
- */
-const countVowelDeviations = function( word, syllables ) {
-	const syllableCountIterator = new SyllableCountIterator( syllables );
-	return syllableCountIterator.countSyllables( word );
-};
-
-/**
- * Returns the number of syllables for the word if it is in the list of full word deviations.
- *
- * @param {String} word         The word to retrieve the syllables for.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {number} The number of syllables found.
- */
-const countFullWordDeviations = function( word, syllables ) {
-	const fullWordDeviations = syllables.deviations.words.full;
-
-	const deviation = find( fullWordDeviations, function( fullWordDeviation ) {
-		return fullWordDeviation.word === word;
-	} );
-
-	if ( ! isUndefined( deviation ) ) {
-		return deviation.syllables;
-	}
-
-	return 0;
-};
-
-/**
- * Creates an array of deviation fragments for a certain locale.
- *
- * @param {Object} syllableConfig Syllable config for a certain locale.
- * @returns {DeviationFragment[]} A list of deviation fragments
- */
-function createDeviationFragments( syllableConfig ) {
-	let deviationFragments = [];
-
-	const deviations = syllableConfig.deviations;
-
-	deviationFragments = flatMap( deviations.words.fragments, function( fragments, fragmentLocation ) {
-		return map( fragments, function( fragment ) {
-			fragment.location = fragmentLocation;
-
-			return new DeviationFragment( fragment );
-		} );
-	} );
-
-	return deviationFragments;
-}
-
-const createDeviationFragmentsMemoized = memoize( createDeviationFragments );
-
-/**
- * Counts syllables in partial exclusions. If these are found, returns the number of syllables found, and the modified word.
- * The word is modified so the excluded part isn't counted by the normal syllable counter.
- *
- * @param {String} word 		The word to count syllables of.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {object} The number of syllables found and the modified word.
- */
-const countPartialWordDeviations = function( word, syllables ) {
-	const deviationFragments = createDeviationFragmentsMemoized( syllables );
-	let remainingParts = word;
-	let syllableCount = 0;
-
-	forEach( deviationFragments, function( deviationFragment ) {
-		if ( deviationFragment.occursIn( remainingParts ) ) {
-			remainingParts = deviationFragment.removeFrom( remainingParts );
-			syllableCount += deviationFragment.getSyllables();
-		}
-	} );
-
-	return { word: remainingParts, syllableCount: syllableCount };
-};
-
-/**
- * Count the number of syllables in a word, using vowels and exceptions.
- *
- * @param {String} word         The word to count the number of syllables of.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {number} The number of syllables found in a word.
- */
-const countUsingVowels = function( word, syllables ) {
-	let syllableCount = 0;
-
-	syllableCount += countVowelGroups( word, syllables );
-	if ( ! isUndefined( syllables.deviations ) && ! isUndefined( syllables.deviations.vowels ) ) {
-		syllableCount += countVowelDeviations( word, syllables );
-	}
-	return syllableCount;
-};
-
-/**
- * Counts the number of syllables in a word.
- *
- * @param {string} word         The word to count syllables of.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {number} The syllable count for the word.
- */
-export const countSyllablesInWord = function( word, syllables ) {
-	let syllableCount = 0;
-
-	if ( ! isUndefined( syllables.deviations ) && ! isUndefined( syllables.deviations.words ) ) {
-		if ( ! isUndefined( syllables.deviations.words.full ) ) {
-			const fullWordExclusion = countFullWordDeviations( word, syllables );
-			if ( fullWordExclusion !== 0 ) {
-				return fullWordExclusion;
-			}
-		}
-
-		if ( ! isUndefined( syllables.deviations.words.fragments ) ) {
-			const partialExclusions = countPartialWordDeviations( word, syllables );
-			word = partialExclusions.word;
-			syllableCount += partialExclusions.syllableCount;
-		}
-	}
-	syllableCount += countUsingVowels( word, syllables );
-
-	return syllableCount;
-};
-
-/**
- * Counts the number of syllables in a text per word based on vowels.
- * Uses exclusion words for words that cannot be matched with vowel matching.
- *
- * @param {String} text         The text to count the syllables of.
- * @param {Object} syllables    The syllables data for the specific language.
- *
- * @returns {int} The total number of syllables found in the text.
- */
-const countSyllablesInText = function( text, syllables ) {
-	text = text.toLocaleLowerCase();
-	const words = getWords( text );
-
-	const syllableCounts = map( words,  function( word ) {
-		return countSyllablesInWord( word, syllables );
-	} );
-
-	return sum( syllableCounts );
-};
-
-export default countSyllablesInText;

package/src/languageProcessing/helpers/syllables/syllableCountIterator.js

@@ -1,56 +0,0 @@
-import SyllableCountStep from "./syllableCountStep.js";
-
-import { forEach, isUndefined } from "lodash";
-
-/**
- * A SyllableCountIterator contains individual SyllableCountSteps.
- */
-export default class SyllableCountIterator {
-	/**
-	 * Creates a syllable count iterator.
-	 *
-	 * @param {object} config The config object containing an array with syllable exclusions.
-	 * @constructor
-	 */
-	constructor( config ) {
-		this.countSteps = [];
-		if ( ! isUndefined( config ) ) {
-			this.createSyllableCountSteps( config.deviations.vowels );
-		}
-	}
-
-	/**
-	 * Creates a syllable count step object for each exclusion.
-	 *
-	 * @param {object} syllableCounts The object containing all exclusion syllables including the multipliers.
-	 * @returns {void}
-	 */
-	createSyllableCountSteps( syllableCounts ) {
-		forEach( syllableCounts, function( syllableCountStep ) {
-			this.countSteps.push( new SyllableCountStep( syllableCountStep ) );
-		}.bind( this ) );
-	}
-
-	/**
-	 * Returns all available count steps.
-	 *
-	 * @returns {Array} All available count steps.
-	 */
-	getAvailableSyllableCountSteps() {
-		return this.countSteps;
-	}
-
-	/**
-	 * Counts the syllables for all the steps and returns the total syllable count.
-	 *
-	 * @param {String} word The word to count syllables in.
-	 * @returns {number} The number of syllables found based on exclusions.
-	 */
-	countSyllables( word ) {
-		let syllableCount = 0;
-		forEach( this.countSteps, function( step ) {
-			syllableCount += step.countSyllables( word );
-		} );
-		return syllableCount;
-	}
-}

package/src/languageProcessing/helpers/syllables/syllableCountStep.js

@@ -1,68 +0,0 @@
-import { isUndefined } from "lodash";
-
-import arrayToRegex from "../regex/createRegexFromArray.js";
-
-/**
- * A SyllableCountStep is an individual step in a SyllableCountIterator.
- */
-export default class SyllableCountStep {
-	/**
-	 * Constructs a language syllable regex that contains a regex for matching syllable exclusion.
-	 *
-	 * @param {object} syllableRegex The object containing the syllable exclusions.
-	 * @constructor
-	 */
-	constructor( syllableRegex ) {
-		this._hasRegex = false;
-		this._regex = "";
-		this._multiplier = "";
-		this.createRegex( syllableRegex );
-	}
-
-	/**
-	 * Checks whether a valid regex has been set.
-	 *
-	 * @returns {boolean} True if a regex has been set, false if not.
-	 */
-	hasRegex() {
-		return this._hasRegex;
-	}
-
-	/**
-	 * Creates a regex based on the given syllable exclusions, and sets the multiplier to use.
-	 *
-	 * @param {object} syllableRegex The object containing the syllable exclusions and multiplier.
-	 * @returns {void}
-	 */
-	createRegex( syllableRegex ) {
-		if ( ! isUndefined( syllableRegex ) && ! isUndefined( syllableRegex.fragments ) ) {
-			this._hasRegex = true;
-			this._regex = arrayToRegex( syllableRegex.fragments, true );
-			this._multiplier = syllableRegex.countModifier;
-		}
-	}
-
-	/**
-	 * Returns the stored regular expression.
-	 *
-	 * @returns {RegExp} The stored regular expression.
-	 */
-	getRegex() {
-		return this._regex;
-	}
-
-	/**
-	 * Matches syllable exclusions in a given word and returns the number found multiplied by the
-	 * given multiplier. The result of this multiplication is the syllable count.
-	 *
-	 * @param {String} word The word to match for syllable exclusions.
-	 * @returns {number} The amount of syllables found.
-	 */
-	countSyllables( word ) {
-		if ( this._hasRegex ) {
-			const match = word.match( this._regex ) || [];
-			return match.length * this._multiplier;
-		}
-		return 0;
-	}
-}

package/src/languageProcessing/helpers/transform/transformWordsWithHyphens.js

@@ -1,17 +0,0 @@
-/**
- * Splits words from an array that contain hyphens and adds the elements to the array if they are not yet there.
- *
- * @param {Array} array The array to check and modify.
- *
- * @returns {Array} A new array with the elements containing hyphens split.
- */
-export default function( array ) {
-	let newArray = array;
-	array.forEach( element => {
-		element = element.split( "-" );
-		if ( element.length > 0 && element.filter( unit => ! array.includes( unit ) ).length > 0 ) {
-			newArray = newArray.concat( element );
-		}
-	} );
-	return newArray;
-}

package/src/languageProcessing/helpers/transliterate/replaceDiacritics.js

@@ -1,22 +0,0 @@
-/** @module stringProcessing/replaceDiacritics */
-
-import diacriticsRemovalMap from "../../../config/diacritics.js";
-
-/**
- * Replaces all diacritics from the text based on the diacritics removal map.
- *
- * @param {string} text The text to remove diacritics from.
- *
- * @returns {string} The text with all diacritics replaced.
- */
-export default function( text ) {
-	const map = diacriticsRemovalMap();
-
-	for ( let i = 0; i < map.length; i++ ) {
-		text = text.replace(
-			map[ i ].letters,
-			map[ i ].base
-		);
-	}
-	return text;
-}