axyseo 2.0.0-alpha.0.0.20 → 2.0.0-alpha.0.0.4

package/src/languageProcessing/helpers/transliterate/specialCharacterMappings.js

@@ -0,0 +1,214 @@
+import { filter, includes, isEqual, memoize } from "lodash";
+import getWords from "../word/getWords";
+
+/**
+ * Gets positions of the first character of a word in the input text.
+ *
+ * @param {string} text The original text, for which the indices of word beginnings need to be determined.
+ *
+ * @returns {Array} indices The array of indices in the text at which words start.
+ */
+function getIndicesOfWords( text ) {
+	const indices = [];
+	const words = getWords( text );
+	let startSearchFrom = 0;
+
+	words.forEach( function( word ) {
+		const currentIndex = text.indexOf( word, startSearchFrom );
+		indices.push( currentIndex );
+		startSearchFrom = currentIndex + word.length;
+	} );
+
+	return indices;
+}
+
+
+/**
+ * Gets indices of a specific character in the input text.
+ *
+ * @param {string} text             The original text, for which the indices of specific characters have to be determined.
+ * @param {string} characterToFind  The character that needs to be found in the text.
+ *
+ * @returns {Array} indices The array of indices in the text at which the characterToFind occurs.
+ */
+function getIndicesOfCharacter( text, characterToFind ) {
+	const indices = [];
+
+	if ( text.indexOf( characterToFind ) > -1 ) {
+		for ( let i = 0; i < text.length; i++ ) {
+			if ( text[ i ] === characterToFind ) {
+				indices.push( i );
+			}
+		}
+	}
+
+	return indices;
+}
+
+/**
+ * Compares two arrays of which the second array is the sub-array of the first array.
+ * Returns the array of elements of the first array which are not in the second array.
+ *
+ * @param {Array} bigArray The array with all elements.
+ * @param {Array} subarray The array with some elements from the bigArray.
+ *
+ * @returns {Array} difference An array of all elements of bigArray which are not in subarray.
+ */
+function arraysDifference( bigArray, subarray ) {
+	return filter( bigArray, function( element ) {
+		return ( ! includes( subarray, element ) );
+	} );
+}
+
+/**
+ * Compares two arrays and returns the array of elements that occur in both arrays.
+ *
+ * @param {Array} firstArray    The first array with elements to compare.
+ * @param {Array} secondArray   The second array with elements to compare.
+ *
+ * @returns {Array} overlap An array of all elements of firstArray which are also in secondArray.
+ */
+function arraysOverlap( firstArray, secondArray ) {
+	return filter( firstArray, function( element ) {
+		return ( includes( secondArray, element ) );
+	} );
+}
+
+
+/**
+ * Generates all possible combinations of the elements of an array (treated as unique).
+ * https://gist.github.com/jpillora/4435759
+ *
+ * @param {Array} collection The array with elements that should be combined.
+ *
+ * @returns {Array} result An array of all possible combinations of elements of the original array.
+ */
+function combinations( collection ) {
+	/**
+	 * A recursive function that iterates through all elements of an array to produce its combinations.
+	 *
+	 * @param {Array} xs        The array to start accumulating with.
+	 * @param {Array} array     The array with elements that should be combined.
+	 *
+	 * @returns {Array} result An array of all possible combinations of elements of the original array.
+	 */
+	function acc( xs, array ) {
+		const x = xs[ 0 ];
+
+		if ( typeof x === "undefined" ) {
+			return array;
+		}
+
+		for ( let i = 0, l = array.length; i < l; ++i ) {
+			array.push( array[ i ].concat( x ) );
+		}
+		return acc( xs.slice( 1 ), array );
+	}
+	return acc( collection, [ [ ] ] ).slice( 1 ).concat( [ [] ] );
+}
+
+
+/**
+ * Replaces characters on specified indices in the input text.
+ *
+ * @param {string} text         The original text, for which the characters have to be substituted.
+ * @param {Array} indices       The array of indices that have to be substituted.
+ * @param {string} substitute   The character that is used to substitute in the text.
+ *
+ * @returns {string} result The string of the original text with the characters on specified indices are substituted with the substitute character.
+ */
+function replaceCharactersByIndex( text, indices, substitute ) {
+	const modifiedTextSplitByLetter = text.split( "" );
+
+	indices.forEach( function( index ) {
+		modifiedTextSplitByLetter.splice( index, 1, substitute );
+	} );
+
+	return modifiedTextSplitByLetter.join( "" );
+}
+
+/**
+ * Generates upper and lower case for Turkish strings that contain characters İ or ı, which appear to not be processed correctly by regexes.
+ *
+ * @param {string} text The text to build possible upper and lower case alternatives.
+ *
+ * @returns {Array} An array of strings that contains all possible upper and lower case alternatives of the original string
+ */
+function replaceTurkishIs( text ) {
+	// Get indices of all occurrences of İ, I, i, or ı.
+	const indicesOfAllIs = getIndicesOfCharacter( text, "İ" ).concat(
+		getIndicesOfCharacter( text, "I" ),
+		getIndicesOfCharacter( text, "i" ),
+		getIndicesOfCharacter( text, "ı" )
+	);
+	indicesOfAllIs.sort();
+
+	// If there are no Is return the text
+	if ( indicesOfAllIs.length === 0 ) {
+		return [ text ];
+	}
+	const indicesOfIsInWordBeginnings = arraysOverlap( getIndicesOfWords( text ), indicesOfAllIs );
+
+	const results = [];
+
+	// First round of creating combinations: assign which indices will be replaced by İ
+	const combinationsDottedI = combinations( indicesOfIsInWordBeginnings );
+
+	combinationsDottedI.forEach( function( oneCombinationDottedI ) {
+		// If the combination is full array, just add it to results immediately without going through the rest of iterations.
+		if ( isEqual( oneCombinationDottedI, indicesOfIsInWordBeginnings ) ) {
+			results.push( [ oneCombinationDottedI, [], [], [] ] );
+		} else {
+			const indicesNotDottedI = arraysDifference( indicesOfIsInWordBeginnings, oneCombinationDottedI );
+
+			// Second round of creating combinations: assign which indices will be replaced by I
+			const combinationsDotlessI = combinations( indicesNotDottedI );
+			combinationsDotlessI.forEach( function( oneCombinationDotlessI ) {
+				// If the combination is full array, just add it to results immediately without going through the rest of iterations.
+				if ( isEqual( oneCombinationDotlessI, indicesNotDottedI ) ) {
+					results.push( [ oneCombinationDottedI, oneCombinationDotlessI, [], [] ] );
+				} else {
+					const indicesSmalli = arraysDifference( indicesNotDottedI, oneCombinationDotlessI );
+
+					// Third round of creating combinations: assign which indices will be replaced by i
+					const combinationsDottedi = combinations( indicesSmalli );
+
+					combinationsDottedi.forEach( function( oneCombinationDottedi ) {
+						// If the combination is full array, just add it to results immediately without going through the rest of iterations.
+						if ( isEqual( oneCombinationDottedi, indicesSmalli ) ) {
+							results.push( [ oneCombinationDottedI, oneCombinationDotlessI, oneCombinationDottedi, [] ] );
+						} else {
+							const oneCombinationDotlessi = arraysDifference( indicesSmalli, oneCombinationDottedi );
+
+							results.push( [ oneCombinationDottedI, oneCombinationDotlessI, oneCombinationDottedi, oneCombinationDotlessi ] );
+						}
+					} );
+				}
+			} );
+		}
+	} );
+
+	const textAlternations = [];
+
+	results.forEach( function( result ) {
+		const toDottedI = replaceCharactersByIndex( text, result[ 0 ], "İ" );
+		const toDotlessI = replaceCharactersByIndex( toDottedI, result[ 1 ], "I" );
+		const toDottedi = replaceCharactersByIndex( toDotlessI, result[ 2 ], "i" );
+		const toDotlessi = replaceCharactersByIndex( toDottedi, result[ 3 ], "ı" );
+		textAlternations.push( toDotlessi );
+	} );
+
+	return textAlternations;
+}
+
+const replaceTurkishIsMemoized = memoize( replaceTurkishIs );
+
+export {
+	getIndicesOfWords,
+	getIndicesOfCharacter,
+	arraysDifference,
+	arraysOverlap,
+	combinations,
+	replaceTurkishIs,
+	replaceTurkishIsMemoized,
+};

package/src/languageProcessing/helpers/transliterate/transliterate.js

@@ -0,0 +1,20 @@
+import transliterationsMap from "../../../config/getTransliterations.js";
+
+/**
+ * Replaces all special characters from the text based on the transliterations map.
+ *
+ * @param {string} text     The text to remove special characters from.
+ * @param {string} locale   The locale.
+ *
+ * @returns {string} The text with all special characters replaced.
+ */
+export default function( text, locale ) {
+	const map = transliterationsMap( locale );
+	for ( let i = 0; i < map.length; i++ ) {
+		text = text.replace(
+			map[ i ].letter,
+			map[ i ].alternative
+		);
+	}
+	return text;
+}

package/src/languageProcessing/helpers/transliterate/transliterateWPstyle.js

@@ -0,0 +1,21 @@
+import transliterationsMap from "../../../config/transliterationsWPstyle.js";
+
+/**
+ * Replaces all special characters from the text based on the transliterations map specific for WP
+ * (and different from the one used by YoastSEO.js).
+ *
+ * @param {string} text     The text to remove special characters from.
+ * @param {string} locale   The locale.
+ *
+ * @returns {string} The text with all special characters replaced.
+ */
+export default function( text, locale ) {
+	const map = transliterationsMap( locale );
+	for ( let i = map.length - 1; i >= 0; i-- ) {
+		text = text.replace(
+			map[ i ].letter,
+			map[ i ].alternative
+		);
+	}
+	return text;
+}

package/src/languageProcessing/helpers/url/parseSlug.js

@@ -0,0 +1,10 @@
+/**
+ * Parses the slug by transforming hyphens and underscores into white space.
+ *
+ * @param {string} slug The slug to parse
+ *
+ * @returns {string} The parsed slug.
+ */
+export default function( slug ) {
+	return slug.replace( /[-_]/ig, " " );
+}

package/src/languageProcessing/helpers/url/url.js

@@ -0,0 +1,172 @@
+const urlFromAnchorRegex = /href=(["'])([^"']+)\1/i;
+import urlMethods from "url";
+
+/**
+ * Removes a hash from a URL, assumes a well formed URL.
+ *
+ * @param {string} url The URL to remove a hash from.
+ * @returns {string} The URL without the hash.
+ */
+function removeHash( url ) {
+	return url.split( "#" )[ 0 ];
+}
+
+/**
+ * Removes all query args from a URL, assumes a well formed URL.
+ *
+ * @param {string} url The URL to remove the query args from.
+ * @returns {string} The URL without the query args.
+ */
+function removeQueryArgs( url ) {
+	return url.split( "?" )[ 0 ];
+}
+
+/**
+ * Removes the trailing slash of a URL.
+ *
+ * @param {string} url The URL to remove the trailing slash from.
+ * @returns {string} A URL without a trailing slash.
+ */
+function removeTrailingSlash( url ) {
+	return url.replace( /\/$/, "" );
+}
+
+/**
+ * Adds a trailing slash to a URL if it is not present.
+ *
+ * @param {string} url The URL to add a trailing slash to.
+ * @returns {string} A URL with a trailing slash.
+ */
+function addTrailingSlash( url ) {
+	return removeTrailingSlash( url ) + "/";
+}
+
+/**
+ * Retrieves the URL from an anchor tag.
+ *
+ * @param {string} anchorTag An anchor tag.
+ * @returns {string} The URL in the anchor tag.
+ */
+function getFromAnchorTag( anchorTag ) {
+	const urlMatch = urlFromAnchorRegex.exec( anchorTag );
+
+	return ( urlMatch === null ) ? "" : urlMatch[ 2 ];
+}
+
+/**
+ * Returns whether or not the given URLs are equal.
+ *
+ * @param {string} urlA The first URL to compare.
+ * @param {string} urlB The second URL to compare.
+ *
+ * @returns {boolean} Whether or not the given URLs are equal.
+ */
+function areEqual( urlA, urlB ) {
+	// Make sure we are comparing URLs without query arguments and hashes.
+	urlA = removeQueryArgs( removeHash( urlA ) );
+	urlB = removeQueryArgs( removeHash( urlB ) );
+
+	return addTrailingSlash( urlA ) === addTrailingSlash( urlB );
+}
+
+/**
+ * Returns the domain name of a URL.
+ *
+ * @param {string} url The URL to retrieve the domain name of.
+ * @returns {string} The domain name of the URL.
+ */
+function getHostname( url ) {
+	url = urlMethods.parse( url );
+
+	return url.hostname;
+}
+
+/**
+ * Returns the protocol of a URL.
+ *
+ * Note that the colon (http:) is also part of the protocol, conform to node's url.parse api.
+ *
+ * @param {string} url The URL to retrieve the protocol of.
+ * @returns {string|null} The protocol of the URL or null if no protocol is present.
+ */
+function getProtocol( url ) {
+	return urlMethods.parse( url ).protocol;
+}
+
+/**
+ * Determine whether an anchor URL is internal.
+ *
+ * @param {string} anchorUrl 		The anchor URL to test.
+ * @param {string} siteUrlOrDomain  The current site's URL or domain.
+ *
+ * @returns {boolean} Whether or not the anchor URL is internal.
+ */
+function isInternalLink( anchorUrl, siteUrlOrDomain ) {
+	const parsedAnchorUrl = urlMethods.parse( anchorUrl, false, true );
+	const anchorUrlHostName = parsedAnchorUrl.hostname;
+
+	// Check if the anchor URL starts with a single slash.
+	if ( anchorUrl.indexOf( "//" ) === -1 && anchorUrl.indexOf( "/" ) === 0 ) {
+		return true;
+	}
+
+	// Check if the anchor URL starts with a # indicating a fragment.
+	if ( anchorUrl.indexOf( "#" ) === 0 ) {
+		return false;
+	}
+
+	// No host of the anchor URL indicates an internal link.
+	if ( ! anchorUrlHostName ) {
+		return true;
+	}
+
+	// If the siteUrlOrDomain variable is a domain, it would be idential to the anchor URL's hostname in case of an internal link.
+	if ( anchorUrlHostName === siteUrlOrDomain ) {
+		return true;
+	}
+
+	// If the siteUrlOrDomain variable is a URL and it shares the hostname with the anchor URL, it's an internal link.
+	const parsedSiteUrlOrDomain = urlMethods.parse( siteUrlOrDomain );
+	const siteUrlOrDomainHostName = parsedSiteUrlOrDomain.hostname;
+	return anchorUrlHostName === siteUrlOrDomainHostName;
+}
+
+/**
+ * Checks whether the protocol is either HTTP: or HTTPS:.
+ *
+ * @param {string} protocol The protocol to test.
+ *
+ * @returns {boolean} Whether the protocol is http(s):.
+ */
+function protocolIsHttpScheme( protocol ) {
+	if ( ! protocol ) {
+		return false;
+	}
+
+	return ( protocol === "http:" || protocol === "https:" );
+}
+
+/**
+ * Determines whether the link is a relative fragment URL.
+ *
+ * @param {string} url The URL to test.
+ *
+ * @returns {boolean} Whether the link is a relative fragment URL.
+ */
+function isRelativeFragmentURL( url ) {
+	return url.indexOf( "#" ) === 0;
+}
+
+export default {
+	removeHash: removeHash,
+	removeQueryArgs: removeQueryArgs,
+	removeTrailingSlash: removeTrailingSlash,
+	addTrailingSlash: addTrailingSlash,
+	getFromAnchorTag: getFromAnchorTag,
+	areEqual: areEqual,
+	getHostname: getHostname,
+	getProtocol: getProtocol,
+	isInternalLink: isInternalLink,
+	protocolIsHttpScheme: protocolIsHttpScheme,
+	isRelativeFragmentURL: isRelativeFragmentURL,
+};

package/src/languageProcessing/helpers/word/addWordboundary.js

@@ -0,0 +1,37 @@
+/**
+ * Returns a string that can be used in a regex to match a matchString with word boundaries.
+ *
+ * @param {string}  matchString                 The string to generate a regex string for.
+ * @param {boolean} [positiveLookAhead=false]   Boolean indicating whether or not to include a positive look ahead
+ * for the word boundaries at the end.
+ * @param {string} [extraWordBoundary=""]       Extra characters to match a word boundary on.
+ * @param {string} [language=""]                The language used to determine the word boundary.
+ *
+ * @returns {string} A regex string that matches the matchString with word boundaries.
+ */
+export default function( matchString, positiveLookAhead = false, extraWordBoundary = "", language = "" ) {
+	let wordBoundary, wordBoundaryEnd;
+
+	if ( language === "id" ) {
+		wordBoundary = "[ \\u00a0\\n\\r\\t.,()”“〝〞〟‟„\"+;!¡?¿:/»«‹›" + extraWordBoundary + "<>";
+	} else {
+		/*
+		 * \u00a0 - no-break space
+		 * \u2014 - em dash
+         * \u06d4 - Urdu full stop
+         * \u061f - Arabic question mark
+         * \u060C - Arabic comma
+         * \u061B - Arabic semicolon
+         */
+		wordBoundary = "[ \\u00a0\\u2014\\u06d4\\u061f\\u060C\\u061B\\n\\r\\t.,()”“〝〞〟‟„\"+\\-;!¡?¿:/»«‹›" + extraWordBoundary + "<>";
+	}
+
+	const wordBoundaryStart = "(^|" + wordBoundary + "'‘’‛`])";
+	if ( positiveLookAhead ) {
+		wordBoundaryEnd = "($|((?=" + wordBoundary + "]))|((['‘’‛`])(" + wordBoundary + "])))";
+	} else {
+		wordBoundaryEnd = "($|(" + wordBoundary + "])|((['‘’‛`])(" + wordBoundary + "])))";
+	}
+
+	return wordBoundaryStart + matchString + wordBoundaryEnd;
+}

package/src/languageProcessing/helpers/word/areWordsInSentence.js

@@ -0,0 +1,16 @@
+import { filter } from "lodash";
+import getWords from "./getWords.js";
+
+/**
+ * Checks if one or more words of a list occur in a sentence.
+ *
+ * @param {Array} words The list of words.
+ * @param {string} sentence The sentence to match against.
+ * @returns {Boolean} Whether the sentence contains the word.
+ */
+export default function areWordsInSentence( words, sentence ) {
+	const matches = filter( getWords( sentence ), function( word ) {
+		return words.includes( word.toLocaleLowerCase() );
+	} );
+	return matches.length !== 0;
+}

package/src/languageProcessing/helpers/word/countMetaDescriptionLength.js

@@ -0,0 +1,18 @@
+/**
+ * Check the length of the description.
+ *
+ * @param {string}  date          The date.
+ * @param {string}  description   The meta description.
+ *
+ * @returns {number} The length of the description.
+ */
+export default function( date, description ) {
+	let descriptionLength = description.length;
+	/* If the meta description is preceded by a date, two spaces and a hyphen (" - ") are added as well. Therefore,
+	three needs to be added to the total length. */
+	if ( date !== "" && descriptionLength > 0 ) {
+		descriptionLength += date.length + 3;
+	}
+
+	return descriptionLength;
+}

package/src/languageProcessing/helpers/word/countWords.js

@@ -0,0 +1,14 @@
+/** @module stringProcessing/countWords */
+
+import getWords from "./getWords.js";
+
+/**
+ * Calculates the word count of a certain text.
+ *
+ * @param {string} text The text to be counted.
+ *
+ * @returns {number} The word count of the given text.
+ */
+export default function( text ) {
+	return getWords( text ).length;
+}

package/src/languageProcessing/helpers/word/createPunctuationTokens.js

@@ -0,0 +1,42 @@
+import { punctuationRegexEnd, punctuationRegexStart } from "../sanitize/removePunctuation";
+import { hashedHtmlEntitiesRegexEnd, hashedHtmlEntitiesRegexStart } from "../../../helpers/htmlEntities";
+
+/**
+ * Removes punctuation from the beginning and end of a word token, and creates separate tokens from them.
+ *
+ * @param {string[]} rawTokens	The tokens that may contain punctuation at the beginning and end of words.
+ *
+ * @returns {string[]} The tokens with the punctuation moved into separate tokens.
+ */
+const createPunctuationTokens = ( rawTokens ) => {
+	const tokenTexts = [];
+	rawTokens.forEach( token => {
+		// Pretokens contains all that occurs before the first letter of the token.
+		const preTokens = [];
+		// Posttokens contains all that occurs after the last letter of the token.
+		const postTokens = [];
+
+		// Add all punctuation marks that occur before the first letter of the token to the pretokens array.
+		// Also, prevent matching with a hashed HTML entity in the beginning of the token.
+		while ( punctuationRegexStart.test( token ) && ! hashedHtmlEntitiesRegexStart.test( token ) ) {
+			preTokens.push( token[ 0 ] );
+			token = token.slice( 1 );
+		}
+		// Add all punctuation marks that occur after the last letter of the token to the posttokens array.
+		// Also, prevent matching with a hashed HTML entity at the end of the token.
+		while ( punctuationRegexEnd.test( token ) && ! hashedHtmlEntitiesRegexEnd.test( token ) ) {
+			// Using unshift here because we are iterating from the end of the string to the beginning,
+			// and we want to keep the order of the punctuation marks.
+			// Therefore, we add them to the start of the array.
+			postTokens.unshift( token[ token.length - 1 ] );
+			token = token.slice( 0, -1 );
+		}
+
+		let currentTokens = [ ...preTokens, token, ...postTokens ];
+		currentTokens = currentTokens.filter( x => x !== "" );
+		tokenTexts.push( ...currentTokens );
+	} );
+	return tokenTexts;
+};
+
+export default createPunctuationTokens;

package/src/languageProcessing/helpers/word/filterWordsFromArray.js

@@ -0,0 +1,15 @@
+import { filter, includes } from "lodash";
+
+/**
+ * Filters certain words from an array of words.
+ *
+ * @param {Array} array The words to check.
+ * @param {Array} words The words to filter.
+ *
+ * @returns {Array} The original array with the certain words filtered out.
+ */
+export default function filterWordsFromArray( array, words = [] ) {
+	return filter( array, function( word ) {
+		return ( ! includes( words, word.trim().toLocaleLowerCase() ) );
+	} );
+}

package/src/languageProcessing/helpers/word/followsIndex.js

@@ -0,0 +1,25 @@
+import { isEmpty, forEach, includes } from "lodash";
+
+/**
+ * Checks whether a given word is followed by any word from a given list.
+ *
+ * @param {Array} followingWords    The array of objects with matches and indices.
+ * @param {Object} match            The object with the match and index to test the following words for.
+ *
+ * @returns {boolean} Returns true if the match is followed by a given word, otherwise returns false.
+ */
+export default function( followingWords, match ) {
+	if ( isEmpty( followingWords ) ) {
+		return false;
+	}
+
+	// The followingWordIndices include the preceding space.
+	const wordAfterMatchIndex = match.index + match.match.length;
+	const followingWordsIndices = [];
+
+	forEach( followingWords, function( followingWord ) {
+		followingWordsIndices.push( followingWord.index );
+	} );
+
+	return includes( followingWordsIndices, wordAfterMatchIndex );
+}

package/src/languageProcessing/helpers/word/getAllWordsFromTree.js

@@ -0,0 +1,23 @@
+import getSentencesFromTree from "../sentence/getSentencesFromTree";
+import { flatMap } from "lodash";
+import removePunctuation from "../sanitize/removePunctuation";
+
+/**
+ * Gets the words from the tree, i.e. from the paragraph and heading nodes.
+ * These two node types are the nodes that should contain words for the analysis.
+ *
+ * @param {Paper} paper The paper to get the tree and words from.
+ *
+ * @returns {String[]} Array of words retrieved from the tree.
+ */
+export default function( paper ) {
+	const sentences = getSentencesFromTree( paper );
+	// Get all the tokens from each sentence.
+	const tokens = sentences.map( sentence => sentence.tokens );
+	let words = flatMap( tokens ).map( token => token.text );
+	// Remove punctuation and spaces.
+	words = words.map( token => removePunctuation( token ) );
+
+	// Filter out empty tokens.
+	return words.filter( word => word.trim() !== "" );
+}

package/src/languageProcessing/helpers/word/getWords.js

@@ -0,0 +1,43 @@
+/** @module stringProcessing/countWords */
+import sanitizeString from "../sanitize/sanitizeString";
+import { filter, flatMap } from "lodash";
+import removePunctuation, { punctuationRegexString } from "../sanitize/removePunctuation.js";
+
+const punctuationRegex = new RegExp( `([${punctuationRegexString}])`, "g" );
+
+/**
+ * Returns an array with words used in the text.
+ *
+ * @param {string} text The text to be counted.
+ * @param {string} [wordBoundaryRegexString=\\s] The regex string for the word boundary that should be used to split the text into words.
+ * @param {boolean} [shouldRemovePunctuation=true] If punctuation should be removed. Defaults to `true`.
+ *
+ * @returns {Array} The array with all words.
+ */
+export default function( text, wordBoundaryRegexString = "\\s", shouldRemovePunctuation = true ) {
+	// Unify whitespaces and non-breaking spaces, remove table of content and strip the tags and multiple spaces.
+	text = sanitizeString( text );
+
+	if ( text === "" ) {
+		return [];
+	}
+
+	const wordBoundaryRegex = new RegExp( wordBoundaryRegexString, "g" );
+
+	let words = text.split( wordBoundaryRegex );
+
+	if ( shouldRemovePunctuation ) {
+		words = words.map( removePunctuation );
+	} else {
+		// If punctuation is not removed, punctuation marks are tokenized as if they were words.
+		words = flatMap( words, ( word ) => {
+			const newWord = word.replace( punctuationRegex, " $1 " );
+			return newWord.split( " " );
+		} );
+	}
+
+	return filter( words, function( word ) {
+		return word.trim() !== "";
+	} );
+}
+