axyseo 2.0.0-alpha.0.0.4 → 2.0.0-alpha.0.0.6

package/src/languageProcessing/helpers/transliterate/specialCharacterMappings.js

@@ -1,214 +0,0 @@
-import { filter, includes, isEqual, memoize } from "lodash";
-import getWords from "../word/getWords";
-
-/**
- * Gets positions of the first character of a word in the input text.
- *
- * @param {string} text The original text, for which the indices of word beginnings need to be determined.
- *
- * @returns {Array} indices The array of indices in the text at which words start.
- */
-function getIndicesOfWords( text ) {
-	const indices = [];
-	const words = getWords( text );
-	let startSearchFrom = 0;
-
-	words.forEach( function( word ) {
-		const currentIndex = text.indexOf( word, startSearchFrom );
-		indices.push( currentIndex );
-		startSearchFrom = currentIndex + word.length;
-	} );
-
-	return indices;
-}
-
-
-/**
- * Gets indices of a specific character in the input text.
- *
- * @param {string} text             The original text, for which the indices of specific characters have to be determined.
- * @param {string} characterToFind  The character that needs to be found in the text.
- *
- * @returns {Array} indices The array of indices in the text at which the characterToFind occurs.
- */
-function getIndicesOfCharacter( text, characterToFind ) {
-	const indices = [];
-
-	if ( text.indexOf( characterToFind ) > -1 ) {
-		for ( let i = 0; i < text.length; i++ ) {
-			if ( text[ i ] === characterToFind ) {
-				indices.push( i );
-			}
-		}
-	}
-
-	return indices;
-}
-
-/**
- * Compares two arrays of which the second array is the sub-array of the first array.
- * Returns the array of elements of the first array which are not in the second array.
- *
- * @param {Array} bigArray The array with all elements.
- * @param {Array} subarray The array with some elements from the bigArray.
- *
- * @returns {Array} difference An array of all elements of bigArray which are not in subarray.
- */
-function arraysDifference( bigArray, subarray ) {
-	return filter( bigArray, function( element ) {
-		return ( ! includes( subarray, element ) );
-	} );
-}
-
-/**
- * Compares two arrays and returns the array of elements that occur in both arrays.
- *
- * @param {Array} firstArray    The first array with elements to compare.
- * @param {Array} secondArray   The second array with elements to compare.
- *
- * @returns {Array} overlap An array of all elements of firstArray which are also in secondArray.
- */
-function arraysOverlap( firstArray, secondArray ) {
-	return filter( firstArray, function( element ) {
-		return ( includes( secondArray, element ) );
-	} );
-}
-
-
-/**
- * Generates all possible combinations of the elements of an array (treated as unique).
- * https://gist.github.com/jpillora/4435759
- *
- * @param {Array} collection The array with elements that should be combined.
- *
- * @returns {Array} result An array of all possible combinations of elements of the original array.
- */
-function combinations( collection ) {
-	/**
-	 * A recursive function that iterates through all elements of an array to produce its combinations.
-	 *
-	 * @param {Array} xs        The array to start accumulating with.
-	 * @param {Array} array     The array with elements that should be combined.
-	 *
-	 * @returns {Array} result An array of all possible combinations of elements of the original array.
-	 */
-	function acc( xs, array ) {
-		const x = xs[ 0 ];
-
-		if ( typeof x === "undefined" ) {
-			return array;
-		}
-
-		for ( let i = 0, l = array.length; i < l; ++i ) {
-			array.push( array[ i ].concat( x ) );
-		}
-		return acc( xs.slice( 1 ), array );
-	}
-	return acc( collection, [ [ ] ] ).slice( 1 ).concat( [ [] ] );
-}
-
-
-/**
- * Replaces characters on specified indices in the input text.
- *
- * @param {string} text         The original text, for which the characters have to be substituted.
- * @param {Array} indices       The array of indices that have to be substituted.
- * @param {string} substitute   The character that is used to substitute in the text.
- *
- * @returns {string} result The string of the original text with the characters on specified indices are substituted with the substitute character.
- */
-function replaceCharactersByIndex( text, indices, substitute ) {
-	const modifiedTextSplitByLetter = text.split( "" );
-
-	indices.forEach( function( index ) {
-		modifiedTextSplitByLetter.splice( index, 1, substitute );
-	} );
-
-	return modifiedTextSplitByLetter.join( "" );
-}
-
-/**
- * Generates upper and lower case for Turkish strings that contain characters İ or ı, which appear to not be processed correctly by regexes.
- *
- * @param {string} text The text to build possible upper and lower case alternatives.
- *
- * @returns {Array} An array of strings that contains all possible upper and lower case alternatives of the original string
- */
-function replaceTurkishIs( text ) {
-	// Get indices of all occurrences of İ, I, i, or ı.
-	const indicesOfAllIs = getIndicesOfCharacter( text, "İ" ).concat(
-		getIndicesOfCharacter( text, "I" ),
-		getIndicesOfCharacter( text, "i" ),
-		getIndicesOfCharacter( text, "ı" )
-	);
-	indicesOfAllIs.sort();
-
-	// If there are no Is return the text
-	if ( indicesOfAllIs.length === 0 ) {
-		return [ text ];
-	}
-	const indicesOfIsInWordBeginnings = arraysOverlap( getIndicesOfWords( text ), indicesOfAllIs );
-
-	const results = [];
-
-	// First round of creating combinations: assign which indices will be replaced by İ
-	const combinationsDottedI = combinations( indicesOfIsInWordBeginnings );
-
-	combinationsDottedI.forEach( function( oneCombinationDottedI ) {
-		// If the combination is full array, just add it to results immediately without going through the rest of iterations.
-		if ( isEqual( oneCombinationDottedI, indicesOfIsInWordBeginnings ) ) {
-			results.push( [ oneCombinationDottedI, [], [], [] ] );
-		} else {
-			const indicesNotDottedI = arraysDifference( indicesOfIsInWordBeginnings, oneCombinationDottedI );
-
-			// Second round of creating combinations: assign which indices will be replaced by I
-			const combinationsDotlessI = combinations( indicesNotDottedI );
-			combinationsDotlessI.forEach( function( oneCombinationDotlessI ) {
-				// If the combination is full array, just add it to results immediately without going through the rest of iterations.
-				if ( isEqual( oneCombinationDotlessI, indicesNotDottedI ) ) {
-					results.push( [ oneCombinationDottedI, oneCombinationDotlessI, [], [] ] );
-				} else {
-					const indicesSmalli = arraysDifference( indicesNotDottedI, oneCombinationDotlessI );
-
-					// Third round of creating combinations: assign which indices will be replaced by i
-					const combinationsDottedi = combinations( indicesSmalli );
-
-					combinationsDottedi.forEach( function( oneCombinationDottedi ) {
-						// If the combination is full array, just add it to results immediately without going through the rest of iterations.
-						if ( isEqual( oneCombinationDottedi, indicesSmalli ) ) {
-							results.push( [ oneCombinationDottedI, oneCombinationDotlessI, oneCombinationDottedi, [] ] );
-						} else {
-							const oneCombinationDotlessi = arraysDifference( indicesSmalli, oneCombinationDottedi );
-
-							results.push( [ oneCombinationDottedI, oneCombinationDotlessI, oneCombinationDottedi, oneCombinationDotlessi ] );
-						}
-					} );
-				}
-			} );
-		}
-	} );
-
-	const textAlternations = [];
-
-	results.forEach( function( result ) {
-		const toDottedI = replaceCharactersByIndex( text, result[ 0 ], "İ" );
-		const toDotlessI = replaceCharactersByIndex( toDottedI, result[ 1 ], "I" );
-		const toDottedi = replaceCharactersByIndex( toDotlessI, result[ 2 ], "i" );
-		const toDotlessi = replaceCharactersByIndex( toDottedi, result[ 3 ], "ı" );
-		textAlternations.push( toDotlessi );
-	} );
-
-	return textAlternations;
-}
-
-const replaceTurkishIsMemoized = memoize( replaceTurkishIs );
-
-export {
-	getIndicesOfWords,
-	getIndicesOfCharacter,
-	arraysDifference,
-	arraysOverlap,
-	combinations,
-	replaceTurkishIs,
-	replaceTurkishIsMemoized,
-};

package/src/languageProcessing/helpers/transliterate/transliterate.js

@@ -1,20 +0,0 @@
-import transliterationsMap from "../../../config/getTransliterations.js";
-
-/**
- * Replaces all special characters from the text based on the transliterations map.
- *
- * @param {string} text     The text to remove special characters from.
- * @param {string} locale   The locale.
- *
- * @returns {string} The text with all special characters replaced.
- */
-export default function( text, locale ) {
-	const map = transliterationsMap( locale );
-	for ( let i = 0; i < map.length; i++ ) {
-		text = text.replace(
-			map[ i ].letter,
-			map[ i ].alternative
-		);
-	}
-	return text;
-}

package/src/languageProcessing/helpers/transliterate/transliterateWPstyle.js

@@ -1,21 +0,0 @@
-import transliterationsMap from "../../../config/transliterationsWPstyle.js";
-
-/**
- * Replaces all special characters from the text based on the transliterations map specific for WP
- * (and different from the one used by YoastSEO.js).
- *
- * @param {string} text     The text to remove special characters from.
- * @param {string} locale   The locale.
- *
- * @returns {string} The text with all special characters replaced.
- */
-export default function( text, locale ) {
-	const map = transliterationsMap( locale );
-	for ( let i = map.length - 1; i >= 0; i-- ) {
-		text = text.replace(
-			map[ i ].letter,
-			map[ i ].alternative
-		);
-	}
-	return text;
-}

package/src/languageProcessing/helpers/url/parseSlug.js

@@ -1,10 +0,0 @@
-/**
- * Parses the slug by transforming hyphens and underscores into white space.
- *
- * @param {string} slug The slug to parse
- *
- * @returns {string} The parsed slug.
- */
-export default function( slug ) {
-	return slug.replace( /[-_]/ig, " " );
-}

package/src/languageProcessing/helpers/url/url.js

@@ -1,172 +0,0 @@
-const urlFromAnchorRegex = /href=(["'])([^"']+)\1/i;
-import urlMethods from "url";
-
-/**
- * Removes a hash from a URL, assumes a well formed URL.
- *
- * @param {string} url The URL to remove a hash from.
- * @returns {string} The URL without the hash.
- */
-function removeHash( url ) {
-	return url.split( "#" )[ 0 ];
-}
-
-/**
- * Removes all query args from a URL, assumes a well formed URL.
- *
- * @param {string} url The URL to remove the query args from.
- * @returns {string} The URL without the query args.
- */
-function removeQueryArgs( url ) {
-	return url.split( "?" )[ 0 ];
-}
-
-/**
- * Removes the trailing slash of a URL.
- *
- * @param {string} url The URL to remove the trailing slash from.
- * @returns {string} A URL without a trailing slash.
- */
-function removeTrailingSlash( url ) {
-	return url.replace( /\/$/, "" );
-}
-
-/**
- * Adds a trailing slash to a URL if it is not present.
- *
- * @param {string} url The URL to add a trailing slash to.
- * @returns {string} A URL with a trailing slash.
- */
-function addTrailingSlash( url ) {
-	return removeTrailingSlash( url ) + "/";
-}
-
-/**
- * Retrieves the URL from an anchor tag.
- *
- * @param {string} anchorTag An anchor tag.
- * @returns {string} The URL in the anchor tag.
- */
-function getFromAnchorTag( anchorTag ) {
-	const urlMatch = urlFromAnchorRegex.exec( anchorTag );
-
-	return ( urlMatch === null ) ? "" : urlMatch[ 2 ];
-}
-
-/**
- * Returns whether or not the given URLs are equal.
- *
- * @param {string} urlA The first URL to compare.
- * @param {string} urlB The second URL to compare.
- *
- * @returns {boolean} Whether or not the given URLs are equal.
- */
-function areEqual( urlA, urlB ) {
-	// Make sure we are comparing URLs without query arguments and hashes.
-	urlA = removeQueryArgs( removeHash( urlA ) );
-	urlB = removeQueryArgs( removeHash( urlB ) );
-
-	return addTrailingSlash( urlA ) === addTrailingSlash( urlB );
-}
-
-/**
- * Returns the domain name of a URL.
- *
- * @param {string} url The URL to retrieve the domain name of.
- * @returns {string} The domain name of the URL.
- */
-function getHostname( url ) {
-	url = urlMethods.parse( url );
-
-	return url.hostname;
-}
-
-/**
- * Returns the protocol of a URL.
- *
- * Note that the colon (http:) is also part of the protocol, conform to node's url.parse api.
- *
- * @param {string} url The URL to retrieve the protocol of.
- * @returns {string|null} The protocol of the URL or null if no protocol is present.
- */
-function getProtocol( url ) {
-	return urlMethods.parse( url ).protocol;
-}
-
-/**
- * Determine whether an anchor URL is internal.
- *
- * @param {string} anchorUrl 		The anchor URL to test.
- * @param {string} siteUrlOrDomain  The current site's URL or domain.
- *
- * @returns {boolean} Whether or not the anchor URL is internal.
- */
-function isInternalLink( anchorUrl, siteUrlOrDomain ) {
-	const parsedAnchorUrl = urlMethods.parse( anchorUrl, false, true );
-	const anchorUrlHostName = parsedAnchorUrl.hostname;
-
-	// Check if the anchor URL starts with a single slash.
-	if ( anchorUrl.indexOf( "//" ) === -1 && anchorUrl.indexOf( "/" ) === 0 ) {
-		return true;
-	}
-
-	// Check if the anchor URL starts with a # indicating a fragment.
-	if ( anchorUrl.indexOf( "#" ) === 0 ) {
-		return false;
-	}
-
-	// No host of the anchor URL indicates an internal link.
-	if ( ! anchorUrlHostName ) {
-		return true;
-	}
-
-	// If the siteUrlOrDomain variable is a domain, it would be idential to the anchor URL's hostname in case of an internal link.
-	if ( anchorUrlHostName === siteUrlOrDomain ) {
-		return true;
-	}
-
-	// If the siteUrlOrDomain variable is a URL and it shares the hostname with the anchor URL, it's an internal link.
-	const parsedSiteUrlOrDomain = urlMethods.parse( siteUrlOrDomain );
-	const siteUrlOrDomainHostName = parsedSiteUrlOrDomain.hostname;
-	return anchorUrlHostName === siteUrlOrDomainHostName;
-}
-
-/**
- * Checks whether the protocol is either HTTP: or HTTPS:.
- *
- * @param {string} protocol The protocol to test.
- *
- * @returns {boolean} Whether the protocol is http(s):.
- */
-function protocolIsHttpScheme( protocol ) {
-	if ( ! protocol ) {
-		return false;
-	}
-
-	return ( protocol === "http:" || protocol === "https:" );
-}
-
-/**
- * Determines whether the link is a relative fragment URL.
- *
- * @param {string} url The URL to test.
- *
- * @returns {boolean} Whether the link is a relative fragment URL.
- */
-function isRelativeFragmentURL( url ) {
-	return url.indexOf( "#" ) === 0;
-}
-
-export default {
-	removeHash: removeHash,
-	removeQueryArgs: removeQueryArgs,
-	removeTrailingSlash: removeTrailingSlash,
-	addTrailingSlash: addTrailingSlash,
-	getFromAnchorTag: getFromAnchorTag,
-	areEqual: areEqual,
-	getHostname: getHostname,
-	getProtocol: getProtocol,
-	isInternalLink: isInternalLink,
-	protocolIsHttpScheme: protocolIsHttpScheme,
-	isRelativeFragmentURL: isRelativeFragmentURL,
-};

package/src/languageProcessing/helpers/word/addWordboundary.js

@@ -1,37 +0,0 @@
-/**
- * Returns a string that can be used in a regex to match a matchString with word boundaries.
- *
- * @param {string}  matchString                 The string to generate a regex string for.
- * @param {boolean} [positiveLookAhead=false]   Boolean indicating whether or not to include a positive look ahead
- * for the word boundaries at the end.
- * @param {string} [extraWordBoundary=""]       Extra characters to match a word boundary on.
- * @param {string} [language=""]                The language used to determine the word boundary.
- *
- * @returns {string} A regex string that matches the matchString with word boundaries.
- */
-export default function( matchString, positiveLookAhead = false, extraWordBoundary = "", language = "" ) {
-	let wordBoundary, wordBoundaryEnd;
-
-	if ( language === "id" ) {
-		wordBoundary = "[ \\u00a0\\n\\r\\t.,()”“〝〞〟‟„\"+;!¡?¿:/»«‹›" + extraWordBoundary + "<>";
-	} else {
-		/*
-		 * \u00a0 - no-break space
-		 * \u2014 - em dash
-         * \u06d4 - Urdu full stop
-         * \u061f - Arabic question mark
-         * \u060C - Arabic comma
-         * \u061B - Arabic semicolon
-         */
-		wordBoundary = "[ \\u00a0\\u2014\\u06d4\\u061f\\u060C\\u061B\\n\\r\\t.,()”“〝〞〟‟„\"+\\-;!¡?¿:/»«‹›" + extraWordBoundary + "<>";
-	}
-
-	const wordBoundaryStart = "(^|" + wordBoundary + "'‘’‛`])";
-	if ( positiveLookAhead ) {
-		wordBoundaryEnd = "($|((?=" + wordBoundary + "]))|((['‘’‛`])(" + wordBoundary + "])))";
-	} else {
-		wordBoundaryEnd = "($|(" + wordBoundary + "])|((['‘’‛`])(" + wordBoundary + "])))";
-	}
-
-	return wordBoundaryStart + matchString + wordBoundaryEnd;
-}

package/src/languageProcessing/helpers/word/areWordsInSentence.js

@@ -1,16 +0,0 @@
-import { filter } from "lodash";
-import getWords from "./getWords.js";
-
-/**
- * Checks if one or more words of a list occur in a sentence.
- *
- * @param {Array} words The list of words.
- * @param {string} sentence The sentence to match against.
- * @returns {Boolean} Whether the sentence contains the word.
- */
-export default function areWordsInSentence( words, sentence ) {
-	const matches = filter( getWords( sentence ), function( word ) {
-		return words.includes( word.toLocaleLowerCase() );
-	} );
-	return matches.length !== 0;
-}

package/src/languageProcessing/helpers/word/countMetaDescriptionLength.js

@@ -1,18 +0,0 @@
-/**
- * Check the length of the description.
- *
- * @param {string}  date          The date.
- * @param {string}  description   The meta description.
- *
- * @returns {number} The length of the description.
- */
-export default function( date, description ) {
-	let descriptionLength = description.length;
-	/* If the meta description is preceded by a date, two spaces and a hyphen (" - ") are added as well. Therefore,
-	three needs to be added to the total length. */
-	if ( date !== "" && descriptionLength > 0 ) {
-		descriptionLength += date.length + 3;
-	}
-
-	return descriptionLength;
-}

package/src/languageProcessing/helpers/word/countWords.js

@@ -1,14 +0,0 @@
-/** @module stringProcessing/countWords */
-
-import getWords from "./getWords.js";
-
-/**
- * Calculates the word count of a certain text.
- *
- * @param {string} text The text to be counted.
- *
- * @returns {number} The word count of the given text.
- */
-export default function( text ) {
-	return getWords( text ).length;
-}

package/src/languageProcessing/helpers/word/createPunctuationTokens.js

@@ -1,42 +0,0 @@
-import { punctuationRegexEnd, punctuationRegexStart } from "../sanitize/removePunctuation";
-import { hashedHtmlEntitiesRegexEnd, hashedHtmlEntitiesRegexStart } from "../../../helpers/htmlEntities";
-
-/**
- * Removes punctuation from the beginning and end of a word token, and creates separate tokens from them.
- *
- * @param {string[]} rawTokens	The tokens that may contain punctuation at the beginning and end of words.
- *
- * @returns {string[]} The tokens with the punctuation moved into separate tokens.
- */
-const createPunctuationTokens = ( rawTokens ) => {
-	const tokenTexts = [];
-	rawTokens.forEach( token => {
-		// Pretokens contains all that occurs before the first letter of the token.
-		const preTokens = [];
-		// Posttokens contains all that occurs after the last letter of the token.
-		const postTokens = [];
-
-		// Add all punctuation marks that occur before the first letter of the token to the pretokens array.
-		// Also, prevent matching with a hashed HTML entity in the beginning of the token.
-		while ( punctuationRegexStart.test( token ) && ! hashedHtmlEntitiesRegexStart.test( token ) ) {
-			preTokens.push( token[ 0 ] );
-			token = token.slice( 1 );
-		}
-		// Add all punctuation marks that occur after the last letter of the token to the posttokens array.
-		// Also, prevent matching with a hashed HTML entity at the end of the token.
-		while ( punctuationRegexEnd.test( token ) && ! hashedHtmlEntitiesRegexEnd.test( token ) ) {
-			// Using unshift here because we are iterating from the end of the string to the beginning,
-			// and we want to keep the order of the punctuation marks.
-			// Therefore, we add them to the start of the array.
-			postTokens.unshift( token[ token.length - 1 ] );
-			token = token.slice( 0, -1 );
-		}
-
-		let currentTokens = [ ...preTokens, token, ...postTokens ];
-		currentTokens = currentTokens.filter( x => x !== "" );
-		tokenTexts.push( ...currentTokens );
-	} );
-	return tokenTexts;
-};
-
-export default createPunctuationTokens;

package/src/languageProcessing/helpers/word/filterWordsFromArray.js

@@ -1,15 +0,0 @@
-import { filter, includes } from "lodash";
-
-/**
- * Filters certain words from an array of words.
- *
- * @param {Array} array The words to check.
- * @param {Array} words The words to filter.
- *
- * @returns {Array} The original array with the certain words filtered out.
- */
-export default function filterWordsFromArray( array, words = [] ) {
-	return filter( array, function( word ) {
-		return ( ! includes( words, word.trim().toLocaleLowerCase() ) );
-	} );
-}

package/src/languageProcessing/helpers/word/followsIndex.js

@@ -1,25 +0,0 @@
-import { isEmpty, forEach, includes } from "lodash";
-
-/**
- * Checks whether a given word is followed by any word from a given list.
- *
- * @param {Array} followingWords    The array of objects with matches and indices.
- * @param {Object} match            The object with the match and index to test the following words for.
- *
- * @returns {boolean} Returns true if the match is followed by a given word, otherwise returns false.
- */
-export default function( followingWords, match ) {
-	if ( isEmpty( followingWords ) ) {
-		return false;
-	}
-
-	// The followingWordIndices include the preceding space.
-	const wordAfterMatchIndex = match.index + match.match.length;
-	const followingWordsIndices = [];
-
-	forEach( followingWords, function( followingWord ) {
-		followingWordsIndices.push( followingWord.index );
-	} );
-
-	return includes( followingWordsIndices, wordAfterMatchIndex );
-}

package/src/languageProcessing/helpers/word/getAllWordsFromTree.js

@@ -1,23 +0,0 @@
-import getSentencesFromTree from "../sentence/getSentencesFromTree";
-import { flatMap } from "lodash";
-import removePunctuation from "../sanitize/removePunctuation";
-
-/**
- * Gets the words from the tree, i.e. from the paragraph and heading nodes.
- * These two node types are the nodes that should contain words for the analysis.
- *
- * @param {Paper} paper The paper to get the tree and words from.
- *
- * @returns {String[]} Array of words retrieved from the tree.
- */
-export default function( paper ) {
-	const sentences = getSentencesFromTree( paper );
-	// Get all the tokens from each sentence.
-	const tokens = sentences.map( sentence => sentence.tokens );
-	let words = flatMap( tokens ).map( token => token.text );
-	// Remove punctuation and spaces.
-	words = words.map( token => removePunctuation( token ) );
-
-	// Filter out empty tokens.
-	return words.filter( word => word.trim() !== "" );
-}

package/src/languageProcessing/helpers/word/getWords.js

@@ -1,43 +0,0 @@
-/** @module stringProcessing/countWords */
-import sanitizeString from "../sanitize/sanitizeString";
-import { filter, flatMap } from "lodash";
-import removePunctuation, { punctuationRegexString } from "../sanitize/removePunctuation.js";
-
-const punctuationRegex = new RegExp( `([${punctuationRegexString}])`, "g" );
-
-/**
- * Returns an array with words used in the text.
- *
- * @param {string} text The text to be counted.
- * @param {string} [wordBoundaryRegexString=\\s] The regex string for the word boundary that should be used to split the text into words.
- * @param {boolean} [shouldRemovePunctuation=true] If punctuation should be removed. Defaults to `true`.
- *
- * @returns {Array} The array with all words.
- */
-export default function( text, wordBoundaryRegexString = "\\s", shouldRemovePunctuation = true ) {
-	// Unify whitespaces and non-breaking spaces, remove table of content and strip the tags and multiple spaces.
-	text = sanitizeString( text );
-
-	if ( text === "" ) {
-		return [];
-	}
-
-	const wordBoundaryRegex = new RegExp( wordBoundaryRegexString, "g" );
-
-	let words = text.split( wordBoundaryRegex );
-
-	if ( shouldRemovePunctuation ) {
-		words = words.map( removePunctuation );
-	} else {
-		// If punctuation is not removed, punctuation marks are tokenized as if they were words.
-		words = flatMap( words, ( word ) => {
-			const newWord = word.replace( punctuationRegex, " $1 " );
-			return newWord.split( " " );
-		} );
-	}
-
-	return filter( words, function( word ) {
-		return word.trim() !== "";
-	} );
-}
-