axyseo 2.0.0-alpha.0.0.20 → 2.0.0-alpha.0.0.4

package/src/languageProcessing/helpers/prominentWords/determineProminentWords.js

@@ -0,0 +1,238 @@
+import { memoize, uniq } from "lodash";
+
+import getWords from "../word/getWords";
+import { normalizeSingle } from "../sanitize/quotes";
+import ProminentWord from "../../values/ProminentWord";
+
+const specialCharacters = /[1234567890‘’“”"'.…?!:;,¿¡«»&*@#±^%$|~=+§`[\](){}⟨⟩<>/\\–\-\u2014\u00d7\s]/g;
+
+/**
+ * Returns only those prominent words that occur more than a certain number of times and do not consist of special characters.
+ *
+ * @param {ProminentWord[]} prominentWords A list of prominent words.
+ * @param {int} [minimalNumberOfOccurrences] A minimal number of occurrences that is needed for a relevant prominentWord, default 2.
+ *
+ * @returns {ProminentWord[]} Only relevant words.
+ */
+function filterProminentWords( prominentWords, minimalNumberOfOccurrences = 2 ) {
+	prominentWords = prominentWords.filter( function( word ) {
+		return (
+			word.getOccurrences() >= minimalNumberOfOccurrences &&
+			word.getWord().replace( specialCharacters, "" ) !== ""
+		);
+	} );
+	return prominentWords;
+}
+
+/**
+ * Sorts prominent words based on their number of occurrences and length.
+ *
+ * @param {ProminentWord[]} prominentWords The prominent words to sort.
+ *
+ * @returns {void}
+ */
+function sortProminentWords( prominentWords ) {
+	prominentWords.sort( function( wordA, wordB ) {
+		const difference = wordB.getOccurrences() - wordA.getOccurrences();
+		// The word with the highest number of occurrences comes first.
+		if ( difference !== 0 ) {
+			return difference;
+		}
+
+		// In case of a tie on occurrence number, the alphabetically first word comes first.
+		return wordA.getStem().localeCompare( wordB.getStem() );
+	} );
+}
+
+/**
+ * Collapses prominent words that have the same stem.
+ *
+ * @param {ProminentWord[]} prominentWords All prominentWords.
+ *
+ * @returns {ProminentWord[]} The original array with collapsed duplicates.
+ */
+function collapseProminentWordsOnStem( prominentWords ) {
+	if ( prominentWords.length === 0 ) {
+		return [];
+	}
+
+	// Sort the input array by stem
+	prominentWords.sort( function( wordA, wordB ) {
+		return wordA.getStem().localeCompare( wordB.getStem() );
+	} );
+
+	const collapsedProminentWords = [];
+	let previousWord = new ProminentWord(
+		prominentWords[ 0 ].getWord(),
+		prominentWords[ 0 ].getStem(),
+		prominentWords[ 0 ].getOccurrences()
+	);
+
+	for ( let i = 1; i < prominentWords.length; i++ ) {
+		const currentWord = new ProminentWord(
+			prominentWords[ i ].getWord(),
+			prominentWords[ i ].getStem(),
+			prominentWords[ i ].getOccurrences()
+		);
+
+		/*
+		 * Compare the stem of the current word in the loop with the previously available stem.
+		 * If they are equal, the word should be collapsed.
+		 * When collapsing, the numbers of occurrences get summed.
+		 * If the stem happens to equal the real word that occurred in the text, we can be sure it's ok to display it
+		 * to the customer. So, the stem reassigns the word.
+		 */
+		if ( currentWord.getStem() ===  previousWord.getStem() ) {
+			previousWord.setOccurrences( previousWord.getOccurrences() + currentWord.getOccurrences() );
+
+			if ( currentWord.getWord() === previousWord.getStem() || currentWord.getWord().toLocaleLowerCase() === previousWord.getStem() ) {
+				previousWord.setWord( currentWord.getWord() );
+			}
+		} else {
+			collapsedProminentWords.push( previousWord );
+			previousWord = currentWord;
+		}
+	}
+
+	collapsedProminentWords.push( previousWord );
+
+	return collapsedProminentWords;
+}
+
+/**
+ * Retrieves a list of all abbreviations from the text. Returns an empty array if the input text is empty.
+ *
+ * @param {string} text A text.
+ *
+ * @returns {string[]} A list of abbreviations from the list.
+ */
+function retrieveAbbreviations( text ) {
+	const words = getWords( normalizeSingle( text ) );
+	const abbreviations = [];
+
+	words.forEach( function( word ) {
+		if ( word.length > 1 && word.length < 5 && word === word.toLocaleUpperCase() ) {
+			abbreviations.push( word.toLocaleLowerCase() );
+		}
+	} );
+
+	return uniq( abbreviations );
+}
+
+/**
+ * Computes prominent words from an array of words. In order to do so, checks whether the word is included in the list of
+ * function words and determines the number of occurrences for every word. Then checks if any two words have the same stem
+ * and if so collapses over them.
+ *
+ * @param {string[]} words          The words to determine relevance for.
+ * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
+ * @param {Function} stemmer        The available stemmer.
+ * @param {Array} functionWords     The available function words list.
+ *
+ * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
+ */
+function computeProminentWords( words, abbreviations, stemmer, functionWords ) {
+	if ( words.length === 0 ) {
+		return [];
+	}
+
+	const uniqueContentWords = uniq( words.filter( word => ! functionWords.includes( word.trim() ) ) );
+	const prominentWords = [];
+
+	uniqueContentWords.forEach( function( word ) {
+		if ( abbreviations.includes( word ) ) {
+			prominentWords.push( new ProminentWord(
+				word.toLocaleUpperCase(),
+				word,
+				words.filter( element => element === word ).length
+			) );
+		} else {
+			prominentWords.push( new ProminentWord(
+				word,
+				stemmer( word ),
+				words.filter( element => element === word ).length
+			) );
+		}
+	} );
+
+	return collapseProminentWordsOnStem( prominentWords );
+}
+
+/**
+ * Caches prominent words depending on text words.
+ * Only the words and abbreviations are used as the cache key as the stemmer and function words
+ * are config and shouldn't change in the scope of one request.
+ *
+ * @param {string[]} words          The words to determine relevance for.
+ * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
+ * @param {Function} stemmer        The available stemmer.
+ * @param {Array} functionWords     The available function words list.
+ *
+ * @returns {function} The function that collects prominent words for a given set of text words, language and morphologyData.
+ */
+const computeProminentWordsMemoized = memoize( ( words, abbreviations, stemmer, functionWords ) => {
+	return computeProminentWords( words, abbreviations, stemmer, functionWords );
+}, ( words, abbreviations ) => {
+	return words.join( "," ) + "," + abbreviations.join( "," );
+} );
+
+
+/**
+ * Gets prominent words from the paper text.
+ *
+ * @param {string} text             The text to retrieve the prominent words from.
+ * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
+ * @param {Function} stemmer        The available stemmer.
+ * @param {Array} functionWords     The available function words list.
+ * @param {function} getWordsCustomHelper   The custom helper to get words.
+ *
+ * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
+ */
+function getProminentWords( text, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {
+	if ( text === "" ) {
+		return [];
+	}
+
+	const words = getWordsCustomHelper
+		? getWordsCustomHelper( normalizeSingle( text ).toLocaleLowerCase() )
+		: getWords( normalizeSingle( text ).toLocaleLowerCase() );
+
+	return computeProminentWordsMemoized( words, abbreviations, stemmer, functionWords );
+}
+
+/**
+ * Gets prominent words from keyphrase and synonyms, metadescription, title, and subheadings.
+ *
+ * @param {string[]} attributes     The array with attributes to process.
+ * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
+ * @param {Function} stemmer        The available stemmer.
+ * @param {Array} functionWords     The available function words list.
+ * @param {function} getWordsCustomHelper   The custom helper to get words.
+ *
+ * @returns {ProminentWord[]} Prominent words from the paper attributes.
+ */
+function getProminentWordsFromPaperAttributes( attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {
+	const wordsFromAttributes = getWordsCustomHelper
+		? getWordsCustomHelper( attributes.join( " " ).toLocaleLowerCase() )
+		: getWords( attributes.join( " " ).toLocaleLowerCase() );
+
+	return computeProminentWords( wordsFromAttributes, abbreviations, stemmer, functionWords );
+}
+
+export {
+	getProminentWords,
+	getProminentWordsFromPaperAttributes,
+	filterProminentWords,
+	sortProminentWords,
+	collapseProminentWordsOnStem,
+	retrieveAbbreviations,
+};
+
+export default {
+	getProminentWords,
+	getProminentWordsFromPaperAttributes,
+	filterProminentWords,
+	sortProminentWords,
+	collapseProminentWordsOnStem,
+	retrieveAbbreviations,
+};

package/src/languageProcessing/helpers/regex/createRegexFromArray.js

@@ -0,0 +1,35 @@
+/** @module stringProcessing/createRegexFromArray */
+
+import { map } from "lodash";
+import addWordBoundary from "../word/addWordboundary.js";
+import replaceDiacritics from "../transliterate/replaceDiacritics";
+import sanitizeString from "../sanitize/sanitizeString";
+
+/**
+ * Creates a regex of combined strings from the input array.
+ *
+ * @param {string[]} array                      The array with strings
+ * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.
+ * @param {string} [extraBoundary=""]           A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.
+ *
+ * @returns {RegExp} regex                              The regex created from the array.
+ */
+export default function( array, disableWordBoundary = false, extraBoundary = "", doReplaceDiacritics = false ) {
+	array = map( array, function( string ) {
+		if ( doReplaceDiacritics ) {
+			string = replaceDiacritics( string );
+		}
+
+		string = sanitizeString( string );
+
+		if ( disableWordBoundary ) {
+			return string;
+		}
+		return addWordBoundary( string, true, extraBoundary );
+	} );
+
+	const regexString = "(" + array.join( ")|(" ) + ")";
+
+	return new RegExp( regexString, "ig" );
+}

package/src/languageProcessing/helpers/regex/createRegexFromDoubleArray.js

@@ -0,0 +1,34 @@
+/** @module stringProcessing/createRegexFromDoubleArray */
+
+import addWordBoundary from "../word/addWordboundary.js";
+
+/**
+ * Creates a regex string of combined strings from the input array.
+ *
+ * @param {array} array The array containing the various parts of a transition word combination.
+ *
+ * @returns {array} The array with replaced entries.
+ */
+const wordCombinationToRegexString = function( array ) {
+	array = array.map( function( word ) {
+		return addWordBoundary( word );
+	} );
+	return array.join( "(.*?)" );
+};
+
+/**
+ * Creates a regex of combined strings from the input array, containing arrays with two entries.
+ *
+ * @param {array} array The array containing arrays.
+ *
+ * @returns {RegExp} The regex created from the array.
+ */
+export default function( array ) {
+	array = array.map( function( wordCombination ) {
+		return wordCombinationToRegexString( wordCombination );
+	} );
+
+	const regexString = "(" + array.join( ")|(" ) + ")";
+
+	return new RegExp( regexString, "ig" );
+}

package/src/languageProcessing/helpers/regex/createWordRegex.js

@@ -0,0 +1,30 @@
+/** @module stringProcessing/stringToRegex */
+import { escapeRegExp, isUndefined, memoize } from "lodash";
+import sanitizeString from "../sanitize/sanitizeString";
+import replaceDiacritics from "../transliterate/replaceDiacritics.js";
+import addWordBoundary from "../word/addWordboundary.js";
+
+/**
+ * Creates a regex from a string so it can be matched everywhere in the same way.
+ *
+ * @param {string}  string                     The string to make a regex from.
+ * @param {string}  [extraBoundary=""]         A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.
+ *
+ * @returns {RegExp} regex The regex made from the keyword
+ */
+export default memoize( function( string, extraBoundary, doReplaceDiacritics ) {
+	if ( isUndefined( extraBoundary ) ) {
+		extraBoundary = "";
+	}
+
+	if ( isUndefined( doReplaceDiacritics ) || doReplaceDiacritics === true ) {
+		string = replaceDiacritics( string );
+	}
+
+	string = sanitizeString( string );
+	string = escapeRegExp( string );
+	string = addWordBoundary( string, false, extraBoundary );
+
+	return new RegExp( string, "ig" );
+} );

package/src/languageProcessing/helpers/regex/matchStringWithRegex.js

@@ -0,0 +1,19 @@
+/** @module stringProcessing/matchStringWithRegex */
+
+/**
+ * Checks a string with a regex, return all matches found with that regex.
+ *
+ * @param {String} 		text        The text to match.
+ * @param {object} 		regex  		A compiled regular expression.
+ *
+ * @returns {Array} Array with matches, empty array if no matches found.
+ */
+export default function( text, regex ) {
+	let matches = text.match( regex );
+
+	if ( matches === null ) {
+		matches = [];
+	}
+
+	return matches;
+}

package/src/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js

@@ -0,0 +1,14 @@
+/**
+ * Checks whether a given regex matches a given word and if so, performs the replacement.
+ *
+ * @param {string}      word 					The word that may need to be modified.
+ * @param {string[]}    regexAndReplacement     A regex and the required replacement.
+ *
+ * @returns {?string} The modified stem or undefined if no match was found.
+ */
+export default function searchAndReplaceWithOneRegex( word, regexAndReplacement ) {
+	if ( word.search( new RegExp( regexAndReplacement[ 0 ] ) ) !== -1 ) {
+		word = word.replace( new RegExp( regexAndReplacement[ 0 ] ), regexAndReplacement[ 1 ] );
+		return word;
+	}
+}

package/src/languageProcessing/helpers/sanitize/doubleQuotes.js

@@ -0,0 +1,12 @@
+/**
+ * Returns an array with double quotes.
+ *
+ * Japanese-specific quotes:
+ * U300c - left corner bracket.
+ * U300d - right corner bracket.
+ * U300e - left white corner bracket.
+ * U300f - right white corner bracket.
+ *
+ * @returns {string[]} The array with double quotes..
+ */
+export default [ "“", "”", "〝", "〞", "〟", "‟", "„", "\"", "\u300c", "\u300d", "\u300e", "\u300f" ];

package/src/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js

@@ -0,0 +1,131 @@
+
+/**
+ * Creates a regex to filter shortcodes from HTML.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {RegExp} The regex to recognize the shortcodes.
+ */
+export const createShortcodeTagsRegex = shortcodeTags => {
+	const shortcodeTagsRegexString = `\\[\\/?(${ shortcodeTags.join( "|" ) })[^\\]]*\\]`;
+	return new RegExp( shortcodeTagsRegexString, "g" );
+};
+
+/**
+ * Filters shortcodes from HTML string.
+ * @param {string} html The HTML to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {string} The filtered HTML.
+ */
+export const filterShortcodesFromHTML = ( html, shortcodeTags ) => {
+	if ( ! shortcodeTags || shortcodeTags.length === 0 ) {
+		return html;
+	}
+	const shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );
+	return html.replace( shortcodeTagsRegex, "" );
+};
+
+/**
+ * Checks if a token is part of an opening shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of an opening shortcode.
+ */
+const tokenIsPartOfOpeningShortcode = ( tokens, index ) => {
+	return tokens[ index - 1 ] && tokens[ index - 1 ].text === "[";
+};
+
+/**
+ * Checks if a token is part of a closing shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of a closing shortcode.
+ */
+const tokenIsPartOfClosingShortcode = ( tokens, index ) => {
+	return  tokens[ index - 1 ] && tokens[ index - 1 ].text === "/"  && tokens[ index - 2 ]  && tokens[ index - 2 ].text === "[";
+};
+
+/**
+ * Checks if a token is part of a shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @param {string[]} shortcodeTags An array of active shortcode tags.
+ * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).
+ * @returns {boolean} Whether the token is part of a shortcode.
+ */
+const tokenIsShortcode = ( tokens, index, shortcodeTags, encounteredClosingBracket ) => {
+	return encounteredClosingBracket && shortcodeTags.includes( tokens[ index ].text ) &&
+		( tokenIsPartOfOpeningShortcode( tokens, index ) || tokenIsPartOfClosingShortcode( tokens, index )  );
+};
+
+/**
+ * Filters shortcodes from a sentence.
+ * @param {Sentence} sentence The sentence to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ *
+ */
+const filterShortcodesFromSentence = ( sentence, shortcodeTags, shortcodeTagsRegex ) => {
+	const tokens = sentence.tokens;
+
+	let encounteredClosingBracket = false;
+	// traverse through tokens backwards
+	for ( let i = tokens.length - 1; i >= 0; i-- ) {
+		if ( tokens[ i ].text === "]" ) {
+			encounteredClosingBracket = true;
+		}
+
+		if ( tokenIsShortcode( tokens, i, shortcodeTags, encounteredClosingBracket ) ) {
+			while ( tokens[ i ].text !== "]" ) {
+				tokens.splice( i, 1 );
+			}
+			tokens.splice( i, 1 );
+			encounteredClosingBracket = false;
+
+			while ( tokens[ i - 1 ] && "[/".includes( tokens[ i - 1 ].text ) ) {
+				tokens.splice( i - 1, 1 );
+				i--;
+			}
+		}
+	}
+	sentence.tokens = tokens;
+
+	sentence.text = sentence.text.replace( shortcodeTagsRegex, "" );
+};
+
+/**
+ * A recursive function that filters shortcodes from sentences.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ */
+const filterShortcodesFromSentences = ( tree, shortcodeTags, shortcodeTagsRegex ) => {
+	if ( tree.sentences ) {
+		tree.sentences.forEach( sentence => {
+			filterShortcodesFromSentence( sentence, shortcodeTags, shortcodeTagsRegex );
+		} );
+	}
+
+	if ( tree.childNodes ) {
+		tree.childNodes.forEach( childNode => {
+			filterShortcodesFromSentences( childNode, shortcodeTags, shortcodeTagsRegex );
+		} );
+	}
+};
+
+/**
+ * Filters shortcodes from the tree.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {void}
+ */
+const filterShortcodesFromTree = ( tree, shortcodeTags ) => {
+	if ( ! shortcodeTags || shortcodeTags.length === 0 ) {
+		return;
+	}
+	const shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );
+
+	filterShortcodesFromSentences( tree, shortcodeTags, shortcodeTagsRegex );
+};
+
+export default filterShortcodesFromTree;

package/src/languageProcessing/helpers/sanitize/mergeListItems.js

@@ -0,0 +1,24 @@
+/**
+ * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.
+ * That way, lists with single words don't result in a skewed keyphrase distribution result.
+ *
+ * @param {string}  text    The text in which to remove the list structures.
+ *
+ * @returns {string} The text with all list structures removed.
+ */
+export function mergeListItems( text ) {
+	const listTags = /<\/?(o|ul)(?:[^>]+)?>/g;
+	const listItemTags = /\s?<\/?li(?:[^>]+)?>\s?/g;
+
+	/*
+	 * Steps:
+	 * 1) Remove all ul and ol tags.
+	 * 2) Replace all li tags with spaces to avoid list items being stuck together.
+	 * 3) Replace multiple instances of whitespace with a single instance.
+	 */
+	text = text.replace( listTags, "" );
+	text = text.replace( listItemTags, " " );
+	text = text.replace( /\s+/g, " " );
+
+	return text;
+}

package/src/languageProcessing/helpers/sanitize/parseSynonyms.js

@@ -0,0 +1,20 @@
+/** @module stringProcessing/parseSynonyms */
+
+import stripSpaces from "./stripSpaces.js";
+import removePunctuationExceptQuotes from "./removePunctuationExceptQuotes.js";
+
+/**
+ * Parses synonyms from a comma-separated string into an array.
+ *
+ * @param {String} synonyms The text to match
+ *
+ * @returns {Array} An array with all synonyms.
+ */
+export default function( synonyms ) {
+	let synonymsSplit = synonyms.split( "," );
+
+	synonymsSplit = synonymsSplit.map( synonym =>
+		removePunctuationExceptQuotes( stripSpaces( synonym ) ) ).filter( synonym => synonym );
+
+	return synonymsSplit;
+}

package/src/languageProcessing/helpers/sanitize/quotes.js

@@ -0,0 +1,46 @@
+const SINGLE_QUOTES_ARRAY = [ "'", "‘", "’", "‛", "`", "‹", "›" ];
+const SINGLE_QUOTES_REGEX = new RegExp( "[" + SINGLE_QUOTES_ARRAY.join( "" ) + "]", "g" );
+
+/**
+ * Normalizes single quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeSingleQuotes( text ) {
+	return text.replace( SINGLE_QUOTES_REGEX, "'" );
+}
+
+/**
+ * Normalizes double quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeDoubleQuotes( text ) {
+	return text.replace( /[“”〝〞〟‟„『』«»]/g, "\"" );
+}
+
+/**
+ * Normalizes quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeQuotes( text ) {
+	return normalizeDoubleQuotes( normalizeSingleQuotes( text ) );
+}
+
+export {
+	normalizeSingleQuotes as normalizeSingle,
+	normalizeDoubleQuotes as normalizeDouble,
+	normalizeQuotes as normalize,
+	SINGLE_QUOTES_REGEX,
+	SINGLE_QUOTES_ARRAY,
+};
+
+export default {
+	normalizeSingle: normalizeSingleQuotes,
+	normalizeDouble: normalizeDoubleQuotes,
+	normalize: normalizeQuotes,
+};

package/src/languageProcessing/helpers/sanitize/removeEmailAddresses.js

@@ -0,0 +1,12 @@
+const emailRegex = new RegExp( "[^\\s@]+@[^\\s@]+\\.[^\\s@]+", "igm" );
+
+/**
+ * Removes email addresses from a text.
+ *
+ * @param {string} text The text to remove emails from.
+ *
+ * @returns {string} The text without email addresses.
+ */
+export default function( text ) {
+	return text.replace( emailRegex, "" );
+}