axyseo 2.0.0-alpha.0.0.4 → 2.0.0-alpha.0.0.6

package/src/languageProcessing/helpers/prominentWords/determineProminentWords.js

@@ -1,238 +0,0 @@
-import { memoize, uniq } from "lodash";
-
-import getWords from "../word/getWords";
-import { normalizeSingle } from "../sanitize/quotes";
-import ProminentWord from "../../values/ProminentWord";
-
-const specialCharacters = /[1234567890‘’“”"'.…?!:;,¿¡«»&*@#±^%$|~=+§`[\](){}⟨⟩<>/\\–\-\u2014\u00d7\s]/g;
-
-/**
- * Returns only those prominent words that occur more than a certain number of times and do not consist of special characters.
- *
- * @param {ProminentWord[]} prominentWords A list of prominent words.
- * @param {int} [minimalNumberOfOccurrences] A minimal number of occurrences that is needed for a relevant prominentWord, default 2.
- *
- * @returns {ProminentWord[]} Only relevant words.
- */
-function filterProminentWords( prominentWords, minimalNumberOfOccurrences = 2 ) {
-	prominentWords = prominentWords.filter( function( word ) {
-		return (
-			word.getOccurrences() >= minimalNumberOfOccurrences &&
-			word.getWord().replace( specialCharacters, "" ) !== ""
-		);
-	} );
-	return prominentWords;
-}
-
-/**
- * Sorts prominent words based on their number of occurrences and length.
- *
- * @param {ProminentWord[]} prominentWords The prominent words to sort.
- *
- * @returns {void}
- */
-function sortProminentWords( prominentWords ) {
-	prominentWords.sort( function( wordA, wordB ) {
-		const difference = wordB.getOccurrences() - wordA.getOccurrences();
-		// The word with the highest number of occurrences comes first.
-		if ( difference !== 0 ) {
-			return difference;
-		}
-
-		// In case of a tie on occurrence number, the alphabetically first word comes first.
-		return wordA.getStem().localeCompare( wordB.getStem() );
-	} );
-}
-
-/**
- * Collapses prominent words that have the same stem.
- *
- * @param {ProminentWord[]} prominentWords All prominentWords.
- *
- * @returns {ProminentWord[]} The original array with collapsed duplicates.
- */
-function collapseProminentWordsOnStem( prominentWords ) {
-	if ( prominentWords.length === 0 ) {
-		return [];
-	}
-
-	// Sort the input array by stem
-	prominentWords.sort( function( wordA, wordB ) {
-		return wordA.getStem().localeCompare( wordB.getStem() );
-	} );
-
-	const collapsedProminentWords = [];
-	let previousWord = new ProminentWord(
-		prominentWords[ 0 ].getWord(),
-		prominentWords[ 0 ].getStem(),
-		prominentWords[ 0 ].getOccurrences()
-	);
-
-	for ( let i = 1; i < prominentWords.length; i++ ) {
-		const currentWord = new ProminentWord(
-			prominentWords[ i ].getWord(),
-			prominentWords[ i ].getStem(),
-			prominentWords[ i ].getOccurrences()
-		);
-
-		/*
-		 * Compare the stem of the current word in the loop with the previously available stem.
-		 * If they are equal, the word should be collapsed.
-		 * When collapsing, the numbers of occurrences get summed.
-		 * If the stem happens to equal the real word that occurred in the text, we can be sure it's ok to display it
-		 * to the customer. So, the stem reassigns the word.
-		 */
-		if ( currentWord.getStem() ===  previousWord.getStem() ) {
-			previousWord.setOccurrences( previousWord.getOccurrences() + currentWord.getOccurrences() );
-
-			if ( currentWord.getWord() === previousWord.getStem() || currentWord.getWord().toLocaleLowerCase() === previousWord.getStem() ) {
-				previousWord.setWord( currentWord.getWord() );
-			}
-		} else {
-			collapsedProminentWords.push( previousWord );
-			previousWord = currentWord;
-		}
-	}
-
-	collapsedProminentWords.push( previousWord );
-
-	return collapsedProminentWords;
-}
-
-/**
- * Retrieves a list of all abbreviations from the text. Returns an empty array if the input text is empty.
- *
- * @param {string} text A text.
- *
- * @returns {string[]} A list of abbreviations from the list.
- */
-function retrieveAbbreviations( text ) {
-	const words = getWords( normalizeSingle( text ) );
-	const abbreviations = [];
-
-	words.forEach( function( word ) {
-		if ( word.length > 1 && word.length < 5 && word === word.toLocaleUpperCase() ) {
-			abbreviations.push( word.toLocaleLowerCase() );
-		}
-	} );
-
-	return uniq( abbreviations );
-}
-
-/**
- * Computes prominent words from an array of words. In order to do so, checks whether the word is included in the list of
- * function words and determines the number of occurrences for every word. Then checks if any two words have the same stem
- * and if so collapses over them.
- *
- * @param {string[]} words          The words to determine relevance for.
- * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
- * @param {Function} stemmer        The available stemmer.
- * @param {Array} functionWords     The available function words list.
- *
- * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
- */
-function computeProminentWords( words, abbreviations, stemmer, functionWords ) {
-	if ( words.length === 0 ) {
-		return [];
-	}
-
-	const uniqueContentWords = uniq( words.filter( word => ! functionWords.includes( word.trim() ) ) );
-	const prominentWords = [];
-
-	uniqueContentWords.forEach( function( word ) {
-		if ( abbreviations.includes( word ) ) {
-			prominentWords.push( new ProminentWord(
-				word.toLocaleUpperCase(),
-				word,
-				words.filter( element => element === word ).length
-			) );
-		} else {
-			prominentWords.push( new ProminentWord(
-				word,
-				stemmer( word ),
-				words.filter( element => element === word ).length
-			) );
-		}
-	} );
-
-	return collapseProminentWordsOnStem( prominentWords );
-}
-
-/**
- * Caches prominent words depending on text words.
- * Only the words and abbreviations are used as the cache key as the stemmer and function words
- * are config and shouldn't change in the scope of one request.
- *
- * @param {string[]} words          The words to determine relevance for.
- * @param {string[]} abbreviations  Abbreviations that should not be stemmed.
- * @param {Function} stemmer        The available stemmer.
- * @param {Array} functionWords     The available function words list.
- *
- * @returns {function} The function that collects prominent words for a given set of text words, language and morphologyData.
- */
-const computeProminentWordsMemoized = memoize( ( words, abbreviations, stemmer, functionWords ) => {
-	return computeProminentWords( words, abbreviations, stemmer, functionWords );
-}, ( words, abbreviations ) => {
-	return words.join( "," ) + "," + abbreviations.join( "," );
-} );
-
-
-/**
- * Gets prominent words from the paper text.
- *
- * @param {string} text             The text to retrieve the prominent words from.
- * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
- * @param {Function} stemmer        The available stemmer.
- * @param {Array} functionWords     The available function words list.
- * @param {function} getWordsCustomHelper   The custom helper to get words.
- *
- * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.
- */
-function getProminentWords( text, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {
-	if ( text === "" ) {
-		return [];
-	}
-
-	const words = getWordsCustomHelper
-		? getWordsCustomHelper( normalizeSingle( text ).toLocaleLowerCase() )
-		: getWords( normalizeSingle( text ).toLocaleLowerCase() );
-
-	return computeProminentWordsMemoized( words, abbreviations, stemmer, functionWords );
-}
-
-/**
- * Gets prominent words from keyphrase and synonyms, metadescription, title, and subheadings.
- *
- * @param {string[]} attributes     The array with attributes to process.
- * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.
- * @param {Function} stemmer        The available stemmer.
- * @param {Array} functionWords     The available function words list.
- * @param {function} getWordsCustomHelper   The custom helper to get words.
- *
- * @returns {ProminentWord[]} Prominent words from the paper attributes.
- */
-function getProminentWordsFromPaperAttributes( attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {
-	const wordsFromAttributes = getWordsCustomHelper
-		? getWordsCustomHelper( attributes.join( " " ).toLocaleLowerCase() )
-		: getWords( attributes.join( " " ).toLocaleLowerCase() );
-
-	return computeProminentWords( wordsFromAttributes, abbreviations, stemmer, functionWords );
-}
-
-export {
-	getProminentWords,
-	getProminentWordsFromPaperAttributes,
-	filterProminentWords,
-	sortProminentWords,
-	collapseProminentWordsOnStem,
-	retrieveAbbreviations,
-};
-
-export default {
-	getProminentWords,
-	getProminentWordsFromPaperAttributes,
-	filterProminentWords,
-	sortProminentWords,
-	collapseProminentWordsOnStem,
-	retrieveAbbreviations,
-};

package/src/languageProcessing/helpers/regex/createRegexFromArray.js

@@ -1,35 +0,0 @@
-/** @module stringProcessing/createRegexFromArray */
-
-import { map } from "lodash";
-import addWordBoundary from "../word/addWordboundary.js";
-import replaceDiacritics from "../transliterate/replaceDiacritics";
-import sanitizeString from "../sanitize/sanitizeString";
-
-/**
- * Creates a regex of combined strings from the input array.
- *
- * @param {string[]} array                      The array with strings
- * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.
- * @param {string} [extraBoundary=""]           A string that is used as extra boundary for the regex.
- * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.
- *
- * @returns {RegExp} regex                              The regex created from the array.
- */
-export default function( array, disableWordBoundary = false, extraBoundary = "", doReplaceDiacritics = false ) {
-	array = map( array, function( string ) {
-		if ( doReplaceDiacritics ) {
-			string = replaceDiacritics( string );
-		}
-
-		string = sanitizeString( string );
-
-		if ( disableWordBoundary ) {
-			return string;
-		}
-		return addWordBoundary( string, true, extraBoundary );
-	} );
-
-	const regexString = "(" + array.join( ")|(" ) + ")";
-
-	return new RegExp( regexString, "ig" );
-}

package/src/languageProcessing/helpers/regex/createRegexFromDoubleArray.js

@@ -1,34 +0,0 @@
-/** @module stringProcessing/createRegexFromDoubleArray */
-
-import addWordBoundary from "../word/addWordboundary.js";
-
-/**
- * Creates a regex string of combined strings from the input array.
- *
- * @param {array} array The array containing the various parts of a transition word combination.
- *
- * @returns {array} The array with replaced entries.
- */
-const wordCombinationToRegexString = function( array ) {
-	array = array.map( function( word ) {
-		return addWordBoundary( word );
-	} );
-	return array.join( "(.*?)" );
-};
-
-/**
- * Creates a regex of combined strings from the input array, containing arrays with two entries.
- *
- * @param {array} array The array containing arrays.
- *
- * @returns {RegExp} The regex created from the array.
- */
-export default function( array ) {
-	array = array.map( function( wordCombination ) {
-		return wordCombinationToRegexString( wordCombination );
-	} );
-
-	const regexString = "(" + array.join( ")|(" ) + ")";
-
-	return new RegExp( regexString, "ig" );
-}

package/src/languageProcessing/helpers/regex/createWordRegex.js

@@ -1,30 +0,0 @@
-/** @module stringProcessing/stringToRegex */
-import { escapeRegExp, isUndefined, memoize } from "lodash";
-import sanitizeString from "../sanitize/sanitizeString";
-import replaceDiacritics from "../transliterate/replaceDiacritics.js";
-import addWordBoundary from "../word/addWordboundary.js";
-
-/**
- * Creates a regex from a string so it can be matched everywhere in the same way.
- *
- * @param {string}  string                     The string to make a regex from.
- * @param {string}  [extraBoundary=""]         A string that is used as extra boundary for the regex.
- * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.
- *
- * @returns {RegExp} regex The regex made from the keyword
- */
-export default memoize( function( string, extraBoundary, doReplaceDiacritics ) {
-	if ( isUndefined( extraBoundary ) ) {
-		extraBoundary = "";
-	}
-
-	if ( isUndefined( doReplaceDiacritics ) || doReplaceDiacritics === true ) {
-		string = replaceDiacritics( string );
-	}
-
-	string = sanitizeString( string );
-	string = escapeRegExp( string );
-	string = addWordBoundary( string, false, extraBoundary );
-
-	return new RegExp( string, "ig" );
-} );

package/src/languageProcessing/helpers/regex/matchStringWithRegex.js

@@ -1,19 +0,0 @@
-/** @module stringProcessing/matchStringWithRegex */
-
-/**
- * Checks a string with a regex, return all matches found with that regex.
- *
- * @param {String} 		text        The text to match.
- * @param {object} 		regex  		A compiled regular expression.
- *
- * @returns {Array} Array with matches, empty array if no matches found.
- */
-export default function( text, regex ) {
-	let matches = text.match( regex );
-
-	if ( matches === null ) {
-		matches = [];
-	}
-
-	return matches;
-}

package/src/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js

@@ -1,14 +0,0 @@
-/**
- * Checks whether a given regex matches a given word and if so, performs the replacement.
- *
- * @param {string}      word 					The word that may need to be modified.
- * @param {string[]}    regexAndReplacement     A regex and the required replacement.
- *
- * @returns {?string} The modified stem or undefined if no match was found.
- */
-export default function searchAndReplaceWithOneRegex( word, regexAndReplacement ) {
-	if ( word.search( new RegExp( regexAndReplacement[ 0 ] ) ) !== -1 ) {
-		word = word.replace( new RegExp( regexAndReplacement[ 0 ] ), regexAndReplacement[ 1 ] );
-		return word;
-	}
-}

package/src/languageProcessing/helpers/sanitize/doubleQuotes.js

@@ -1,12 +0,0 @@
-/**
- * Returns an array with double quotes.
- *
- * Japanese-specific quotes:
- * U300c - left corner bracket.
- * U300d - right corner bracket.
- * U300e - left white corner bracket.
- * U300f - right white corner bracket.
- *
- * @returns {string[]} The array with double quotes..
- */
-export default [ "“", "”", "〝", "〞", "〟", "‟", "„", "\"", "\u300c", "\u300d", "\u300e", "\u300f" ];

package/src/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js

@@ -1,131 +0,0 @@
-
-/**
- * Creates a regex to filter shortcodes from HTML.
- * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
- * @returns {RegExp} The regex to recognize the shortcodes.
- */
-export const createShortcodeTagsRegex = shortcodeTags => {
-	const shortcodeTagsRegexString = `\\[\\/?(${ shortcodeTags.join( "|" ) })[^\\]]*\\]`;
-	return new RegExp( shortcodeTagsRegexString, "g" );
-};
-
-/**
- * Filters shortcodes from HTML string.
- * @param {string} html The HTML to filter.
- * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
- * @returns {string} The filtered HTML.
- */
-export const filterShortcodesFromHTML = ( html, shortcodeTags ) => {
-	if ( ! shortcodeTags || shortcodeTags.length === 0 ) {
-		return html;
-	}
-	const shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );
-	return html.replace( shortcodeTagsRegex, "" );
-};
-
-/**
- * Checks if a token is part of an opening shortcode.
- * @param {Token[]} tokens The tokens to check.
- * @param {number} index The index of the current token.
- * @returns {boolean} Whether the token is part of an opening shortcode.
- */
-const tokenIsPartOfOpeningShortcode = ( tokens, index ) => {
-	return tokens[ index - 1 ] && tokens[ index - 1 ].text === "[";
-};
-
-/**
- * Checks if a token is part of a closing shortcode.
- * @param {Token[]} tokens The tokens to check.
- * @param {number} index The index of the current token.
- * @returns {boolean} Whether the token is part of a closing shortcode.
- */
-const tokenIsPartOfClosingShortcode = ( tokens, index ) => {
-	return  tokens[ index - 1 ] && tokens[ index - 1 ].text === "/"  && tokens[ index - 2 ]  && tokens[ index - 2 ].text === "[";
-};
-
-/**
- * Checks if a token is part of a shortcode.
- * @param {Token[]} tokens The tokens to check.
- * @param {number} index The index of the current token.
- * @param {string[]} shortcodeTags An array of active shortcode tags.
- * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).
- * @returns {boolean} Whether the token is part of a shortcode.
- */
-const tokenIsShortcode = ( tokens, index, shortcodeTags, encounteredClosingBracket ) => {
-	return encounteredClosingBracket && shortcodeTags.includes( tokens[ index ].text ) &&
-		( tokenIsPartOfOpeningShortcode( tokens, index ) || tokenIsPartOfClosingShortcode( tokens, index )  );
-};
-
-/**
- * Filters shortcodes from a sentence.
- * @param {Sentence} sentence The sentence to filter.
- * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
- * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
- * @returns {void}
- *
- */
-const filterShortcodesFromSentence = ( sentence, shortcodeTags, shortcodeTagsRegex ) => {
-	const tokens = sentence.tokens;
-
-	let encounteredClosingBracket = false;
-	// traverse through tokens backwards
-	for ( let i = tokens.length - 1; i >= 0; i-- ) {
-		if ( tokens[ i ].text === "]" ) {
-			encounteredClosingBracket = true;
-		}
-
-		if ( tokenIsShortcode( tokens, i, shortcodeTags, encounteredClosingBracket ) ) {
-			while ( tokens[ i ].text !== "]" ) {
-				tokens.splice( i, 1 );
-			}
-			tokens.splice( i, 1 );
-			encounteredClosingBracket = false;
-
-			while ( tokens[ i - 1 ] && "[/".includes( tokens[ i - 1 ].text ) ) {
-				tokens.splice( i - 1, 1 );
-				i--;
-			}
-		}
-	}
-	sentence.tokens = tokens;
-
-	sentence.text = sentence.text.replace( shortcodeTagsRegex, "" );
-};
-
-/**
- * A recursive function that filters shortcodes from sentences.
- * @param {Node} tree The tree to filter.
- * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
- * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
- * @returns {void}
- */
-const filterShortcodesFromSentences = ( tree, shortcodeTags, shortcodeTagsRegex ) => {
-	if ( tree.sentences ) {
-		tree.sentences.forEach( sentence => {
-			filterShortcodesFromSentence( sentence, shortcodeTags, shortcodeTagsRegex );
-		} );
-	}
-
-	if ( tree.childNodes ) {
-		tree.childNodes.forEach( childNode => {
-			filterShortcodesFromSentences( childNode, shortcodeTags, shortcodeTagsRegex );
-		} );
-	}
-};
-
-/**
- * Filters shortcodes from the tree.
- * @param {Node} tree The tree to filter.
- * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
- * @returns {void}
- */
-const filterShortcodesFromTree = ( tree, shortcodeTags ) => {
-	if ( ! shortcodeTags || shortcodeTags.length === 0 ) {
-		return;
-	}
-	const shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );
-
-	filterShortcodesFromSentences( tree, shortcodeTags, shortcodeTagsRegex );
-};
-
-export default filterShortcodesFromTree;

package/src/languageProcessing/helpers/sanitize/mergeListItems.js

@@ -1,24 +0,0 @@
-/**
- * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.
- * That way, lists with single words don't result in a skewed keyphrase distribution result.
- *
- * @param {string}  text    The text in which to remove the list structures.
- *
- * @returns {string} The text with all list structures removed.
- */
-export function mergeListItems( text ) {
-	const listTags = /<\/?(o|ul)(?:[^>]+)?>/g;
-	const listItemTags = /\s?<\/?li(?:[^>]+)?>\s?/g;
-
-	/*
-	 * Steps:
-	 * 1) Remove all ul and ol tags.
-	 * 2) Replace all li tags with spaces to avoid list items being stuck together.
-	 * 3) Replace multiple instances of whitespace with a single instance.
-	 */
-	text = text.replace( listTags, "" );
-	text = text.replace( listItemTags, " " );
-	text = text.replace( /\s+/g, " " );
-
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/parseSynonyms.js

@@ -1,20 +0,0 @@
-/** @module stringProcessing/parseSynonyms */
-
-import stripSpaces from "./stripSpaces.js";
-import removePunctuationExceptQuotes from "./removePunctuationExceptQuotes.js";
-
-/**
- * Parses synonyms from a comma-separated string into an array.
- *
- * @param {String} synonyms The text to match
- *
- * @returns {Array} An array with all synonyms.
- */
-export default function( synonyms ) {
-	let synonymsSplit = synonyms.split( "," );
-
-	synonymsSplit = synonymsSplit.map( synonym =>
-		removePunctuationExceptQuotes( stripSpaces( synonym ) ) ).filter( synonym => synonym );
-
-	return synonymsSplit;
-}

package/src/languageProcessing/helpers/sanitize/quotes.js

@@ -1,46 +0,0 @@
-const SINGLE_QUOTES_ARRAY = [ "'", "‘", "’", "‛", "`", "‹", "›" ];
-const SINGLE_QUOTES_REGEX = new RegExp( "[" + SINGLE_QUOTES_ARRAY.join( "" ) + "]", "g" );
-
-/**
- * Normalizes single quotes to 'regular' quotes.
- *
- * @param {string} text Text to normalize.
- * @returns {string} The normalized text.
- */
-function normalizeSingleQuotes( text ) {
-	return text.replace( SINGLE_QUOTES_REGEX, "'" );
-}
-
-/**
- * Normalizes double quotes to 'regular' quotes.
- *
- * @param {string} text Text to normalize.
- * @returns {string} The normalized text.
- */
-function normalizeDoubleQuotes( text ) {
-	return text.replace( /[“”〝〞〟‟„『』«»]/g, "\"" );
-}
-
-/**
- * Normalizes quotes to 'regular' quotes.
- *
- * @param {string} text Text to normalize.
- * @returns {string} The normalized text.
- */
-function normalizeQuotes( text ) {
-	return normalizeDoubleQuotes( normalizeSingleQuotes( text ) );
-}
-
-export {
-	normalizeSingleQuotes as normalizeSingle,
-	normalizeDoubleQuotes as normalizeDouble,
-	normalizeQuotes as normalize,
-	SINGLE_QUOTES_REGEX,
-	SINGLE_QUOTES_ARRAY,
-};
-
-export default {
-	normalizeSingle: normalizeSingleQuotes,
-	normalizeDouble: normalizeDoubleQuotes,
-	normalize: normalizeQuotes,
-};

package/src/languageProcessing/helpers/sanitize/removeEmailAddresses.js

@@ -1,12 +0,0 @@
-const emailRegex = new RegExp( "[^\\s@]+@[^\\s@]+\\.[^\\s@]+", "igm" );
-
-/**
- * Removes email addresses from a text.
- *
- * @param {string} text The text to remove emails from.
- *
- * @returns {string} The text without email addresses.
- */
-export default function( text ) {
-	return text.replace( emailRegex, "" );
-}