axyseo 2.0.0-alpha.0.0.20 → 2.0.0-alpha.0.0.4

package/src/languageProcessing/helpers/sentence/SentenceTokenizer.js

@@ -0,0 +1,640 @@
+import { isNaN, isUndefined, map } from "lodash";
+
+import core from "tokenizer2/core";
+
+import { normalize as normalizeQuotes } from "../sanitize/quotes.js";
+
+import abbreviations from "../../languages/en/config/abbreviations";
+
+import createRegexFromArray from "../regex/createRegexFromArray";
+import wordBoundaries from "../../../config/wordBoundaries";
+
+// All characters that indicate a sentence delimiter.
+const fullStop = ".";
+
+const fullStopRegex = new RegExp( "^[" + fullStop + "]$" );
+const smallerThanContentRegex = /^<[^><]*$/;
+const htmlStartRegex = /^<([^>\s/]+)[^>]*>$/mi;
+const htmlEndRegex = /^<\/([^>\s]+)[^>]*>$/mi;
+
+const blockStartRegex = /^\s*[[({]\s*$/;
+const blockEndRegex = /^\s*[\])}]\s*$/;
+
+const abbreviationsPreparedForRegex = abbreviations.map( ( abbreviation ) => abbreviation.replace( ".", "\\." ) );
+const abbreviationsRegex = createRegexFromArray( abbreviationsPreparedForRegex );
+
+const wordBoundariesForRegex = "(^|$|[" + wordBoundaries().map( ( boundary ) => "\\" + boundary ).join( "" ) + "])";
+const lastCharacterPartOfInitialsRegex = new RegExp( wordBoundariesForRegex + "[A-Za-z]$" );
+
+// Constants to be used in isValidTagPair.
+// A regex to get the tag type.
+const tagTypeRegex = /<\/?([^\s]+?)(\s|>)/;
+// Semantic tags (as opposed to style tags) are tags that are used to structure the text.
+const semanticTags = [ "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "span", "li", "main" ];
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+export default class SentenceTokenizer {
+	/**
+	 * Constructor
+	 * @constructor
+	 */
+	constructor() {
+		/*
+         * \u2026 - ellipsis.
+         * \u06D4 - Urdu full stop.
+         * \u061f - Arabic question mark.
+        */
+		this.sentenceDelimiters = "”〞〟„』›»’‛`\"?!\u2026\u06d4\u061f";
+	}
+
+	/**
+	 * Gets the sentence delimiters.
+	 *
+	 * @returns {string} The sentence delimiters.
+	 */
+	getSentenceDelimiters() {
+		return this.sentenceDelimiters;
+	}
+
+	/**
+	 * Returns whether or not a certain character is a number.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the character is a capital letter.
+	 */
+	isNumber( character ) {
+		return ! isNaN( parseInt( character, 10 ) );
+	}
+
+	/**
+	 * Returns whether or not a given HTML tag is a break tag.
+	 *
+	 * @param {string} htmlTag The HTML tag to check.
+	 * @returns {boolean} Whether or not the given HTML tag is a break tag.
+	 */
+	isBreakTag( htmlTag ) {
+		return /<\/?br/.test( htmlTag );
+	}
+
+	/**
+	 * Returns whether or not a given character is quotation mark.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the given character is a quotation mark.
+	 */
+	isQuotation( character ) {
+		character = normalizeQuotes( character );
+
+		return "'" === character ||
+			"\"" === character;
+	}
+
+	/**
+	 * A mock definition of this function. This function is only used in extensions for languages that use an ordinal dot.
+	 *
+	 * @returns {boolean} Always returns false as it is a language specific implementation if a language has an ordinal dot.
+	 */
+	endsWithOrdinalDot() {
+		return false;
+	}
+
+	/**
+	 * Returns whether or not a given character is a punctuation mark that can be at the beginning
+	 * of a sentence, like ¿ and ¡ used in Spanish.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the given character is a punctuation mark.
+	 */
+	isPunctuation( character ) {
+		return "¿" === character ||
+			"¡" === character;
+	}
+
+	/**
+	 * Removes duplicate whitespace from a given text.
+	 *
+	 * @param {string} text The text with duplicate whitespace.
+	 * @returns {string} The text without duplicate whitespace.
+	 */
+	removeDuplicateWhitespace( text ) {
+		return text.replace( /\s+/, " " );
+	}
+
+	/**
+	 * Returns whether or not a certain character is a capital letter.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the character is a capital letter.
+	 */
+	isCapitalLetter( character ) {
+		return character !== character.toLocaleLowerCase();
+	}
+
+	/**
+	 * Checks whether the given character is a smaller than sign.
+	 *
+	 * This function is used to make sure that tokenizing the content after
+	 * the smaller than sign works as expected.
+	 * E.g. 'A sentence. < Hello world!' = ['A sentence.', '< Hello world!'].
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether the character is a smaller than sign ('<') or not.
+	 */
+	isSmallerThanSign( character ) {
+		return character === "<";
+	}
+
+	/**
+	 * Retrieves the next two characters from an array with the two next tokens.
+	 *
+	 * @param {Array} nextTokens The two next tokens. Might be undefined.
+	 * @returns {string} The next two characters.
+	 */
+	getNextTwoCharacters( nextTokens ) {
+		let next = "";
+
+		if ( ! isUndefined( nextTokens[ 0 ] ) ) {
+			next += nextTokens[ 0 ].src;
+		}
+
+		if ( ! isUndefined( nextTokens[ 1 ] ) ) {
+			next += nextTokens[ 1 ].src;
+		}
+
+		next = this.removeDuplicateWhitespace( next );
+
+		return next;
+	}
+
+	/**
+	 * Checks whether a character is from a language that's written from right to left.
+	 * These languages don't have capital letter forms. Therefore any letter from these languages is a
+	 * potential sentence beginning.
+	 *
+	 * @param {string} letter The letter to check.
+	 *
+	 * @returns {boolean} Whether the letter is from an LTR language.
+	 */
+	isLetterFromSpecificLanguage( letter ) {
+		const ltrLetterRanges = [
+			// Hebrew characters.
+			/^[\u0590-\u05fe]+$/i,
+			// Arabic characters (used for Arabic, Farsi, Urdu).
+			/^[\u0600-\u06FF]+$/i,
+			// Additional Farsi characters.
+			/^[\uFB8A\u067E\u0686\u06AF]+$/i,
+		];
+
+		return (
+			ltrLetterRanges.some( ltrLetterRange => ltrLetterRange.test( letter ) )
+		);
+	}
+
+	/**
+	 * Checks if the sentenceBeginning beginning is a valid beginning.
+	 *
+	 * @param {string} sentenceBeginning The beginning of the sentence to validate.
+	 * @returns {boolean} Returns true if it is a valid beginning, false if it is not.
+	 */
+	isValidSentenceBeginning( sentenceBeginning ) {
+		return ( this.isCapitalLetter( sentenceBeginning ) ||
+				this.isLetterFromSpecificLanguage( sentenceBeginning ) ||
+				this.isNumber( sentenceBeginning ) ||
+				this.isQuotation( sentenceBeginning ) ||
+				this.isPunctuation( sentenceBeginning ) ||
+				this.isSmallerThanSign( sentenceBeginning ) );
+	}
+
+	/**
+	 * Checks if the token is a valid sentence start.
+	 *
+	 * @param {Object} token The token to validate.
+	 * @returns {boolean} Returns true if the token is valid sentence start, false if it is not.
+	 */
+	isSentenceStart( token ) {
+		return ( ! isUndefined( token ) && (
+			"html-start" === token.type ||
+			"html-end" === token.type ||
+			"block-start" === token.type
+		) );
+	}
+
+	/**
+	 * Checks if the token is a valid sentence ending. A valid sentence ending is either a full stop or another
+	 * delimiter such as "?", "!", etc.
+	 *
+	 * @param {Object} token The token to validate.
+	 * @returns {boolean} Returns true if the token is valid sentence ending, false if it is not.
+	 */
+	isSentenceEnding( token ) {
+		return (
+			! isUndefined( token ) &&
+			( token.type === "full-stop" || token.type === "sentence-delimiter" )
+		);
+	}
+
+	/**
+	 * Checks if a full stop is part of a person's initials.
+	 *
+	 * Tests if tokens exist. Then tests if the tokens are of the right type.
+	 * For previous token, it checks if the sentence ends with a single letter.
+	 * For nextToken it checks if it is a single letter.
+	 * Checks if next token is followed by a full stop.
+	 *
+	 * @param {object} token The current token (must be a full stop).
+	 * @param {object} previousToken The token before the full stop.
+	 * @param {object} nextToken The token following the full stop.
+	 * @param {object} secondToNextToken The second token after the full stop.
+	 * @returns {boolean} True if a full stop is part of a person's initials, False if the full stop is not part of a person's initials.
+	 */
+	isPartOfPersonInitial( token, previousToken, nextToken, secondToNextToken ) {
+		return ( ! isUndefined( token ) &&
+			! isUndefined( nextToken ) &&
+			! isUndefined( secondToNextToken ) &&
+			! isUndefined( previousToken ) &&
+			token.type === "full-stop" &&
+			previousToken.type === "sentence" &&
+			lastCharacterPartOfInitialsRegex.test( previousToken.src ) &&
+			nextToken.type === "sentence" &&
+			nextToken.src.trim().length === 1 &&
+			secondToNextToken.type === "full-stop"
+		);
+	}
+
+	/**
+	 * Tokens that represent a '<', followed by content until it enters another '<' or '>'
+	 * gets another pass by the tokenizer.
+	 *
+	 * @param {Object} token A token of type 'smaller-than-sign-content'.
+	 * @param {string[]} tokenSentences The current array of found sentences. Sentences may get added by this method.
+	 * @param {string} currentSentence The current sentence. Sentence parts may get appended by this method.
+	 * @returns {{tokenSentences, currentSentence}} The found sentences and the current sentence, appended when necessary.
+	 */
+	tokenizeSmallerThanContent( token, tokenSentences, currentSentence ) {
+		/*
+			Remove the '<' from the text, to avoid matching this rule
+			recursively again and again.
+			We add it again later on.
+		*/
+		const localText = token.src.substring( 1 );
+
+		// Tokenize the current smaller-than-content token without the first '<'.
+		const tokenizerResult = this.createTokenizer();
+		this.tokenize( tokenizerResult.tokenizer, localText );
+		const localSentences = this.getSentencesFromTokens( tokenizerResult.tokens, false );
+
+		localSentences[ 0 ] = isUndefined( localSentences[ 0 ] ) ? "<" : "<" + localSentences[ 0 ];
+
+		/*
+		 * When the first sentence has a valid sentence beginning.
+		 * Add the currently build sentence to the sentences.
+		 * Start building the next sentence.
+		 */
+		if ( this.isValidSentenceBeginning( localSentences[ 0 ] ) ) {
+			tokenSentences.push( currentSentence );
+			currentSentence = "";
+		}
+		currentSentence += localSentences[ 0 ];
+
+		if ( localSentences.length > 1 ) {
+			/*
+				There is a new sentence after the first,
+				add and reset the current sentence.
+			 */
+			tokenSentences.push( currentSentence );
+			currentSentence = "";
+
+			// Remove the first sentence (we do not need to add it again).
+			localSentences.shift();
+			// Last sentence gets special treatment.
+			const lastSentence = localSentences.pop();
+
+			// Add the remaining found sentences.
+			localSentences.forEach( sentence => {
+				tokenSentences.push( sentence );
+			} );
+
+			const sentenceEndRegex = new RegExp( "[" + fullStop + this.getSentenceDelimiters() + "]$" );
+
+			// Check if the last sentence has a valid sentence ending.
+			if ( lastSentence.match( sentenceEndRegex ) ) {
+				// If so, add it as a sentence.
+				tokenSentences.push( lastSentence );
+			} else {
+				// If not, start making a new one.
+				currentSentence = lastSentence;
+			}
+		}
+		return {
+			tokenSentences,
+			currentSentence,
+		};
+	}
+
+	/**
+	 * Creates a tokenizer.
+	 *
+	 * @returns {Object} The tokenizer and the tokens.
+	 */
+	createTokenizer() {
+		const sentenceDelimiterRegex = new RegExp( "^[" + this.getSentenceDelimiters() + "]$" );
+		const sentenceRegex = new RegExp( "^[^" + fullStop + this.getSentenceDelimiters() + "<\\(\\)\\[\\]]+$" );
+
+		const tokens = [];
+		const tokenizer = core( function( token ) {
+			tokens.push( token );
+		} );
+
+		tokenizer.addRule( fullStopRegex, "full-stop" );
+		tokenizer.addRule( smallerThanContentRegex, "smaller-than-sign-content" );
+		tokenizer.addRule( htmlStartRegex, "html-start" );
+		tokenizer.addRule( htmlEndRegex, "html-end" );
+		tokenizer.addRule( blockStartRegex, "block-start" );
+		tokenizer.addRule( blockEndRegex, "block-end" );
+		tokenizer.addRule( sentenceDelimiterRegex, "sentence-delimiter" );
+		tokenizer.addRule( sentenceRegex, "sentence" );
+
+		return {
+			tokenizer,
+			tokens,
+		};
+	}
+
+	/**
+	 * Tokenizes the given text using the given tokenizer.
+	 *
+	 * @param {Object} tokenizer The tokenizer to use.
+	 * @param {string} text The text to tokenize.
+	 * @returns {void}
+	 */
+	tokenize( tokenizer, text ) {
+		tokenizer.onText( text );
+
+		try {
+			tokenizer.end();
+		} catch ( e ) {
+			console.error( "Tokenizer end error:", e, e.tokenizer2 );
+		}
+	}
+
+	/**
+	 * Checks if a string ends with an abbreviation.
+	 * @param {string} currentSentence A (part of) a sentence.
+	 * @returns {boolean} True if the string ends with an abbreviation that is in abbreviations.js. Otherwise, False.
+	 */
+	endsWithAbbreviation( currentSentence ) {
+		const matchedAbbreviations = currentSentence.match( abbreviationsRegex );
+
+		if ( ! matchedAbbreviations ) {
+			return false;
+		}
+
+		const lastAbbreviation = matchedAbbreviations.pop();
+		return currentSentence.endsWith( lastAbbreviation );
+	}
+
+	/**
+	 * Checks whether the given tokens are a valid html tag pair.
+	 * Note that this method is not a full html tag validator. It should be replaced with a better solution once the html parser is implemented.
+	 *
+	 * @param {object} firstToken   The first token to check. It is asserted that this token contains/is an opening html tag.
+	 * @param {object} lastToken    The last token to check. It is asserted that this token contains/is a closing html tag.
+	 *
+	 * @returns {boolean} True if the tokens are a valid html tag pair. Otherwise, False.
+	 */
+	isValidTagPair( firstToken, lastToken ) {
+		const firstTokenText = firstToken.src;
+		const lastTokenText = lastToken.src;
+
+		// Get the tag types.
+		const firstTagType = firstTokenText.match( tagTypeRegex )[ 1 ];
+		const lastTagType  = lastTokenText.match( tagTypeRegex )[ 1 ];
+
+
+		// Check if the tags are the same and if they are a semantic tag (p, div, h1, h2, h3, h4, h5, h6, span).
+		return firstTagType === lastTagType && semanticTags.includes( firstTagType );
+	}
+
+	/**
+	 * Returns an array of sentences for a given array of tokens, assumes that the text has already been split into blocks.
+	 *
+	 * @param {Object[]} tokenArray The tokens from the sentence tokenizer.
+	 * @param {boolean} [trimSentences=true] Whether to trim the sentences at the end or not.
+	 *
+	 * @returns {string[]} A list of sentences.
+	 */
+	getSentencesFromTokens( tokenArray, trimSentences = true ) {
+		let tokenSentences = [], currentSentence = "", nextSentenceStart, sliced;
+
+		// Drop the first and last HTML tag if both are present.
+		do {
+			sliced = false;
+			const firstToken = tokenArray[ 0 ];
+			const lastToken = tokenArray[ tokenArray.length - 1 ];
+
+			if ( firstToken && lastToken && firstToken.type === "html-start" &&
+				lastToken.type === "html-end" && this.isValidTagPair( firstToken, lastToken ) ) {
+				tokenArray = tokenArray.slice( 1, tokenArray.length - 1 );
+
+				sliced = true;
+			}
+		} while ( sliced && tokenArray.length > 1 );
+
+		tokenArray.forEach( ( token, i ) => {
+			let hasNextSentence, nextCharacters, tokenizeResults;
+			const nextToken = tokenArray[ i + 1 ];
+			const previousToken = tokenArray[ i - 1 ];
+			const secondToNextToken = tokenArray[ i + 2 ];
+			nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
+
+			// For a new sentence we need to check the next two characters.
+			hasNextSentence = nextCharacters.length >= 2;
+			nextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : "";
+
+			switch ( token.type ) {
+				case "html-start":
+				case "html-end":
+					if ( this.isBreakTag( token.src ) ) {
+						tokenSentences.push( currentSentence );
+						currentSentence = "";
+					} else {
+						currentSentence += token.src;
+					}
+					break;
+
+				case "smaller-than-sign-content":
+					tokenizeResults = this.tokenizeSmallerThanContent( token, tokenSentences, currentSentence );
+					tokenSentences = tokenizeResults.tokenSentences;
+					currentSentence = tokenizeResults.currentSentence;
+					break;
+				case "sentence":
+					currentSentence += token.src;
+					break;
+				case "sentence-delimiter":
+					currentSentence += token.src;
+
+					/*
+				     * Only split text into sentences if:
+				     * the next token is defined, AND
+				     * the next token type is neither "block-end" nor "sentence-delimiter", AND
+				     * the next token first character is a white space
+				    */
+					if ( ! isUndefined( nextToken ) &&
+						"block-end" !== nextToken.type &&
+						"sentence-delimiter" !== nextToken.type &&
+						this.isCharacterASpace( nextToken.src[ 0 ] ) ) {
+						// Don't split on quotation marks unless they're preceded by a full stop.
+						if ( this.isQuotation( token.src ) && previousToken && previousToken.src !== "." ) {
+							break;
+						}
+						/*
+				         * Only split on ellipsis or quotation marks when:
+					     * a) There is a next sentence, and the next character is a valid sentence beginning preceded by a white space, OR
+					     * b) The next token is a sentence start
+					    */
+						if ( this.isQuotation( token.src ) || token.src === "…" ) {
+							currentSentence = this.getValidSentence( hasNextSentence,
+								nextSentenceStart,
+								nextCharacters,
+								nextToken,
+								tokenSentences,
+								currentSentence );
+						} else {
+							tokenSentences.push( currentSentence );
+							currentSentence = "";
+						}
+					}
+					break;
+
+				case "full-stop":
+					currentSentence += token.src;
+					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
+
+					// For a new sentence we need to check the next two characters.
+					hasNextSentence = nextCharacters.length >= 2;
+					nextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : "";
+
+					// If the current sentence ends with an abbreviation, the full stop does not split the sentence.
+					if ( this.endsWithAbbreviation( currentSentence ) ) {
+						break;
+					}
+
+					// It should not split the text if the first character of the potential next sentence is a number.
+					if ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {
+						break;
+					}
+
+					// If the full stop is part of a person's initials, don't split sentence.
+					if ( this.isPartOfPersonInitial( token, previousToken, nextToken, secondToNextToken ) ) {
+						break;
+					}
+
+					// If the full stop is an ordinal dot (in German), then don't break the sentence.
+					// This check should be done after  hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) (above).
+					// Because otherwise it could break before that test.
+					if ( this.endsWithOrdinalDot( currentSentence ) ) {
+						break;
+					}
+
+					/*
+					 * Only split on full stop when:
+					 * a) There is a next sentence, and the next character is a valid sentence beginning preceded by a white space, OR
+					 * b) The next token is a sentence start
+					 */
+					currentSentence = this.getValidSentence( hasNextSentence,
+						nextSentenceStart,
+						nextCharacters,
+						nextToken,
+						tokenSentences,
+						currentSentence );
+
+					break;
+
+				case "block-start":
+					currentSentence += token.src;
+					break;
+
+				case "block-end":
+					currentSentence += token.src;
+
+					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
+
+					// For a new sentence we need to check the next two characters.
+					hasNextSentence = nextCharacters.length >= 2;
+					nextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : "";
+
+					/* Don't split if:
+					 * - The next character is a number. For example: IPv4-numbers.
+					 * - The block end is preceded by a valid sentence ending, but not followed by a valid sentence beginning.
+					 */
+					if (
+						hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ||
+						( this.isSentenceEnding( previousToken ) &&
+							( ! ( this.isValidSentenceBeginning( nextSentenceStart ) || this.isSentenceStart( nextToken ) ) ) )
+					) {
+						break;
+					}
+
+					/*
+					 * Split if:
+					 * - The block end is preceded by a sentence ending and followed by a valid sentence beginning.
+					 */
+					if (
+						this.isSentenceEnding( previousToken ) &&
+						( this.isSentenceStart( nextToken ) || this.isValidSentenceBeginning( nextSentenceStart ) )
+					) {
+						tokenSentences.push( currentSentence );
+						currentSentence = "";
+					}
+					break;
+			}
+		} );
+
+		if ( "" !== currentSentence ) {
+			tokenSentences.push( currentSentence );
+		}
+
+		if ( trimSentences ) {
+			tokenSentences = map( tokenSentences, function( sentence ) {
+				return sentence.trim();
+			} );
+		}
+
+		return tokenSentences;
+	}
+
+	/**
+	 * Gets the current sentence when:
+	 * a) There is a next sentence, and the next character is a valid sentence beginning preceded by a white space, OR
+	 * b) The next token is a sentence start
+	 *
+	 * @param {boolean} hasNextSentence     Whether the next characters are more than two.
+	 * @param {string} nextSentenceStart    The second character of the next characters.
+	 * @param {string} nextCharacters       The string values of the next two tokens.
+	 * @param {object} nextToken            The next token object.
+	 * @param {array} tokenSentences        The array of pushed valid sentences.
+	 * @param {string} currentSentence      The current sentence.
+	 *
+	 * @returns {string} The current sentence.
+	 */
+	getValidSentence( hasNextSentence, nextSentenceStart, nextCharacters, nextToken, tokenSentences, currentSentence ) {
+		if ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) && this.isCharacterASpace( nextCharacters[ 0 ] ) ) ||
+			this.isSentenceStart( nextToken ) ) {
+			tokenSentences.push( currentSentence );
+			currentSentence = "";
+		}
+		return currentSentence;
+	}
+
+	/**
+	 * Checks if the character is a whitespace.
+	 *
+	 * @param {string} character    The character to check.
+	 * @returns {boolean}   Whether the character is a whitespace.
+	 */
+	isCharacterASpace( character ) {
+		return /\s/.test( character );
+	}
+}

package/src/languageProcessing/helpers/sentence/countSentences.js

@@ -0,0 +1,20 @@
+/** @module stringProcessing/countSentences */
+
+import getSentences from "./getSentences.js";
+
+/**
+ * Counts the number of sentences in a given string.
+ *
+ * @param {string}      text                The text used to count sentences.
+ * @param {function}    memoizedTokenizer   The memoized sentence tokenizer.
+ *
+ * @returns {number} The number of sentences in the text.
+ */
+export default function( text, memoizedTokenizer ) {
+	const sentences = getSentences( text, memoizedTokenizer );
+	let sentenceCount = 0;
+	for ( let i = 0; i < sentences.length; i++ ) {
+		sentenceCount++;
+	}
+	return sentenceCount;
+}