axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/parsedPaper/build/linguisticParsing/Sentence.js

@@ -0,0 +1,89 @@
+/**
+ * Represents a sentence in a text.
+ */
+class Sentence {
+	/**
+	 * Represents a sentence within a text.
+	 *
+	 * @param {string} text     The text of this sentence.
+	 * @param {number} startIndex   The start index of this sentence.
+	 * @param {number} endIndex     The end index of this sentence.
+	 */
+	constructor( text, startIndex = 0, endIndex = 0 ) {
+		this.text = text;
+		this.words = [];
+		this.startIndex = startIndex;
+		this.endIndex = endIndex;
+	}
+
+	/**
+	 * Sets a text for this sentence.
+	 *
+	 * @param {string} text The text to be set for the sentence.
+	 *
+	 * @returns {void}
+	 */
+	setText( text ) {
+		this.text = text;
+	}
+
+	/**
+	 * Appends text to this sentence.
+	 *
+	 * @param {string} text The text to be added to the sentence.
+	 *
+	 * @returns {void}
+	 */
+	appendText( text ) {
+		this.text += text;
+	}
+
+	/**
+	 * Returns the text of this sentence.
+	 *
+	 * @returns {string} The text of this senence.
+	 */
+	getText() {
+		return this.text;
+	}
+
+	/**
+	 * Sets the start index of this sentence.
+	 *
+	 * @param {number} startIndex The start index of this sentence.
+	 * @returns {void}
+	 */
+	setStartIndex( startIndex ) {
+		this.startIndex = startIndex;
+	}
+
+	/**
+	 * Sets the end index of this sentence.
+	 *
+	 * @param {number} endIndex The end index of this sentence.
+	 * @returns {void}
+	 */
+	setEndIndex( endIndex ) {
+		this.endIndex = endIndex;
+	}
+
+	/**
+	 * Returns the start index of this sentence.
+	 *
+	 * @returns {number} The start index of this sentence.
+	 */
+	getStartIndex() {
+		return this.startIndex;
+	}
+
+	/**
+	 * Returns the end index of this sentence.
+	 *
+	 * @returns {number} The end index of this sentence.
+	 */
+	getEndIndex() {
+		return this.endIndex;
+	}
+}
+
+export default Sentence;

package/src/parsedPaper/build/linguisticParsing/SentenceTokenizer.js

@@ -0,0 +1,323 @@
+import { isNaN, isUndefined } from "lodash";
+
+import core from "tokenizer2/core";
+import { normalize as normalizeQuotes } from "../../../languageProcessing/helpers/sanitize/quotes";
+
+import Sentence from "./Sentence";
+
+// All characters that indicate a sentence delimiter.
+const fullStop = ".";
+// The \u2026 character is an ellipsis
+const sentenceDelimiters = "?!;\u2026";
+
+const fullStopRegex = new RegExp( "^[" + fullStop + "]$" );
+const sentenceDelimiterRegex = new RegExp( "^[" + sentenceDelimiters + "]$" );
+const sentenceRegex = new RegExp( "^[^" + fullStop + sentenceDelimiters + "\\(\\)\\[\\]]+$" );
+
+const blockStartRegex = /^\s*[[({]\s*$/;
+const blockEndRegex = /^\s*[\])}]\s*$/;
+
+const whiteSpaceStartRegex = /^\s*/;
+const whiteSpaceEndRegex = /\s*$/;
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+export default class SentenceTokenizer {
+	/**
+	 * Returns whether or not a certain character is a number.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the character is a capital letter.
+	 */
+	isNumber( character ) {
+		return ! isNaN( parseInt( character, 10 ) );
+	}
+
+	/**
+	 * Returns whether or not a given character is quotation mark.
+	 *
+	 * @param {string} character The character to check.
+	 *
+	 * @returns {boolean} Whether or not the given character is a quotation mark.
+	 */
+	isQuotation( character ) {
+		character = normalizeQuotes( character );
+
+		return "'" === character ||
+			"\"" === character;
+	}
+
+	/**
+	 * Returns whether or not a given character is a punctuation mark that can be at the beginning
+	 * of a sentence, like ¿ and ¡ used in Spanish.
+	 *
+	 * @param {string} character The character to check.
+	 *
+	 * @returns {boolean} Whether or not the given character is a punctuation mark.
+	 */
+	isPunctuation( character ) {
+		return "¿" === character ||
+			"¡" === character;
+	}
+
+	/**
+	 * Removes duplicate whitespace from a given text.
+	 *
+	 * @param {string} text The text with duplicate whitespace.
+	 * @returns {string} The text without duplicate whitespace.
+	 */
+	removeDuplicateWhitespace( text ) {
+		return text.replace( /\s+/, " " );
+	}
+
+	/**
+	 * Returns whether or not a certain character is a capital letter.
+	 *
+	 * @param {string} character The character to check.
+	 *
+	 * @returns {boolean} Whether or not the character is a capital letter.
+	 */
+	isCapitalLetter( character ) {
+		return character !== character.toLocaleLowerCase();
+	}
+
+	/**
+	 * Retrieves the next two characters from an array with the two next tokens.
+	 *
+	 * @param {Array} nextTokens The two next tokens. Might be undefined.
+	 *
+	 * @returns {string} The next two characters.
+	 */
+	getNextTwoCharacters( nextTokens ) {
+		let next = "";
+
+		if ( ! isUndefined( nextTokens[ 0 ] ) ) {
+			next += nextTokens[ 0 ].src;
+		}
+
+		if ( ! isUndefined( nextTokens[ 1 ] ) ) {
+			next += nextTokens[ 1 ].src;
+		}
+
+		next = this.removeDuplicateWhitespace( next );
+
+		return next;
+	}
+
+	/**
+	 * Checks if the sentenceBeginning beginning is a valid beginning.
+	 *
+	 * @param {string} sentenceBeginning The beginning of the sentence to validate.
+	 *
+	 * @returns {boolean} Returns true if it is a valid beginning, false if it is not.
+	 */
+	isValidSentenceBeginning( sentenceBeginning ) {
+		return (
+			this.isCapitalLetter( sentenceBeginning ) ||
+			this.isNumber( sentenceBeginning ) ||
+			this.isQuotation( sentenceBeginning ) ||
+			this.isPunctuation( sentenceBeginning )
+		);
+	}
+
+	/**
+	 * Checks if the token is a valid sentence ending.
+	 *
+	 * @param {Object} token The token to validate.
+	 *
+	 * @returns {boolean} Returns true if the token is valid ending, false if it is not.
+	 */
+	isSentenceStart( token ) {
+		return ( ! isUndefined( token ) && (
+			"html-start" === token.type ||
+			"html-end" === token.type ||
+			"block-start" === token.type
+		) );
+	}
+
+	/**
+	 * Creates a tokenizer.
+	 *
+	 * @returns {Object} The tokenizer and the tokens.
+	 */
+	createTokenizer() {
+		const tokens = [];
+		const tokenizer = core( function( token ) {
+			tokens.push( token );
+		} );
+
+		tokenizer.addRule( fullStopRegex, "full-stop" );
+		tokenizer.addRule( blockStartRegex, "block-start" );
+		tokenizer.addRule( blockEndRegex, "block-end" );
+		tokenizer.addRule( sentenceDelimiterRegex, "sentence-delimiter" );
+		tokenizer.addRule( sentenceRegex, "sentence" );
+
+		return {
+			tokenizer,
+			tokens,
+		};
+	}
+
+	/**
+	 * Tokenizes the given text using the given tokenizer.
+	 *
+	 * @param {Tokenizer} tokenizer The tokenizer to use.
+	 * @param {string}    text      The text to tokenize.
+	 *
+	 * @returns {void}
+	 */
+	tokenize( tokenizer, text ) {
+		tokenizer.onText( text );
+
+		try {
+			tokenizer.end();
+		} catch ( e ) {
+			console.error( "Tokenizer end error:", e, e.tokenizer2 );
+		}
+	}
+
+	/**
+	 * Determines the start and end indices of a set of sentences form a text.
+	 *
+	 * @param {Sentence[]} sentences A set of sentences for which to determine indices.
+	 *
+	 * @returns {void}
+	 */
+	determineIndices( sentences ) {
+		let currentIndex = 0;
+
+		for ( const sentence of sentences ) {
+			const startIndex = currentIndex;
+			sentence.setStartIndex( currentIndex );
+			const endIndex = startIndex + sentence.text.length - 1;
+			sentence.setEndIndex( endIndex );
+			currentIndex = endIndex + 1;
+		}
+	}
+
+	/**
+	 * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.
+	 *
+	 * @param {Sentence} sentence The sentence for which to trim the white space at the start.
+	 *
+	 * @returns {void}
+	 */
+	trimWhiteSpaceAtStart( sentence ) {
+		const whiteSpaceLength = sentence.text.match( whiteSpaceStartRegex )[ 0 ].length;
+		sentence.setText( sentence.getText().slice( whiteSpaceLength ) );
+		sentence.setStartIndex( sentence.getStartIndex() + whiteSpaceLength  );
+	}
+
+	/**
+	 * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.
+	 *
+	 * @param {Sentence} sentence The sentence for which to trim the white space at the end.
+	 *
+	 * @returns {void}
+	 */
+	trimWhiteSpaceAtEnd( sentence ) {
+		const whiteSpaceLength = sentence.text.match( whiteSpaceEndRegex )[ 0 ].length;
+		sentence.setText( sentence.getText().slice( 0, sentence.getText().length - whiteSpaceLength ) );
+		sentence.setEndIndex( sentence.getEndIndex() - whiteSpaceLength );
+	}
+
+	/**
+	 * Trims white space from the beginning and end of sentences and adjusts the indices
+	 * of the sentence beginnings and ends accordingly.
+	 *
+	 * @param {Sentence[]} sentences The sentences for which to trim the whitespace.
+	 *
+	 * @returns {void}
+	 */
+	trimWhiteSpaces( sentences ) {
+		for ( const sentence of sentences ) {
+			this.trimWhiteSpaceAtStart( sentence );
+			this.trimWhiteSpaceAtEnd( sentence );
+		}
+	}
+
+	/**
+	 * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.
+	 *
+	 * @param {Object[]} tokenArray The tokens from the sentence tokenizer.
+	 *
+	 * @returns {Sentence[]} An array of sentence objects.
+	 */
+	getSentencesFromTokens( tokenArray ) {
+		const tokenSentences = [];
+		let currentSentence = new Sentence( "", 0, 0 ),
+			nextSentenceStart;
+
+		tokenArray.forEach( ( token, i ) => {
+			let hasNextSentence, nextCharacters;
+			const nextToken = tokenArray[ i + 1 ];
+			const secondToNextToken = tokenArray[ i + 2 ];
+
+			switch ( token.type ) {
+				case "sentence":
+					currentSentence.appendText( token.src );
+					break;
+
+				case "sentence-delimiter":
+					currentSentence.appendText( token.src );
+					if ( ! isUndefined( nextToken ) && "block-end" !== nextToken.type && "sentence-delimiter" !== nextToken.type ) {
+						tokenSentences.push( currentSentence );
+						currentSentence = new Sentence( "" );
+					}
+					break;
+
+				case "full-stop":
+					currentSentence.appendText( token.src );
+
+					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
+
+					// For a new sentence we need to check the next two characters.
+					hasNextSentence = nextCharacters.length >= 2;
+					nextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : "";
+					// If the next character is a number, never split. For example: IPv4-numbers.
+					if ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {
+						break;
+					}
+					// Only split on sentence delimiters when the next sentence looks like the start of a sentence.
+					if ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {
+						tokenSentences.push( currentSentence );
+						currentSentence = new Sentence( "" );
+					}
+					break;
+
+				case "block-start":
+					currentSentence.appendText( token.src );
+					break;
+
+				case "block-end":
+					currentSentence.appendText( token.src );
+					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
+
+					// For a new sentence we need to check the next two characters.
+					hasNextSentence = nextCharacters.length >= 2;
+					nextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : "";
+					// If the next character is a number, never split. For example: IPv4-numbers.
+					if ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {
+						break;
+					}
+
+					if ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {
+						tokenSentences.push( currentSentence );
+						currentSentence = new Sentence( "" );
+					}
+					break;
+			}
+		} );
+
+		if ( currentSentence.getText()  !== ""  ) {
+			tokenSentences.push( currentSentence );
+		}
+
+		this.determineIndices( tokenSentences );
+
+		this.trimWhiteSpaces( tokenSentences );
+
+		return tokenSentences;
+	}
+}

package/src/parsedPaper/build/linguisticParsing/parseText.js

@@ -0,0 +1,20 @@
+import SentenceTokenizer from "./SentenceTokenizer";
+
+/**
+ * Parses a text into sentences.
+ *
+ * @param {string} text The text to parse.
+ *
+ * @returns {string[]} An array of sentence objects.
+ */
+const parseTextIntoSentences = function( text ) {
+	const  sentenceTokenizer = new SentenceTokenizer();
+	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
+
+	sentenceTokenizer.tokenize( tokenizer, text );
+	return sentenceTokenizer.getSentencesFromTokens( tokens );
+};
+
+export {
+	parseTextIntoSentences,
+};

package/src/parsedPaper/build/tree/TreeBuilder.js

@@ -0,0 +1,75 @@
+import buildTree from "./html/buildTree";
+
+/**
+ * Builds a tree representation of a source text.
+ *
+ * Currently supported languages:
+ *  - HTML (`"html"`)
+ *
+ * @memberOf module:parsedPaper/builder
+ */
+class TreeBuilder {
+	/**
+	 * Creates a new TreeBuilder.
+	 */
+	constructor() {
+		/**
+		 * A registry holding the parse function to be called
+		 * for each supported formatting language.
+		 *
+		 * @type {Object<string,Function>}
+		 * @private
+		 */
+		this._buildFunctions = {
+			html: buildTree,
+		};
+
+		this.build = this.build.bind( this );
+		this.register = this.register.bind( this );
+	}
+
+	/**
+	 * Parses the given source text to a tree representation.
+	 *
+	 * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).
+	 *
+	 * @param {string} sourceText                The source text that needs to be parsed to a tree representation.
+	 * @param {Object} [options]                 Parse options.
+	 * @param {string} [options.language="html"] The formatting language in which the source text has been written, e.g. `"html"` for HTML.
+	 *
+	 * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.
+	 */
+	build( sourceText, options = { language: "html" } ) {
+		return this._buildFunctions[ options.language ]( sourceText );
+	}
+
+	/**
+	 * Registers a build function for the given formatting language.
+	 *
+	 * Call this method if you want to add support for another formatting language.
+	 *
+	 * @example
+	 *   // Create a new build function.
+	 *   const myBuilder = sourceText => {
+	 *       const node = new Paragraph();
+	 *       node.textContainer.appendText( sourceText );
+	 *       return node;
+	 *   };
+	 *
+	 *   // Register the build function for the language "my-language".
+	 *   treeBuilder.register( "my-language", myBuilder );
+	 *
+	 *   // Build the tree using the registered builder.
+	 *   const tree = treeBuilder.build( "some input", { language: "my-language" } );
+	 *
+	 * @param {string}                                                            language      The language to register.
+	 * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.
+	 *
+	 * @returns {void}
+	 */
+	register( language, buildFunction ) {
+		this._buildFunctions[ language ] = buildFunction;
+	}
+}
+
+export default TreeBuilder;

package/src/parsedPaper/build/tree/cleanup/calculateTextIndices.js

@@ -0,0 +1,190 @@
+import { pullAll } from "lodash";
+
+import { ignoredHtmlElements } from "../html/htmlConstants";
+
+/**
+ * Gathers all elements that can be closed given the position of the current element in the source code.
+ *
+ * Elements that can be closed are all elements that are opened before this element, but in which this element is
+ * not nested.
+ * E.g.
+ * ```html
+ * <strong>Hello</strong><em>World<b>!!!</b></em>`
+ * ```
+ * with `<b>!!!</b>` as the current element,
+ * means that the `<strong>` needs to be closed, but `<em>` **not**.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} currentElement The current element.
+ * @param {module:parsedPaper/structure.FormattingElement[]} openElements The elements that are currently open.
+ *
+ * @returns {module:parsedPaper/structure.FormattingElement[]} The elements that can be closed.
+ */
+const elementsThatCanBeClosed = function( currentElement, openElements ) {
+	return openElements.filter( el => {
+		const endTag = el.sourceCodeLocation.endTag;
+		return endTag.endOffset <= currentElement.sourceCodeLocation.startOffset;
+	} );
+};
+
+/**
+ * Closes the elements that can be closed given the position of the current element within the source code.
+ *
+ * This does two things:
+ *  1. The closed element's text end index is calculated based on the current offset.
+ *  2. The closed element's end tag lengths are counted towards the current offset, to make sure that the computed position
+ * of the formatting elements are still correct.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement[]} elementsToClose The list of open elements that need to be closed
+ * @param {number} currentOffset                                      The current offset when parsing the formatting elements
+ *
+ * @returns {number} The updated current offset
+ *
+ * @private
+ */
+const closeElements = function( elementsToClose, currentOffset ) {
+	// Sort, so we close all elements in the right order.
+	elementsToClose.sort( ( a, b ) => a.sourceCodeLocation.endTag.endOffset - b.sourceCodeLocation.endTag.endOffset );
+
+	elementsToClose.forEach( elementToClose => {
+		const endTag = elementToClose.sourceCodeLocation.endTag;
+		// Set the end position as seen in the text.
+		elementToClose.textEndIndex = endTag.startOffset - currentOffset;
+		/*
+		  Add the end tag length of the to be closed element to the total offset.
+		 */
+		const endTagLength = endTag.endOffset - endTag.startOffset;
+		currentOffset += endTagLength;
+	} );
+
+	return currentOffset;
+};
+
+/**
+ * Adds the content length of the given element (the part between the tags) to the current offset
+ * and adds the content to the element as a parameter.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} element The element of which to add the content length.
+ * @param {number} currentOffset                            The current offset to which to add the length to.
+ *
+ * @returns {number} The updated current offset
+ */
+const handleIgnoredContent = function( element, currentOffset ) {
+	// Has 0 length in text, so end = start.
+	element.textEndIndex = element.textStartIndex;
+
+	// Update current offset.
+	const end = element.sourceCodeLocation.endTag ? element.sourceCodeLocation.endTag.startOffset : element.sourceCodeLocation.endOffset;
+	const start = element.sourceCodeLocation.startTag ? element.sourceCodeLocation.startTag.endOffset : element.sourceCodeLocation.startOffset;
+
+	currentOffset += end - start;
+
+	return currentOffset;
+};
+
+
+/**
+ * Sets the start and end text positions of a comment.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {number} The length of the comment.
+ *
+ * @private
+ */
+const computeCommentStartEndTextIndices = function( element, currentOffset ) {
+	element.textStartIndex = element.sourceCodeLocation.startOffset - currentOffset;
+	element.textEndIndex = element.textStartIndex;
+
+	return element.sourceCodeLocation.endOffset - element.sourceCodeLocation.startOffset;
+};
+
+/**
+ * Sets the start and end text positions of one formatting element.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {int} The updated currentOffset.
+ *
+ * @private
+ */
+const computeElementStartTextIndex = function( element, currentOffset ) {
+	const startTag = element.sourceCodeLocation.startTag;
+
+	// For example: "<strong>".length
+	const startTagLength = startTag.endOffset - startTag.startOffset;
+
+	currentOffset += startTagLength;
+
+	// Set start position of element in heading's / paragraph's text.
+	element.textStartIndex = startTag.endOffset - currentOffset;
+
+	/*
+	  Elements that have no end tags (e.g., void element like <img/> or self-closing elements) can be closed immediately.
+	  The text length of those elements will be automatically 0.
+	 */
+	if ( ! element.sourceCodeLocation.endTag ) {
+		element.textEndIndex = element.textStartIndex;
+	}
+
+	return currentOffset;
+};
+
+/**
+ * Sets the start and end position of the text in formatting elements of the given node.
+ *
+ * @param {module:parsedPaper/structure.LeafNode} node The node containing a TextContainer
+ *
+ * @returns {void}
+ *
+ * @private
+ */
+const calculateTextIndices = function( node ) {
+	if ( ! node.textContainer.formatting || node.textContainer.formatting.length === 0 ) {
+		return;
+	}
+
+	const openElements = [];
+
+	/*
+	  Keeps track of the current total size of the start and end tags (and the ignored content)
+	  These should not be counted towards the start and end position of the elements in the text.
+	 */
+	let currentOffset = node.sourceCodeLocation.startTag ? node.sourceCodeLocation.startTag.endOffset : node.sourceCodeLocation.startOffset;
+
+	node.textContainer.formatting.forEach( element => {
+		// Close elements that can be closed and remove them from the list of open elements.
+		const elementsToClose = elementsThatCanBeClosed( element, openElements );
+		currentOffset = closeElements( elementsToClose, currentOffset );
+		pullAll( openElements, elementsToClose );
+
+		// Comments are self-closing formatting elements that are completely ignored in rendering.
+		if ( element.type === "#comment" ) {
+			currentOffset += computeCommentStartEndTextIndices( element, currentOffset );
+			return;
+		}
+
+		currentOffset = computeElementStartTextIndex( element, currentOffset );
+
+		// If there is an endTag, the element should be closed in one of the next iterations of the loop.
+		if (  element.sourceCodeLocation.endTag ) {
+			openElements.push( element );
+		}
+
+		/*
+		  If this element is an ignored element its contents are not in the text,
+		  so its content should be added to the respective formatting element instead,
+		  and the current offset should be updated.
+		 */
+		if ( ignoredHtmlElements.includes( element.type ) ) {
+			currentOffset = handleIgnoredContent( element, currentOffset );
+		}
+	} );
+	// Close all remaining elements.
+	closeElements( openElements, currentOffset );
+};
+
+export default calculateTextIndices;