axyseo 2.0.0-alpha.0.0.4 → 2.0.0-alpha.0.0.5

package/src/parsedPaper/build/linguisticParsing/Sentence.js

@@ -1,89 +0,0 @@
-/**
- * Represents a sentence in a text.
- */
-class Sentence {
-	/**
-	 * Represents a sentence within a text.
-	 *
-	 * @param {string} text     The text of this sentence.
-	 * @param {number} startIndex   The start index of this sentence.
-	 * @param {number} endIndex     The end index of this sentence.
-	 */
-	constructor( text, startIndex = 0, endIndex = 0 ) {
-		this.text = text;
-		this.words = [];
-		this.startIndex = startIndex;
-		this.endIndex = endIndex;
-	}
-
-	/**
-	 * Sets a text for this sentence.
-	 *
-	 * @param {string} text The text to be set for the sentence.
-	 *
-	 * @returns {void}
-	 */
-	setText( text ) {
-		this.text = text;
-	}
-
-	/**
-	 * Appends text to this sentence.
-	 *
-	 * @param {string} text The text to be added to the sentence.
-	 *
-	 * @returns {void}
-	 */
-	appendText( text ) {
-		this.text += text;
-	}
-
-	/**
-	 * Returns the text of this sentence.
-	 *
-	 * @returns {string} The text of this senence.
-	 */
-	getText() {
-		return this.text;
-	}
-
-	/**
-	 * Sets the start index of this sentence.
-	 *
-	 * @param {number} startIndex The start index of this sentence.
-	 * @returns {void}
-	 */
-	setStartIndex( startIndex ) {
-		this.startIndex = startIndex;
-	}
-
-	/**
-	 * Sets the end index of this sentence.
-	 *
-	 * @param {number} endIndex The end index of this sentence.
-	 * @returns {void}
-	 */
-	setEndIndex( endIndex ) {
-		this.endIndex = endIndex;
-	}
-
-	/**
-	 * Returns the start index of this sentence.
-	 *
-	 * @returns {number} The start index of this sentence.
-	 */
-	getStartIndex() {
-		return this.startIndex;
-	}
-
-	/**
-	 * Returns the end index of this sentence.
-	 *
-	 * @returns {number} The end index of this sentence.
-	 */
-	getEndIndex() {
-		return this.endIndex;
-	}
-}
-
-export default Sentence;

package/src/parsedPaper/build/linguisticParsing/SentenceTokenizer.js

@@ -1,323 +0,0 @@
-import { isNaN, isUndefined } from "lodash";
-
-import core from "tokenizer2/core";
-import { normalize as normalizeQuotes } from "../../../languageProcessing/helpers/sanitize/quotes";
-
-import Sentence from "./Sentence";
-
-// All characters that indicate a sentence delimiter.
-const fullStop = ".";
-// The \u2026 character is an ellipsis
-const sentenceDelimiters = "?!;\u2026";
-
-const fullStopRegex = new RegExp( "^[" + fullStop + "]$" );
-const sentenceDelimiterRegex = new RegExp( "^[" + sentenceDelimiters + "]$" );
-const sentenceRegex = new RegExp( "^[^" + fullStop + sentenceDelimiters + "\\(\\)\\[\\]]+$" );
-
-const blockStartRegex = /^\s*[[({]\s*$/;
-const blockEndRegex = /^\s*[\])}]\s*$/;
-
-const whiteSpaceStartRegex = /^\s*/;
-const whiteSpaceEndRegex = /\s*$/;
-
-/**
- * Class for tokenizing a (html) text into sentences.
- */
-export default class SentenceTokenizer {
-	/**
-	 * Returns whether or not a certain character is a number.
-	 *
-	 * @param {string} character The character to check.
-	 * @returns {boolean} Whether or not the character is a capital letter.
-	 */
-	isNumber( character ) {
-		return ! isNaN( parseInt( character, 10 ) );
-	}
-
-	/**
-	 * Returns whether or not a given character is quotation mark.
-	 *
-	 * @param {string} character The character to check.
-	 *
-	 * @returns {boolean} Whether or not the given character is a quotation mark.
-	 */
-	isQuotation( character ) {
-		character = normalizeQuotes( character );
-
-		return "'" === character ||
-			"\"" === character;
-	}
-
-	/**
-	 * Returns whether or not a given character is a punctuation mark that can be at the beginning
-	 * of a sentence, like ¿ and ¡ used in Spanish.
-	 *
-	 * @param {string} character The character to check.
-	 *
-	 * @returns {boolean} Whether or not the given character is a punctuation mark.
-	 */
-	isPunctuation( character ) {
-		return "¿" === character ||
-			"¡" === character;
-	}
-
-	/**
-	 * Removes duplicate whitespace from a given text.
-	 *
-	 * @param {string} text The text with duplicate whitespace.
-	 * @returns {string} The text without duplicate whitespace.
-	 */
-	removeDuplicateWhitespace( text ) {
-		return text.replace( /\s+/, " " );
-	}
-
-	/**
-	 * Returns whether or not a certain character is a capital letter.
-	 *
-	 * @param {string} character The character to check.
-	 *
-	 * @returns {boolean} Whether or not the character is a capital letter.
-	 */
-	isCapitalLetter( character ) {
-		return character !== character.toLocaleLowerCase();
-	}
-
-	/**
-	 * Retrieves the next two characters from an array with the two next tokens.
-	 *
-	 * @param {Array} nextTokens The two next tokens. Might be undefined.
-	 *
-	 * @returns {string} The next two characters.
-	 */
-	getNextTwoCharacters( nextTokens ) {
-		let next = "";
-
-		if ( ! isUndefined( nextTokens[ 0 ] ) ) {
-			next += nextTokens[ 0 ].src;
-		}
-
-		if ( ! isUndefined( nextTokens[ 1 ] ) ) {
-			next += nextTokens[ 1 ].src;
-		}
-
-		next = this.removeDuplicateWhitespace( next );
-
-		return next;
-	}
-
-	/**
-	 * Checks if the sentenceBeginning beginning is a valid beginning.
-	 *
-	 * @param {string} sentenceBeginning The beginning of the sentence to validate.
-	 *
-	 * @returns {boolean} Returns true if it is a valid beginning, false if it is not.
-	 */
-	isValidSentenceBeginning( sentenceBeginning ) {
-		return (
-			this.isCapitalLetter( sentenceBeginning ) ||
-			this.isNumber( sentenceBeginning ) ||
-			this.isQuotation( sentenceBeginning ) ||
-			this.isPunctuation( sentenceBeginning )
-		);
-	}
-
-	/**
-	 * Checks if the token is a valid sentence ending.
-	 *
-	 * @param {Object} token The token to validate.
-	 *
-	 * @returns {boolean} Returns true if the token is valid ending, false if it is not.
-	 */
-	isSentenceStart( token ) {
-		return ( ! isUndefined( token ) && (
-			"html-start" === token.type ||
-			"html-end" === token.type ||
-			"block-start" === token.type
-		) );
-	}
-
-	/**
-	 * Creates a tokenizer.
-	 *
-	 * @returns {Object} The tokenizer and the tokens.
-	 */
-	createTokenizer() {
-		const tokens = [];
-		const tokenizer = core( function( token ) {
-			tokens.push( token );
-		} );
-
-		tokenizer.addRule( fullStopRegex, "full-stop" );
-		tokenizer.addRule( blockStartRegex, "block-start" );
-		tokenizer.addRule( blockEndRegex, "block-end" );
-		tokenizer.addRule( sentenceDelimiterRegex, "sentence-delimiter" );
-		tokenizer.addRule( sentenceRegex, "sentence" );
-
-		return {
-			tokenizer,
-			tokens,
-		};
-	}
-
-	/**
-	 * Tokenizes the given text using the given tokenizer.
-	 *
-	 * @param {Tokenizer} tokenizer The tokenizer to use.
-	 * @param {string}    text      The text to tokenize.
-	 *
-	 * @returns {void}
-	 */
-	tokenize( tokenizer, text ) {
-		tokenizer.onText( text );
-
-		try {
-			tokenizer.end();
-		} catch ( e ) {
-			console.error( "Tokenizer end error:", e, e.tokenizer2 );
-		}
-	}
-
-	/**
-	 * Determines the start and end indices of a set of sentences form a text.
-	 *
-	 * @param {Sentence[]} sentences A set of sentences for which to determine indices.
-	 *
-	 * @returns {void}
-	 */
-	determineIndices( sentences ) {
-		let currentIndex = 0;
-
-		for ( const sentence of sentences ) {
-			const startIndex = currentIndex;
-			sentence.setStartIndex( currentIndex );
-			const endIndex = startIndex + sentence.text.length - 1;
-			sentence.setEndIndex( endIndex );
-			currentIndex = endIndex + 1;
-		}
-	}
-
-	/**
-	 * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.
-	 *
-	 * @param {Sentence} sentence The sentence for which to trim the white space at the start.
-	 *
-	 * @returns {void}
-	 */
-	trimWhiteSpaceAtStart( sentence ) {
-		const whiteSpaceLength = sentence.text.match( whiteSpaceStartRegex )[ 0 ].length;
-		sentence.setText( sentence.getText().slice( whiteSpaceLength ) );
-		sentence.setStartIndex( sentence.getStartIndex() + whiteSpaceLength  );
-	}
-
-	/**
-	 * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.
-	 *
-	 * @param {Sentence} sentence The sentence for which to trim the white space at the end.
-	 *
-	 * @returns {void}
-	 */
-	trimWhiteSpaceAtEnd( sentence ) {
-		const whiteSpaceLength = sentence.text.match( whiteSpaceEndRegex )[ 0 ].length;
-		sentence.setText( sentence.getText().slice( 0, sentence.getText().length - whiteSpaceLength ) );
-		sentence.setEndIndex( sentence.getEndIndex() - whiteSpaceLength );
-	}
-
-	/**
-	 * Trims white space from the beginning and end of sentences and adjusts the indices
-	 * of the sentence beginnings and ends accordingly.
-	 *
-	 * @param {Sentence[]} sentences The sentences for which to trim the whitespace.
-	 *
-	 * @returns {void}
-	 */
-	trimWhiteSpaces( sentences ) {
-		for ( const sentence of sentences ) {
-			this.trimWhiteSpaceAtStart( sentence );
-			this.trimWhiteSpaceAtEnd( sentence );
-		}
-	}
-
-	/**
-	 * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.
-	 *
-	 * @param {Object[]} tokenArray The tokens from the sentence tokenizer.
-	 *
-	 * @returns {Sentence[]} An array of sentence objects.
-	 */
-	getSentencesFromTokens( tokenArray ) {
-		const tokenSentences = [];
-		let currentSentence = new Sentence( "", 0, 0 ),
-			nextSentenceStart;
-
-		tokenArray.forEach( ( token, i ) => {
-			let hasNextSentence, nextCharacters;
-			const nextToken = tokenArray[ i + 1 ];
-			const secondToNextToken = tokenArray[ i + 2 ];
-
-			switch ( token.type ) {
-				case "sentence":
-					currentSentence.appendText( token.src );
-					break;
-
-				case "sentence-delimiter":
-					currentSentence.appendText( token.src );
-					if ( ! isUndefined( nextToken ) && "block-end" !== nextToken.type && "sentence-delimiter" !== nextToken.type ) {
-						tokenSentences.push( currentSentence );
-						currentSentence = new Sentence( "" );
-					}
-					break;
-
-				case "full-stop":
-					currentSentence.appendText( token.src );
-
-					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
-
-					// For a new sentence we need to check the next two characters.
-					hasNextSentence = nextCharacters.length >= 2;
-					nextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : "";
-					// If the next character is a number, never split. For example: IPv4-numbers.
-					if ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {
-						break;
-					}
-					// Only split on sentence delimiters when the next sentence looks like the start of a sentence.
-					if ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {
-						tokenSentences.push( currentSentence );
-						currentSentence = new Sentence( "" );
-					}
-					break;
-
-				case "block-start":
-					currentSentence.appendText( token.src );
-					break;
-
-				case "block-end":
-					currentSentence.appendText( token.src );
-					nextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );
-
-					// For a new sentence we need to check the next two characters.
-					hasNextSentence = nextCharacters.length >= 2;
-					nextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : "";
-					// If the next character is a number, never split. For example: IPv4-numbers.
-					if ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {
-						break;
-					}
-
-					if ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {
-						tokenSentences.push( currentSentence );
-						currentSentence = new Sentence( "" );
-					}
-					break;
-			}
-		} );
-
-		if ( currentSentence.getText()  !== ""  ) {
-			tokenSentences.push( currentSentence );
-		}
-
-		this.determineIndices( tokenSentences );
-
-		this.trimWhiteSpaces( tokenSentences );
-
-		return tokenSentences;
-	}
-}

package/src/parsedPaper/build/linguisticParsing/parseText.js

@@ -1,20 +0,0 @@
-import SentenceTokenizer from "./SentenceTokenizer";
-
-/**
- * Parses a text into sentences.
- *
- * @param {string} text The text to parse.
- *
- * @returns {string[]} An array of sentence objects.
- */
-const parseTextIntoSentences = function( text ) {
-	const  sentenceTokenizer = new SentenceTokenizer();
-	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
-
-	sentenceTokenizer.tokenize( tokenizer, text );
-	return sentenceTokenizer.getSentencesFromTokens( tokens );
-};
-
-export {
-	parseTextIntoSentences,
-};

package/src/parsedPaper/build/tree/TreeBuilder.js

@@ -1,75 +0,0 @@
-import buildTree from "./html/buildTree";
-
-/**
- * Builds a tree representation of a source text.
- *
- * Currently supported languages:
- *  - HTML (`"html"`)
- *
- * @memberOf module:parsedPaper/builder
- */
-class TreeBuilder {
-	/**
-	 * Creates a new TreeBuilder.
-	 */
-	constructor() {
-		/**
-		 * A registry holding the parse function to be called
-		 * for each supported formatting language.
-		 *
-		 * @type {Object<string,Function>}
-		 * @private
-		 */
-		this._buildFunctions = {
-			html: buildTree,
-		};
-
-		this.build = this.build.bind( this );
-		this.register = this.register.bind( this );
-	}
-
-	/**
-	 * Parses the given source text to a tree representation.
-	 *
-	 * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).
-	 *
-	 * @param {string} sourceText                The source text that needs to be parsed to a tree representation.
-	 * @param {Object} [options]                 Parse options.
-	 * @param {string} [options.language="html"] The formatting language in which the source text has been written, e.g. `"html"` for HTML.
-	 *
-	 * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.
-	 */
-	build( sourceText, options = { language: "html" } ) {
-		return this._buildFunctions[ options.language ]( sourceText );
-	}
-
-	/**
-	 * Registers a build function for the given formatting language.
-	 *
-	 * Call this method if you want to add support for another formatting language.
-	 *
-	 * @example
-	 *   // Create a new build function.
-	 *   const myBuilder = sourceText => {
-	 *       const node = new Paragraph();
-	 *       node.textContainer.appendText( sourceText );
-	 *       return node;
-	 *   };
-	 *
-	 *   // Register the build function for the language "my-language".
-	 *   treeBuilder.register( "my-language", myBuilder );
-	 *
-	 *   // Build the tree using the registered builder.
-	 *   const tree = treeBuilder.build( "some input", { language: "my-language" } );
-	 *
-	 * @param {string}                                                            language      The language to register.
-	 * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.
-	 *
-	 * @returns {void}
-	 */
-	register( language, buildFunction ) {
-		this._buildFunctions[ language ] = buildFunction;
-	}
-}
-
-export default TreeBuilder;

package/src/parsedPaper/build/tree/cleanup/calculateTextIndices.js

@@ -1,190 +0,0 @@
-import { pullAll } from "lodash";
-
-import { ignoredHtmlElements } from "../html/htmlConstants";
-
-/**
- * Gathers all elements that can be closed given the position of the current element in the source code.
- *
- * Elements that can be closed are all elements that are opened before this element, but in which this element is
- * not nested.
- * E.g.
- * ```html
- * <strong>Hello</strong><em>World<b>!!!</b></em>`
- * ```
- * with `<b>!!!</b>` as the current element,
- * means that the `<strong>` needs to be closed, but `<em>` **not**.
- *
- * @param {module:parsedPaper/structure.FormattingElement} currentElement The current element.
- * @param {module:parsedPaper/structure.FormattingElement[]} openElements The elements that are currently open.
- *
- * @returns {module:parsedPaper/structure.FormattingElement[]} The elements that can be closed.
- */
-const elementsThatCanBeClosed = function( currentElement, openElements ) {
-	return openElements.filter( el => {
-		const endTag = el.sourceCodeLocation.endTag;
-		return endTag.endOffset <= currentElement.sourceCodeLocation.startOffset;
-	} );
-};
-
-/**
- * Closes the elements that can be closed given the position of the current element within the source code.
- *
- * This does two things:
- *  1. The closed element's text end index is calculated based on the current offset.
- *  2. The closed element's end tag lengths are counted towards the current offset, to make sure that the computed position
- * of the formatting elements are still correct.
- *
- * @param {module:parsedPaper/structure.FormattingElement[]} elementsToClose The list of open elements that need to be closed
- * @param {number} currentOffset                                      The current offset when parsing the formatting elements
- *
- * @returns {number} The updated current offset
- *
- * @private
- */
-const closeElements = function( elementsToClose, currentOffset ) {
-	// Sort, so we close all elements in the right order.
-	elementsToClose.sort( ( a, b ) => a.sourceCodeLocation.endTag.endOffset - b.sourceCodeLocation.endTag.endOffset );
-
-	elementsToClose.forEach( elementToClose => {
-		const endTag = elementToClose.sourceCodeLocation.endTag;
-		// Set the end position as seen in the text.
-		elementToClose.textEndIndex = endTag.startOffset - currentOffset;
-		/*
-		  Add the end tag length of the to be closed element to the total offset.
-		 */
-		const endTagLength = endTag.endOffset - endTag.startOffset;
-		currentOffset += endTagLength;
-	} );
-
-	return currentOffset;
-};
-
-/**
- * Adds the content length of the given element (the part between the tags) to the current offset
- * and adds the content to the element as a parameter.
- *
- * @param {module:parsedPaper/structure.FormattingElement} element The element of which to add the content length.
- * @param {number} currentOffset                            The current offset to which to add the length to.
- *
- * @returns {number} The updated current offset
- */
-const handleIgnoredContent = function( element, currentOffset ) {
-	// Has 0 length in text, so end = start.
-	element.textEndIndex = element.textStartIndex;
-
-	// Update current offset.
-	const end = element.sourceCodeLocation.endTag ? element.sourceCodeLocation.endTag.startOffset : element.sourceCodeLocation.endOffset;
-	const start = element.sourceCodeLocation.startTag ? element.sourceCodeLocation.startTag.endOffset : element.sourceCodeLocation.startOffset;
-
-	currentOffset += end - start;
-
-	return currentOffset;
-};
-
-
-/**
- * Sets the start and end text positions of a comment.
- *
- * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
- * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
- * 																			(e.g., tags, comments).
- *
- * @returns {number} The length of the comment.
- *
- * @private
- */
-const computeCommentStartEndTextIndices = function( element, currentOffset ) {
-	element.textStartIndex = element.sourceCodeLocation.startOffset - currentOffset;
-	element.textEndIndex = element.textStartIndex;
-
-	return element.sourceCodeLocation.endOffset - element.sourceCodeLocation.startOffset;
-};
-
-/**
- * Sets the start and end text positions of one formatting element.
- *
- * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
- * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
- * 																			(e.g., tags, comments).
- *
- * @returns {int} The updated currentOffset.
- *
- * @private
- */
-const computeElementStartTextIndex = function( element, currentOffset ) {
-	const startTag = element.sourceCodeLocation.startTag;
-
-	// For example: "<strong>".length
-	const startTagLength = startTag.endOffset - startTag.startOffset;
-
-	currentOffset += startTagLength;
-
-	// Set start position of element in heading's / paragraph's text.
-	element.textStartIndex = startTag.endOffset - currentOffset;
-
-	/*
-	  Elements that have no end tags (e.g., void element like <img/> or self-closing elements) can be closed immediately.
-	  The text length of those elements will be automatically 0.
-	 */
-	if ( ! element.sourceCodeLocation.endTag ) {
-		element.textEndIndex = element.textStartIndex;
-	}
-
-	return currentOffset;
-};
-
-/**
- * Sets the start and end position of the text in formatting elements of the given node.
- *
- * @param {module:parsedPaper/structure.LeafNode} node The node containing a TextContainer
- *
- * @returns {void}
- *
- * @private
- */
-const calculateTextIndices = function( node ) {
-	if ( ! node.textContainer.formatting || node.textContainer.formatting.length === 0 ) {
-		return;
-	}
-
-	const openElements = [];
-
-	/*
-	  Keeps track of the current total size of the start and end tags (and the ignored content)
-	  These should not be counted towards the start and end position of the elements in the text.
-	 */
-	let currentOffset = node.sourceCodeLocation.startTag ? node.sourceCodeLocation.startTag.endOffset : node.sourceCodeLocation.startOffset;
-
-	node.textContainer.formatting.forEach( element => {
-		// Close elements that can be closed and remove them from the list of open elements.
-		const elementsToClose = elementsThatCanBeClosed( element, openElements );
-		currentOffset = closeElements( elementsToClose, currentOffset );
-		pullAll( openElements, elementsToClose );
-
-		// Comments are self-closing formatting elements that are completely ignored in rendering.
-		if ( element.type === "#comment" ) {
-			currentOffset += computeCommentStartEndTextIndices( element, currentOffset );
-			return;
-		}
-
-		currentOffset = computeElementStartTextIndex( element, currentOffset );
-
-		// If there is an endTag, the element should be closed in one of the next iterations of the loop.
-		if (  element.sourceCodeLocation.endTag ) {
-			openElements.push( element );
-		}
-
-		/*
-		  If this element is an ignored element its contents are not in the text,
-		  so its content should be added to the respective formatting element instead,
-		  and the current offset should be updated.
-		 */
-		if ( ignoredHtmlElements.includes( element.type ) ) {
-			currentOffset = handleIgnoredContent( element, currentOffset );
-		}
-	} );
-	// Close all remaining elements.
-	closeElements( openElements, currentOffset );
-};
-
-export default calculateTextIndices;