axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/parse/build/private/tokenize.js

@@ -0,0 +1,74 @@
+import { Heading, Paragraph } from "../../structure";
+import getTextElementPositions from "./getTextElementPositions";
+import { hashedHtmlEntities } from "../../../helpers/htmlEntities";
+
+/**
+ * Splits the sentence into tokens, determines their positions in the source code, and puts them on the sentence.
+ *
+ * @param {Paragraph|Heading} node The paragraph or heading node to split into sentences.
+ * @param {Sentence} sentence The sentence.
+ * @param {LanguageProcessor} languageProcessor The language processor for the current language.
+ *
+ * @returns {Sentence} The sentence, with tokens.
+ */
+function getTokens( node, sentence, languageProcessor ) {
+	sentence.tokens = languageProcessor.splitIntoTokens( sentence );
+	sentence.tokens = getTextElementPositions( node, sentence.tokens, sentence.sourceCodeRange.startOffset );
+	return sentence;
+}
+
+/**
+ * Splits the node's inner text into sentences, and the sentences into tokens,
+ * using the language processor.
+ *
+ * @param {Paragraph|Heading} node 				The paragraph or heading node to split into sentences.
+ * @param {LanguageProcessor} languageProcessor The language processor to use.
+ *
+ * @returns {Sentence[]} The node's sentences.
+ */
+function getSentences( node, languageProcessor ) {
+	// Split text into sentences.
+	let sentences = languageProcessor.splitIntoSentences( node.innerText() );
+	// Add position information to the sentences.
+	sentences = getTextElementPositions( node, sentences );
+	// Tokenize sentences into tokens.
+	return sentences.map( sentence => {
+		sentence = getTokens( node, sentence, languageProcessor );
+		// Now positions have been determined, change HTML entities that had earlier been converted to hashed versions back to their short version.
+		// For example, "&" was earlier converted into "#amp;" and is now converted into "&".
+		// We make this change in both the Sentence and the accompanying Tokens.
+		hashedHtmlEntities.forEach( ( character, hashedHtmlEntity ) => {
+			// We use a global regex instead of replaceAll to support older browsers.
+			const hashedHtmlEntityRegex = new RegExp( hashedHtmlEntity, "g" );
+			sentence.text = sentence.text.replace( hashedHtmlEntityRegex, character );
+			sentence.tokens.map( token => {
+				token.text = token.text.replace( hashedHtmlEntityRegex, character );
+				return token;
+			} );
+		} );
+		return sentence;
+	} );
+}
+
+/**
+ * Splits any Paragraph and Heading nodes in the tree into sentences and tokens.
+ * Excludes overarching Paragraphs, as those will have (implicit) paragraphs as their children.
+ *
+ * @param {Node} tree The tree to process.
+ * @param {LanguageProcessor} languageProcessor The language processor to use.
+ *
+ * @returns {Node} The processed tree.
+ */
+function tokenize( tree, languageProcessor ) {
+	if ( ( tree instanceof Paragraph && tree.name !== "p-overarching" ) || tree instanceof Heading ) {
+		tree.sentences = getSentences( tree, languageProcessor );
+	}
+
+	if ( tree.childNodes ) {
+		tree.childNodes = tree.childNodes.map( child => tokenize( child, languageProcessor ) );
+	}
+
+	return tree;
+}
+
+export default tokenize;

package/src/parse/language/LanguageProcessor.js

@@ -0,0 +1,74 @@
+import Sentence from "../structure/Sentence";
+import Token from "../structure/Token";
+import splitIntoTokens from "../../languageProcessing/helpers/word/splitIntoTokens";
+
+const whitespaceRegex = /^\s+$/;
+
+
+/**
+ * Contains language-specific logic for splitting a text into sentences and tokens.
+ */
+class LanguageProcessor {
+	/**
+	 * Creates a new language processor.
+	 *
+	 * @param {Researcher} researcher The researcher to use.
+	 */
+	constructor( researcher ) {
+		this.researcher = researcher;
+	}
+
+	/**
+	 * Split text into sentences.
+	 *
+	 * @param {string} text The text to split into sentences.
+	 *
+	 * @returns {Sentence[]} The sentences.
+	 */
+	splitIntoSentences( text ) {
+		const memoizedTokenizer = this.researcher.getHelper( "memoizedTokenizer" );
+		/*
+		 * Set the `trimSentences` flag to false. We want to keep whitespaces to be able to correctly assess the
+		 * position of sentences within the source code.
+		 */
+		const sentences = memoizedTokenizer( text, false );
+
+		/*
+		 * If the last element in the array of sentences contains only whitespaces, remove it.
+		 * This will be the case if the text ends in a whitespace - that whitespace ends up being tokenized as a
+		 * separate sentence. A space at the end of the text is not needed for calculating the position of
+		 * sentences, so it can be safely removed.
+		 */
+		if ( whitespaceRegex.test( sentences[ sentences.length - 1 ] ) ) {
+			sentences.pop();
+		}
+
+		return sentences.map( function( sentence ) {
+			return new Sentence( sentence );
+		} );
+	}
+
+	/**
+	 * Split sentence into tokens.
+	 *
+	 * @param {Sentence} sentence The sentence to split.
+	 *
+	 * @returns {Token[]} The tokens.
+	 */
+	splitIntoTokens( sentence ) {
+		// Retrieve sentence from sentence class
+		const sentenceText = sentence.text;
+
+		// If there is a custom splitIntoTokens helper use its output for retrieving tokens.
+		const tokenTextsCustom = this.researcher.getHelper( "splitIntoTokensCustom" );
+		if ( tokenTextsCustom ) {
+			const tokensCustom = tokenTextsCustom( sentenceText );
+			return tokensCustom.map( tokenText => new Token( tokenText ) );
+		}
+
+		const tokenTexts = splitIntoTokens( sentenceText );
+
+		return tokenTexts.map( tokenText => new Token( tokenText ) );
+	}
+}
+export default LanguageProcessor;

package/src/parse/structure/Heading.js

@@ -0,0 +1,26 @@
+import Node from "./Node";
+
+/**
+ * A heading in the tree.
+ */
+class Heading extends Node {
+	/**
+	 * Creates a new heading.
+	 *
+	 * @param {1|2|3|4|5|6} level The heading level (e.g. `1` for `h1` up to `6` for `h6`).
+	 * @param {Object} attributes This heading's attributes.
+	 * @param {(Node|Text)[]} childNodes This heading's child nodes.
+	 * @param {Object} sourceCodeLocationInfo This heading's location in the source code, from parse5.
+	 */
+	constructor( level, attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
+		super( `h${level}`, attributes, childNodes, sourceCodeLocationInfo );
+		/**
+		 * This heading's level (e.g. `1` for `h1`, `2` for `h2` etc.).
+		 *
+		 * @type {1|2|3|4|5|6}
+		 */
+		this.level = level;
+	}
+}
+
+export default Heading;

package/src/parse/structure/Node.js

@@ -0,0 +1,69 @@
+import { findAllInTree, innerText } from "../traverse";
+import SourceCodeLocation from "./SourceCodeLocation";
+import { isEmpty } from "lodash";
+
+/**
+ * A node in the tree.
+ */
+class Node {
+	/**
+	 * Creates a new node.
+	 *
+	 * @param {string} 			name 					The node's name or tag.
+	 * @param {Object} 			attributes 				This node's attributes.
+	 * @param {(Node|Text)[]} 	childNodes 				This node's child nodes.
+	 * @param {Object} 			sourceCodeLocationInfo 	This node's location in the source code, from parse5.
+	 */
+	constructor( name, attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
+		/**
+		 * This node's name or tag.
+		 * @type {string}
+		 */
+		this.name = name;
+
+		/**
+		 * This node's attributes.
+		 * @type {Object}
+		 */
+		this.attributes = attributes;
+
+		/**
+		 * This node's child nodes.
+		 * @type {(Node|Text)[]}
+		 */
+		this.childNodes = childNodes;
+
+		// Don't add the source code location when unavailable.
+		if ( ! isEmpty( sourceCodeLocationInfo ) ) {
+			/**
+			 * The location of this node inside the HTML.
+			 * @type {SourceCodeLocation}
+			 */
+			this.sourceCodeLocation = new SourceCodeLocation( sourceCodeLocationInfo );
+		}
+	}
+
+	/**
+	 * Finds all nodes in the tree that satisfies the given condition.
+	 *
+	 * @param {function} 	condition 					The condition that a node should satisfy to end up in the list.
+	 * @param {boolean} 	recurseFoundNodes=false 	Whether to recurse into found nodes to see if the condition
+	 *  also applies to sub-nodes of the found node.
+	 *
+	 * @returns {(Node|Text|Paragraph|Heading)[]} The list of nodes that satisfy the condition.
+	 */
+	findAll( condition, recurseFoundNodes = false ) {
+		return findAllInTree( this, condition, recurseFoundNodes );
+	}
+
+	/**
+	 * Returns the inner text (text without any markup) from this node.
+	 *
+	 * @returns {string} The inner text from this node.
+	 */
+	innerText() {
+		return innerText( this );
+	}
+}
+
+export default Node;

package/src/parse/structure/Paragraph.js

@@ -0,0 +1,48 @@
+import Node from "./Node";
+
+/**
+ * A paragraph in the tree.
+ *
+ * @see https://html.spec.whatwg.org/dev/dom.html#paragraphs
+ */
+class Paragraph extends Node {
+	/**
+	 * Creates a new paragraph.
+	 *
+	 * @param {Object} attributes The paragraph's attributes.
+	 * @param {(Node|Text)[]} childNodes This paragraph's child nodes.
+	 * @param {Object} sourceCodeLocationInfo This paragraph's location in the source code, from parse5.
+	 * @param {boolean} isImplicit Whether this paragraph is an implicit paragraph, or an explicit paragraph.
+	 * @param {boolean} isOverarching Whether this paragraph is overarching text that is separated by double line breaks.
+	 */
+	constructor( attributes = {}, childNodes = [], sourceCodeLocationInfo = {},
+		isImplicit = false, isOverarching = false ) {
+		// We rename any overarching `<p>` element to not match it as a stand-alone paragraph downstream.
+		const name = isOverarching ? "p-overarching" : "p";
+		super( name, attributes, childNodes, sourceCodeLocationInfo );
+		/**
+		 * Whether this paragraph is explicit (defined by an explicit `<p>` tag within the markup),
+		 * or implicit (defined by a run of phrasing content).
+		 *
+		 * @see https://html.spec.whatwg.org/dev/dom.html#paragraphs
+		 *
+		 * @type {boolean}
+		 */
+		this.isImplicit = isImplicit;
+	}
+
+	/**
+	 * Creates and returns a new implicit paragraph.
+	 *
+	 * @param {Object} attributes The paragraph's attributes.
+	 * @param {(Node|Text)[]} childNodes This paragraph's child nodes.
+	 * @param {Object} sourceCodeLocationInfo This paragraph's location in the source code, from parse5.
+	 *
+	 * @returns {Paragraph} A new implicit paragraph.
+	 */
+	static createImplicit( attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
+		return new Paragraph( attributes, childNodes, sourceCodeLocationInfo, true );
+	}
+}
+
+export default Paragraph;

package/src/parse/structure/Sentence.js

@@ -0,0 +1,30 @@
+/**
+ * A sentence within a text.
+ */
+class Sentence {
+	/**
+	 * Creates a sentence.
+	 *
+	 * @param {string} text The sentence's text.
+	 */
+	constructor( text ) {
+		/**
+		 * The text in this sentence.
+		 *
+		 * @type {string}
+		 */
+		this.text = text;
+		/**
+		 * The tokens in this sentence.
+		 * @type {Token[]}
+		 */
+		this.tokens = [];
+		/**
+		 * The start and end positions of the sentence in the source code.
+		 * @type {SourceCodeRange}
+		 */
+		this.sourceCodeRange = {};
+	}
+}
+
+export default Sentence;

package/src/parse/structure/SourceCodeLocation.js

@@ -0,0 +1,41 @@
+/**
+ * Represents a range in the source code.
+ *
+ * @typedef {Object} SourceCodeRange
+ * @property {number} startOffset The start position of the range.
+ * @property {number} endOffset The end position of the range.
+ */
+
+/**
+ * Represents a location of a node in the HTML tree.
+ *
+ * @extends SourceCodeRange
+ * @property {SourceCodeRange?} startTag The location of the start tag of the element, if it has one.
+ * @property {SourceCodeRange?} endTag The location of the end tag of the element, if it has one.
+ */
+class SourceCodeLocation {
+	/**
+	 * Creates a new SourceCodeLocation.
+	 *
+	 * @param {Object} sourceCodeLocationInfo This node's location in the source code, from parse5.
+	 */
+	constructor( sourceCodeLocationInfo ) {
+		if ( sourceCodeLocationInfo.startTag ) {
+			this.startTag = {
+				startOffset: sourceCodeLocationInfo.startTag.startOffset,
+				endOffset: sourceCodeLocationInfo.startTag.endOffset,
+			};
+		}
+		if ( sourceCodeLocationInfo.endTag ) {
+			this.endTag = {
+				startOffset: sourceCodeLocationInfo.endTag.startOffset,
+				endOffset: sourceCodeLocationInfo.endTag.endOffset,
+			};
+		}
+
+		this.startOffset = sourceCodeLocationInfo.startOffset;
+		this.endOffset = sourceCodeLocationInfo.endOffset;
+	}
+}
+
+export default SourceCodeLocation;

package/src/parse/structure/Text.js

@@ -0,0 +1,27 @@
+import SourceCodeLocation from "./SourceCodeLocation";
+/**
+ * A text.
+ */
+class Text {
+	/**
+	 * Creates a new Text object, that consist of some text and a source code range.
+	 *
+	 * @param {object} textNode The current #text node in the parse5 tree.
+	 */
+	constructor( textNode ) {
+		this.name = "#text";
+		/**
+		 * This text's content.
+		 *
+		 * @type {string}
+		 */
+		this.value = textNode.value;
+
+		this.sourceCodeRange = new SourceCodeLocation( {
+			startOffset: textNode.sourceCodeLocation.startOffset,
+			endOffset: textNode.sourceCodeLocation.endOffset,
+		} );
+	}
+}
+
+export default Text;

package/src/parse/structure/Token.js

@@ -0,0 +1,24 @@
+import { normalizeSingle } from "../../languageProcessing";
+
+/**
+ * A token representing a word, whitespace or punctuation in the sentence.
+ */
+class Token {
+	/**
+	 * Creates a new token.
+	 *
+	 * @param {string} text The token's text.
+	 * @param {SourceCodeRange} sourceCodeRange The start and end positions of the token in the source code.
+	 */
+	constructor( text, sourceCodeRange = {} ) {
+		// Normalize single quotes so that tokens can be matched with strings regardless of single quote type.
+		this.text = normalizeSingle( text );
+		/**
+		 * The start and end positions of the token in the source code.
+		 * @type {SourceCodeRange}
+		 */
+		this.sourceCodeRange = sourceCodeRange;
+	}
+}
+
+export default Token;

package/src/parse/structure/index.js

@@ -0,0 +1,16 @@
+import Node from "./Node";
+import Heading from "./Heading";
+import Paragraph from "./Paragraph";
+import Text from "./Text";
+
+/**
+ * @module parse/structure
+ *
+ * Contains elements out of which a tree structure can be built.
+ */
+export {
+	Node,
+	Heading,
+	Paragraph,
+	Text,
+};

package/src/parse/traverse/findAllInTree.js

@@ -0,0 +1,58 @@
+/**
+ * Finds all nodes in the tree that satisfies the given condition.
+ *
+ * @param {Object} 		tree 						The tree.
+ * @param {function} 	condition 					The condition that a node should satisfy to end up in the list.
+ * @param {boolean} 	recurseFoundNodes=false 	Whether to recurse into found nodes to see if the condition also applies to
+ * sub-nodes of the found node.
+ * If false, as soon as a node is found that satisfies the condition, it is added to the list and no further recursion is done through its children.
+ * If true, the node is added to the list and its children are also checked for the condition.
+ * If they satisfy the condition, they are also added to the list.
+ * This comes with the risk of adding the same node multiple times to the list.
+ *
+ * Example:
+ * HTML: <div><div><div>foo</div></div></div>
+ * Condition: node => node.nodeName === 'div'
+ * If recurseFoundNodes is false, this will return: [ <div><div><div>foo</div></div></div> ]
+ *
+ * If recurseFoundNodes is true, this will return: [ <div><div><div>foo</div></div></div>, <div><div>foo</div></div>, <div>foo</div> ]
+ *
+ * In assessments where you want to count the number of occurrences of a certain type of node, you should set recurseFoundNodes to false.
+ * For example in the list assessment with a nested list: you only want to count the outer list.
+ *
+ * <ul>
+ *   <li>Coffee</li>
+ *   <li>Tea
+ *     <ul>
+ *       <li>Black tea</li>
+ *       <li>Green tea</li>
+ *     </ul>
+ *   </li>
+ *   <li>Milk</li>
+ * </ul>
+ *
+ * In getTextElementPositions, you want to find all descendant nodes that have position information.
+ * In this case you want to recurse into found nodes.
+ *
+ * @returns {Object[]} The list of nodes that satisfy the condition.
+ */
+export default function findAllInTree( tree, condition, recurseFoundNodes = false ) {
+	const nodes = [];
+
+	if ( ! tree.childNodes ) {
+		return nodes;
+	}
+
+	tree.childNodes.forEach( child => {
+		if ( condition( child ) ) {
+			nodes.push( child );
+			if ( recurseFoundNodes ) {
+				nodes.push( ...findAllInTree( child, condition, recurseFoundNodes ) );
+			}
+		} else {
+			nodes.push( ...findAllInTree( child, condition, recurseFoundNodes ) );
+		}
+	} );
+
+	return nodes;
+}

package/src/parse/traverse/index.js

@@ -0,0 +1,12 @@
+import findAllInTree from "./findAllInTree";
+import innerText from "./innerText";
+
+/**
+ * @module parse/traverse
+ *
+ * Module containing functions to traverse and gather information from the tree.
+ */
+export {
+	findAllInTree,
+	innerText,
+};

package/src/parse/traverse/innerText.js

@@ -0,0 +1,26 @@
+import { isEmpty } from "lodash";
+
+/**
+ * Gathers the text content of the given node.
+ *
+ * @param {Object} node The node to gather the text content from.
+ *
+ * @returns {string} The text content.
+ */
+export default function innerText( node ) {
+	let text = "";
+
+	if ( ! isEmpty( node.childNodes ) ) {
+		node.childNodes.forEach( child => {
+			if ( child.name === "#text" ) {
+				text += child.value;
+			} else if ( child.name === "br" ) {
+				text += "\n";
+			} else {
+				text += innerText( child );
+			}
+		} );
+	}
+
+	return text;
+}

package/src/parsedPaper/ParsedPaper.js

@@ -0,0 +1,92 @@
+/**
+ * A parsed version of the paper, containing a tree representation of the text,
+ * and parsed metaData pertaining to the tree's context.
+ *
+ * @class ParsedPaper
+ *
+ * @memberOf module:parsedPaper
+ * */
+export default class ParsedPaper {
+	/**
+	 * Creates a new ParsedPaper.
+	 */
+	constructor() {
+		/**
+		 * Stores the metadata (e.g. the keywords, metadescription, title of the document).
+		 *
+		 * @type{Object}
+		 * @private
+		 */
+		this._metaData = {};
+
+		/**
+		 * Stores the tree representation of a Paper's text.
+		 *
+		 * @type{module:parsedPaper/structure.Node|Object}
+		 * @private
+		 */
+		this._tree = {};
+	}
+
+	/**
+	 * Sets a tree structure on the ParsedPaper.
+	 *
+	 * @param {module:parsedPaper/structure.Node} tree A tree representation of the paper's text.
+	 *
+	 * @returns {void}
+	 */
+	setTree( tree ) {
+		this._tree = tree;
+	}
+
+	/**
+	 * Returns the tree from the ParsedPaper.
+	 *
+	 * @returns {module:parsedPaper/structure.Node|Object} The tree.
+	 */
+	getTree() {
+		return this._tree;
+	}
+
+	/**
+	 * Sets a metadata value on the ParsedPaper under the passed key.
+	 *
+	 * @param {string} key   The key for the metaData key-value pair.
+	 * @param {*}      value The value for the metaData key-value pair.
+	 *
+	 * @returns {void}
+	 */
+	setMetaProperty( key, value ) {
+		this._metaData[ key ] = value;
+	}
+
+	/**
+	 * Returns the metadata value for a specific key from the ParsedPaper.
+	 *
+	 * @param   {string} key The key to return the value for.
+	 * @returns {*}          The value belonging to this key.
+	 */
+	getMetaProperty( key ) {
+		return this._metaData[ key ];
+	}
+
+	/**
+	 * Sets the metaData on the ParsedPaper.
+	 *
+	 * @param {Object} metaData The metaData pertaining to tree.
+	 *
+	 * @returns {void}
+	 */
+	setMetaData( metaData ) {
+		this._metaData = metaData;
+	}
+
+	/**
+	 * Returns the current metaData of the ParsedPaper.
+	 *
+	 * @returns {Object} The metaData.
+	 */
+	getMetaData() {
+		return this._metaData;
+	}
+}