axyseo 2.0.0-alpha.0.0.2 → 2.0.0-alpha.0.0.21

package/src/parse/build/private/tokenize.js

@@ -1,74 +0,0 @@
-import { Heading, Paragraph } from "../../structure";
-import getTextElementPositions from "./getTextElementPositions";
-import { hashedHtmlEntities } from "../../../helpers/htmlEntities";
-
-/**
- * Splits the sentence into tokens, determines their positions in the source code, and puts them on the sentence.
- *
- * @param {Paragraph|Heading} node The paragraph or heading node to split into sentences.
- * @param {Sentence} sentence The sentence.
- * @param {LanguageProcessor} languageProcessor The language processor for the current language.
- *
- * @returns {Sentence} The sentence, with tokens.
- */
-function getTokens( node, sentence, languageProcessor ) {
-	sentence.tokens = languageProcessor.splitIntoTokens( sentence );
-	sentence.tokens = getTextElementPositions( node, sentence.tokens, sentence.sourceCodeRange.startOffset );
-	return sentence;
-}
-
-/**
- * Splits the node's inner text into sentences, and the sentences into tokens,
- * using the language processor.
- *
- * @param {Paragraph|Heading} node 				The paragraph or heading node to split into sentences.
- * @param {LanguageProcessor} languageProcessor The language processor to use.
- *
- * @returns {Sentence[]} The node's sentences.
- */
-function getSentences( node, languageProcessor ) {
-	// Split text into sentences.
-	let sentences = languageProcessor.splitIntoSentences( node.innerText() );
-	// Add position information to the sentences.
-	sentences = getTextElementPositions( node, sentences );
-	// Tokenize sentences into tokens.
-	return sentences.map( sentence => {
-		sentence = getTokens( node, sentence, languageProcessor );
-		// Now positions have been determined, change HTML entities that had earlier been converted to hashed versions back to their short version.
-		// For example, "&" was earlier converted into "#amp;" and is now converted into "&".
-		// We make this change in both the Sentence and the accompanying Tokens.
-		hashedHtmlEntities.forEach( ( character, hashedHtmlEntity ) => {
-			// We use a global regex instead of replaceAll to support older browsers.
-			const hashedHtmlEntityRegex = new RegExp( hashedHtmlEntity, "g" );
-			sentence.text = sentence.text.replace( hashedHtmlEntityRegex, character );
-			sentence.tokens.map( token => {
-				token.text = token.text.replace( hashedHtmlEntityRegex, character );
-				return token;
-			} );
-		} );
-		return sentence;
-	} );
-}
-
-/**
- * Splits any Paragraph and Heading nodes in the tree into sentences and tokens.
- * Excludes overarching Paragraphs, as those will have (implicit) paragraphs as their children.
- *
- * @param {Node} tree The tree to process.
- * @param {LanguageProcessor} languageProcessor The language processor to use.
- *
- * @returns {Node} The processed tree.
- */
-function tokenize( tree, languageProcessor ) {
-	if ( ( tree instanceof Paragraph && tree.name !== "p-overarching" ) || tree instanceof Heading ) {
-		tree.sentences = getSentences( tree, languageProcessor );
-	}
-
-	if ( tree.childNodes ) {
-		tree.childNodes = tree.childNodes.map( child => tokenize( child, languageProcessor ) );
-	}
-
-	return tree;
-}
-
-export default tokenize;

package/src/parse/language/LanguageProcessor.js

@@ -1,74 +0,0 @@
-import Sentence from "../structure/Sentence";
-import Token from "../structure/Token";
-import splitIntoTokens from "../../languageProcessing/helpers/word/splitIntoTokens";
-
-const whitespaceRegex = /^\s+$/;
-
-
-/**
- * Contains language-specific logic for splitting a text into sentences and tokens.
- */
-class LanguageProcessor {
-	/**
-	 * Creates a new language processor.
-	 *
-	 * @param {Researcher} researcher The researcher to use.
-	 */
-	constructor( researcher ) {
-		this.researcher = researcher;
-	}
-
-	/**
-	 * Split text into sentences.
-	 *
-	 * @param {string} text The text to split into sentences.
-	 *
-	 * @returns {Sentence[]} The sentences.
-	 */
-	splitIntoSentences( text ) {
-		const memoizedTokenizer = this.researcher.getHelper( "memoizedTokenizer" );
-		/*
-		 * Set the `trimSentences` flag to false. We want to keep whitespaces to be able to correctly assess the
-		 * position of sentences within the source code.
-		 */
-		const sentences = memoizedTokenizer( text, false );
-
-		/*
-		 * If the last element in the array of sentences contains only whitespaces, remove it.
-		 * This will be the case if the text ends in a whitespace - that whitespace ends up being tokenized as a
-		 * separate sentence. A space at the end of the text is not needed for calculating the position of
-		 * sentences, so it can be safely removed.
-		 */
-		if ( whitespaceRegex.test( sentences[ sentences.length - 1 ] ) ) {
-			sentences.pop();
-		}
-
-		return sentences.map( function( sentence ) {
-			return new Sentence( sentence );
-		} );
-	}
-
-	/**
-	 * Split sentence into tokens.
-	 *
-	 * @param {Sentence} sentence The sentence to split.
-	 *
-	 * @returns {Token[]} The tokens.
-	 */
-	splitIntoTokens( sentence ) {
-		// Retrieve sentence from sentence class
-		const sentenceText = sentence.text;
-
-		// If there is a custom splitIntoTokens helper use its output for retrieving tokens.
-		const tokenTextsCustom = this.researcher.getHelper( "splitIntoTokensCustom" );
-		if ( tokenTextsCustom ) {
-			const tokensCustom = tokenTextsCustom( sentenceText );
-			return tokensCustom.map( tokenText => new Token( tokenText ) );
-		}
-
-		const tokenTexts = splitIntoTokens( sentenceText );
-
-		return tokenTexts.map( tokenText => new Token( tokenText ) );
-	}
-}
-export default LanguageProcessor;

package/src/parse/structure/Heading.js

@@ -1,26 +0,0 @@
-import Node from "./Node";
-
-/**
- * A heading in the tree.
- */
-class Heading extends Node {
-	/**
-	 * Creates a new heading.
-	 *
-	 * @param {1|2|3|4|5|6} level The heading level (e.g. `1` for `h1` up to `6` for `h6`).
-	 * @param {Object} attributes This heading's attributes.
-	 * @param {(Node|Text)[]} childNodes This heading's child nodes.
-	 * @param {Object} sourceCodeLocationInfo This heading's location in the source code, from parse5.
-	 */
-	constructor( level, attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
-		super( `h${level}`, attributes, childNodes, sourceCodeLocationInfo );
-		/**
-		 * This heading's level (e.g. `1` for `h1`, `2` for `h2` etc.).
-		 *
-		 * @type {1|2|3|4|5|6}
-		 */
-		this.level = level;
-	}
-}
-
-export default Heading;

package/src/parse/structure/Node.js

@@ -1,69 +0,0 @@
-import { findAllInTree, innerText } from "../traverse";
-import SourceCodeLocation from "./SourceCodeLocation";
-import { isEmpty } from "lodash";
-
-/**
- * A node in the tree.
- */
-class Node {
-	/**
-	 * Creates a new node.
-	 *
-	 * @param {string} 			name 					The node's name or tag.
-	 * @param {Object} 			attributes 				This node's attributes.
-	 * @param {(Node|Text)[]} 	childNodes 				This node's child nodes.
-	 * @param {Object} 			sourceCodeLocationInfo 	This node's location in the source code, from parse5.
-	 */
-	constructor( name, attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
-		/**
-		 * This node's name or tag.
-		 * @type {string}
-		 */
-		this.name = name;
-
-		/**
-		 * This node's attributes.
-		 * @type {Object}
-		 */
-		this.attributes = attributes;
-
-		/**
-		 * This node's child nodes.
-		 * @type {(Node|Text)[]}
-		 */
-		this.childNodes = childNodes;
-
-		// Don't add the source code location when unavailable.
-		if ( ! isEmpty( sourceCodeLocationInfo ) ) {
-			/**
-			 * The location of this node inside the HTML.
-			 * @type {SourceCodeLocation}
-			 */
-			this.sourceCodeLocation = new SourceCodeLocation( sourceCodeLocationInfo );
-		}
-	}
-
-	/**
-	 * Finds all nodes in the tree that satisfies the given condition.
-	 *
-	 * @param {function} 	condition 					The condition that a node should satisfy to end up in the list.
-	 * @param {boolean} 	recurseFoundNodes=false 	Whether to recurse into found nodes to see if the condition
-	 *  also applies to sub-nodes of the found node.
-	 *
-	 * @returns {(Node|Text|Paragraph|Heading)[]} The list of nodes that satisfy the condition.
-	 */
-	findAll( condition, recurseFoundNodes = false ) {
-		return findAllInTree( this, condition, recurseFoundNodes );
-	}
-
-	/**
-	 * Returns the inner text (text without any markup) from this node.
-	 *
-	 * @returns {string} The inner text from this node.
-	 */
-	innerText() {
-		return innerText( this );
-	}
-}
-
-export default Node;

package/src/parse/structure/Paragraph.js

@@ -1,48 +0,0 @@
-import Node from "./Node";
-
-/**
- * A paragraph in the tree.
- *
- * @see https://html.spec.whatwg.org/dev/dom.html#paragraphs
- */
-class Paragraph extends Node {
-	/**
-	 * Creates a new paragraph.
-	 *
-	 * @param {Object} attributes The paragraph's attributes.
-	 * @param {(Node|Text)[]} childNodes This paragraph's child nodes.
-	 * @param {Object} sourceCodeLocationInfo This paragraph's location in the source code, from parse5.
-	 * @param {boolean} isImplicit Whether this paragraph is an implicit paragraph, or an explicit paragraph.
-	 * @param {boolean} isOverarching Whether this paragraph is overarching text that is separated by double line breaks.
-	 */
-	constructor( attributes = {}, childNodes = [], sourceCodeLocationInfo = {},
-		isImplicit = false, isOverarching = false ) {
-		// We rename any overarching `<p>` element to not match it as a stand-alone paragraph downstream.
-		const name = isOverarching ? "p-overarching" : "p";
-		super( name, attributes, childNodes, sourceCodeLocationInfo );
-		/**
-		 * Whether this paragraph is explicit (defined by an explicit `<p>` tag within the markup),
-		 * or implicit (defined by a run of phrasing content).
-		 *
-		 * @see https://html.spec.whatwg.org/dev/dom.html#paragraphs
-		 *
-		 * @type {boolean}
-		 */
-		this.isImplicit = isImplicit;
-	}
-
-	/**
-	 * Creates and returns a new implicit paragraph.
-	 *
-	 * @param {Object} attributes The paragraph's attributes.
-	 * @param {(Node|Text)[]} childNodes This paragraph's child nodes.
-	 * @param {Object} sourceCodeLocationInfo This paragraph's location in the source code, from parse5.
-	 *
-	 * @returns {Paragraph} A new implicit paragraph.
-	 */
-	static createImplicit( attributes = {}, childNodes = [], sourceCodeLocationInfo = {} ) {
-		return new Paragraph( attributes, childNodes, sourceCodeLocationInfo, true );
-	}
-}
-
-export default Paragraph;

package/src/parse/structure/Sentence.js

@@ -1,30 +0,0 @@
-/**
- * A sentence within a text.
- */
-class Sentence {
-	/**
-	 * Creates a sentence.
-	 *
-	 * @param {string} text The sentence's text.
-	 */
-	constructor( text ) {
-		/**
-		 * The text in this sentence.
-		 *
-		 * @type {string}
-		 */
-		this.text = text;
-		/**
-		 * The tokens in this sentence.
-		 * @type {Token[]}
-		 */
-		this.tokens = [];
-		/**
-		 * The start and end positions of the sentence in the source code.
-		 * @type {SourceCodeRange}
-		 */
-		this.sourceCodeRange = {};
-	}
-}
-
-export default Sentence;

package/src/parse/structure/SourceCodeLocation.js

@@ -1,41 +0,0 @@
-/**
- * Represents a range in the source code.
- *
- * @typedef {Object} SourceCodeRange
- * @property {number} startOffset The start position of the range.
- * @property {number} endOffset The end position of the range.
- */
-
-/**
- * Represents a location of a node in the HTML tree.
- *
- * @extends SourceCodeRange
- * @property {SourceCodeRange?} startTag The location of the start tag of the element, if it has one.
- * @property {SourceCodeRange?} endTag The location of the end tag of the element, if it has one.
- */
-class SourceCodeLocation {
-	/**
-	 * Creates a new SourceCodeLocation.
-	 *
-	 * @param {Object} sourceCodeLocationInfo This node's location in the source code, from parse5.
-	 */
-	constructor( sourceCodeLocationInfo ) {
-		if ( sourceCodeLocationInfo.startTag ) {
-			this.startTag = {
-				startOffset: sourceCodeLocationInfo.startTag.startOffset,
-				endOffset: sourceCodeLocationInfo.startTag.endOffset,
-			};
-		}
-		if ( sourceCodeLocationInfo.endTag ) {
-			this.endTag = {
-				startOffset: sourceCodeLocationInfo.endTag.startOffset,
-				endOffset: sourceCodeLocationInfo.endTag.endOffset,
-			};
-		}
-
-		this.startOffset = sourceCodeLocationInfo.startOffset;
-		this.endOffset = sourceCodeLocationInfo.endOffset;
-	}
-}
-
-export default SourceCodeLocation;

package/src/parse/structure/Text.js

@@ -1,27 +0,0 @@
-import SourceCodeLocation from "./SourceCodeLocation";
-/**
- * A text.
- */
-class Text {
-	/**
-	 * Creates a new Text object, that consist of some text and a source code range.
-	 *
-	 * @param {object} textNode The current #text node in the parse5 tree.
-	 */
-	constructor( textNode ) {
-		this.name = "#text";
-		/**
-		 * This text's content.
-		 *
-		 * @type {string}
-		 */
-		this.value = textNode.value;
-
-		this.sourceCodeRange = new SourceCodeLocation( {
-			startOffset: textNode.sourceCodeLocation.startOffset,
-			endOffset: textNode.sourceCodeLocation.endOffset,
-		} );
-	}
-}
-
-export default Text;

package/src/parse/structure/Token.js

@@ -1,24 +0,0 @@
-import { normalizeSingle } from "../../languageProcessing";
-
-/**
- * A token representing a word, whitespace or punctuation in the sentence.
- */
-class Token {
-	/**
-	 * Creates a new token.
-	 *
-	 * @param {string} text The token's text.
-	 * @param {SourceCodeRange} sourceCodeRange The start and end positions of the token in the source code.
-	 */
-	constructor( text, sourceCodeRange = {} ) {
-		// Normalize single quotes so that tokens can be matched with strings regardless of single quote type.
-		this.text = normalizeSingle( text );
-		/**
-		 * The start and end positions of the token in the source code.
-		 * @type {SourceCodeRange}
-		 */
-		this.sourceCodeRange = sourceCodeRange;
-	}
-}
-
-export default Token;

package/src/parse/structure/index.js

@@ -1,16 +0,0 @@
-import Node from "./Node";
-import Heading from "./Heading";
-import Paragraph from "./Paragraph";
-import Text from "./Text";
-
-/**
- * @module parse/structure
- *
- * Contains elements out of which a tree structure can be built.
- */
-export {
-	Node,
-	Heading,
-	Paragraph,
-	Text,
-};

package/src/parse/traverse/findAllInTree.js

@@ -1,58 +0,0 @@
-/**
- * Finds all nodes in the tree that satisfies the given condition.
- *
- * @param {Object} 		tree 						The tree.
- * @param {function} 	condition 					The condition that a node should satisfy to end up in the list.
- * @param {boolean} 	recurseFoundNodes=false 	Whether to recurse into found nodes to see if the condition also applies to
- * sub-nodes of the found node.
- * If false, as soon as a node is found that satisfies the condition, it is added to the list and no further recursion is done through its children.
- * If true, the node is added to the list and its children are also checked for the condition.
- * If they satisfy the condition, they are also added to the list.
- * This comes with the risk of adding the same node multiple times to the list.
- *
- * Example:
- * HTML: <div><div><div>foo</div></div></div>
- * Condition: node => node.nodeName === 'div'
- * If recurseFoundNodes is false, this will return: [ <div><div><div>foo</div></div></div> ]
- *
- * If recurseFoundNodes is true, this will return: [ <div><div><div>foo</div></div></div>, <div><div>foo</div></div>, <div>foo</div> ]
- *
- * In assessments where you want to count the number of occurrences of a certain type of node, you should set recurseFoundNodes to false.
- * For example in the list assessment with a nested list: you only want to count the outer list.
- *
- * <ul>
- *   <li>Coffee</li>
- *   <li>Tea
- *     <ul>
- *       <li>Black tea</li>
- *       <li>Green tea</li>
- *     </ul>
- *   </li>
- *   <li>Milk</li>
- * </ul>
- *
- * In getTextElementPositions, you want to find all descendant nodes that have position information.
- * In this case you want to recurse into found nodes.
- *
- * @returns {Object[]} The list of nodes that satisfy the condition.
- */
-export default function findAllInTree( tree, condition, recurseFoundNodes = false ) {
-	const nodes = [];
-
-	if ( ! tree.childNodes ) {
-		return nodes;
-	}
-
-	tree.childNodes.forEach( child => {
-		if ( condition( child ) ) {
-			nodes.push( child );
-			if ( recurseFoundNodes ) {
-				nodes.push( ...findAllInTree( child, condition, recurseFoundNodes ) );
-			}
-		} else {
-			nodes.push( ...findAllInTree( child, condition, recurseFoundNodes ) );
-		}
-	} );
-
-	return nodes;
-}

package/src/parse/traverse/index.js

@@ -1,12 +0,0 @@
-import findAllInTree from "./findAllInTree";
-import innerText from "./innerText";
-
-/**
- * @module parse/traverse
- *
- * Module containing functions to traverse and gather information from the tree.
- */
-export {
-	findAllInTree,
-	innerText,
-};

package/src/parse/traverse/innerText.js

@@ -1,26 +0,0 @@
-import { isEmpty } from "lodash";
-
-/**
- * Gathers the text content of the given node.
- *
- * @param {Object} node The node to gather the text content from.
- *
- * @returns {string} The text content.
- */
-export default function innerText( node ) {
-	let text = "";
-
-	if ( ! isEmpty( node.childNodes ) ) {
-		node.childNodes.forEach( child => {
-			if ( child.name === "#text" ) {
-				text += child.value;
-			} else if ( child.name === "br" ) {
-				text += "\n";
-			} else {
-				text += innerText( child );
-			}
-		} );
-	}
-
-	return text;
-}

package/src/parsedPaper/ParsedPaper.js

@@ -1,92 +0,0 @@
-/**
- * A parsed version of the paper, containing a tree representation of the text,
- * and parsed metaData pertaining to the tree's context.
- *
- * @class ParsedPaper
- *
- * @memberOf module:parsedPaper
- * */
-export default class ParsedPaper {
-	/**
-	 * Creates a new ParsedPaper.
-	 */
-	constructor() {
-		/**
-		 * Stores the metadata (e.g. the keywords, metadescription, title of the document).
-		 *
-		 * @type{Object}
-		 * @private
-		 */
-		this._metaData = {};
-
-		/**
-		 * Stores the tree representation of a Paper's text.
-		 *
-		 * @type{module:parsedPaper/structure.Node|Object}
-		 * @private
-		 */
-		this._tree = {};
-	}
-
-	/**
-	 * Sets a tree structure on the ParsedPaper.
-	 *
-	 * @param {module:parsedPaper/structure.Node} tree A tree representation of the paper's text.
-	 *
-	 * @returns {void}
-	 */
-	setTree( tree ) {
-		this._tree = tree;
-	}
-
-	/**
-	 * Returns the tree from the ParsedPaper.
-	 *
-	 * @returns {module:parsedPaper/structure.Node|Object} The tree.
-	 */
-	getTree() {
-		return this._tree;
-	}
-
-	/**
-	 * Sets a metadata value on the ParsedPaper under the passed key.
-	 *
-	 * @param {string} key   The key for the metaData key-value pair.
-	 * @param {*}      value The value for the metaData key-value pair.
-	 *
-	 * @returns {void}
-	 */
-	setMetaProperty( key, value ) {
-		this._metaData[ key ] = value;
-	}
-
-	/**
-	 * Returns the metadata value for a specific key from the ParsedPaper.
-	 *
-	 * @param   {string} key The key to return the value for.
-	 * @returns {*}          The value belonging to this key.
-	 */
-	getMetaProperty( key ) {
-		return this._metaData[ key ];
-	}
-
-	/**
-	 * Sets the metaData on the ParsedPaper.
-	 *
-	 * @param {Object} metaData The metaData pertaining to tree.
-	 *
-	 * @returns {void}
-	 */
-	setMetaData( metaData ) {
-		this._metaData = metaData;
-	}
-
-	/**
-	 * Returns the current metaData of the ParsedPaper.
-	 *
-	 * @returns {Object} The metaData.
-	 */
-	getMetaData() {
-		return this._metaData;
-	}
-}