axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/values/ProminentWord.js

@@ -0,0 +1,95 @@
+/**
+ * Represents a prominent word in the context of relevant words.
+*/
+class ProminentWord {
+	/**
+	 * Constructs Prominent word object.
+	 *
+	 * @constructor
+	 *
+	 * @param {string} word             The word.
+	 * @param {string} [stem]           The stem / base form of the word, defaults to the word.
+	 * @param {number} [occurrences]    The number of occurrences, defaults to 0.
+	 */
+	constructor( word, stem, occurrences ) {
+		this._word = word;
+		this._stem = stem ? stem : word;
+		this._occurrences = occurrences || 0;
+	}
+
+	/**
+	 * Sets the word.
+	 *
+	 * @param {string} word The word to set.
+	 *
+	 * @returns {void}.
+	 */
+	setWord( word ) {
+		this._word = word;
+	}
+
+	/**
+	 * Returns the word.
+	 *
+	 * @returns {string} The word.
+	 */
+	getWord() {
+		return this._word;
+	}
+
+	/**
+	 * Returns the stem of the word.
+	 *
+	 * @returns {string} The stem.
+	 */
+	getStem() {
+		return this._stem;
+	}
+
+	/**
+	 * Sets the number of occurrences to the word.
+	 *
+	 * @param {int} numberOfOccurrences The number of occurrences to set.
+	 *
+	 * @returns {void}.
+	 */
+	setOccurrences( numberOfOccurrences ) {
+		this._occurrences = numberOfOccurrences;
+	}
+
+	/**
+	 * Returns the amount of occurrences of this word.
+	 *
+	 * @returns {number} The number of occurrences.
+	 */
+	getOccurrences() {
+		return this._occurrences;
+	}
+
+	/**
+	 * Serializes the ProminentWord instance to an object.
+	 *
+	 * @returns {Object} The serialized ProminentWord.
+	 */
+	serialize() {
+		return {
+			_parseClass: "ProminentWord",
+			word: this._word,
+			stem: this._stem,
+			occurrences: this._occurrences,
+		};
+	}
+
+	/**
+	 * Parses the object to a ProminentWord.
+	 *
+	 * @param {Object} serialized The serialized object.
+	 *
+	 * @returns {ProminentWord} The parsed ProminentWord.
+	 */
+	static parse( serialized ) {
+		return new ProminentWord( serialized.word, serialized.stem, serialized.occurrences );
+	}
+}
+
+export default ProminentWord;

package/src/languageProcessing/values/Sentence.js

@@ -0,0 +1,111 @@
+/**
+ * Construct the Sentence object and set the sentence text.
+ *
+ * @param {string} sentence The text of the sentence.
+ * @constructor
+ */
+class Sentence {
+	/**
+	 * Constructor.
+	 *
+	 * @param {string} sentence The sentence.
+	 * @constructor
+	 */
+	constructor( sentence ) {
+		this._sentenceText = sentence || "";
+		this._isPassive = false;
+		this._clauses = [];
+	}
+
+	/**
+	 * Returns the sentence text.
+	 *
+	 * @returns {string} The sentence.
+	 */
+	getSentenceText() {
+		return this._sentenceText;
+	}
+
+	/**
+	 * Returns the passiveness of a sentence.
+	 *
+	 * @returns {boolean} True if passive, otherwise returns false.
+	 */
+	isPassive() {
+		return this._isPassive;
+	}
+
+	/**
+	 * Sets the passiveness of the sentence.
+	 *
+	 * @param {boolean} passive Whether the sentence is passive or not.
+	 * @returns {void}
+	 */
+	setPassive( passive ) {
+		this._isPassive = passive;
+	}
+
+	/**
+	 * Returns an array of clauses.
+	 *
+	 * @returns {Clause[]} The clauses of the sentence.
+	 */
+	getClauses() {
+		return this._clauses;
+	}
+
+	/**
+	 * Sets the clauses.
+	 *
+	 * @param {Clause[]} clauses The clauses of the sentence.
+	 *
+	 * @returns {void}
+	 */
+	setClauses( clauses ) {
+		this._clauses = clauses;
+
+		// Directly set the sentence passiveness based on the passiveness of the clauses that are just set.
+		this.setSentencePassiveness();
+	}
+
+	/**
+	 * Sets the passiveness of the sentence. A sentence is passive if it contains at least one passive clause.
+	 *
+	 * @returns {void}
+	 */
+	setSentencePassiveness() {
+		const passiveClauses = this.getClauses().filter( clause => clause.isPassive() === true );
+		this.setPassive( passiveClauses.length > 0 );
+	}
+
+	/**
+	 * Serializes the Sentence instance to an object.
+	 *
+	 * @returns {Object} The serialized Sentence.
+	 */
+	serialize() {
+		return {
+			_parseClass: "Sentence",
+			sentenceText: this._sentenceText,
+			isPassive: this._isPassive,
+			clauses: this._clauses,
+		};
+	}
+
+	/**
+	 * Parses the object to a Sentence.
+	 *
+	 * @param {Object} serialized The serialized object.
+	 *
+	 * @returns {Sentence} The parsed Sentence.
+	 */
+	static parse( serialized ) {
+		const sentence = new Sentence( serialized.sentenceText );
+		sentence.setClauses( serialized.clauses );
+		sentence.setPassive( serialized.isPassive );
+
+		return sentence;
+	}
+}
+
+export default Sentence;

package/src/languageProcessing/values/index.js

@@ -0,0 +1,9 @@
+import Clause from "./Clause";
+import ProminentWord from "./ProminentWord";
+import Sentence from "./Sentence";
+
+export {
+	Clause,
+	ProminentWord,
+	Sentence,
+};

package/src/markers/addMark.js

@@ -0,0 +1,9 @@
+/**
+ * Marks a text with HTML tags
+ *
+ * @param {string} text The unmarked text.
+ * @returns {string} The marked text.
+ */
+export default function( text ) {
+	return "<yoastmark class='yoast-text-mark'>" + text + "</yoastmark>";
+}

package/src/markers/addMarkSingleWord.js

@@ -0,0 +1,32 @@
+import { stripWordBoundariesStart, stripWordBoundariesEnd } from "../languageProcessing/helpers/sanitize/stripWordBoundaries";
+import { escapeRegExp } from "lodash";
+
+/**
+ * Marks a text with HTML tags, deals with word boundaries that were matched by regexes, but which should not be marked.
+ *
+ * @param {string} text The unmarked text.
+ *
+ * @returns {string} The marked text.
+ */
+export default function( text ) {
+	// Strip the word boundaries at the start of the text.
+	const strippedTextStart = stripWordBoundariesStart( text );
+	let wordBoundaryStart = "";
+	let wordBoundaryEnd = "";
+
+	// Get the actual word boundaries from the start of the text.
+	if ( strippedTextStart !== text ) {
+		const wordBoundaryStartIndex = text.search( escapeRegExp( strippedTextStart ) );
+		wordBoundaryStart = text.substring( 0, wordBoundaryStartIndex );
+	}
+
+	// Strip word boundaries at the end of the text.
+	const strippedTextEnd = stripWordBoundariesEnd( strippedTextStart );
+	// Get the actual word boundaries from the end of the text.
+	if ( strippedTextEnd !== strippedTextStart ) {
+		const wordBoundaryEndIndex = strippedTextStart.search( escapeRegExp( strippedTextEnd ) ) + strippedTextEnd.length;
+		wordBoundaryEnd = strippedTextStart.substring( wordBoundaryEndIndex );
+	}
+
+	return wordBoundaryStart + "<yoastmark class='yoast-text-mark'>" + strippedTextEnd + "</yoastmark>" + wordBoundaryEnd;
+}

package/src/markers/index.js

@@ -0,0 +1,7 @@
+import removeMarks from "./removeMarks";
+import addMark from "./addMark";
+
+export {
+	addMark,
+	removeMarks,
+};

package/src/markers/removeDuplicateMarks.js

@@ -0,0 +1,27 @@
+import { uniqBy } from "lodash";
+
+/**
+ * Removes duplicate marks from an array.
+ * If the marks object have position information, however,
+ * we don't want to remove the duplicated objects with the same original strings.
+ *
+ * @param {Array} marks The marks to remove duplications from.
+ *
+ * @returns {Array} A list of de-duplicated marks.
+ */
+function removeDuplicateMarks( marks ) {
+	/*
+	 * We don't remove duplicates when mark has position information, for the reasons below:
+	 * 1. Not removing duplicates is simpler than removing the duplicates by looking at the value of mark.getPosition().
+	 * 2. Our current approach of creating a mark object with position information eliminates the chance of having duplicates.
+	 */
+	if ( !! marks && ( marks.length === 0 || !! marks[ 0 ].hasPosition() ) ) {
+		return marks;
+	}
+
+	return uniqBy( marks, function( mark ) {
+		return mark.getOriginal();
+	} );
+}
+
+export default removeDuplicateMarks;

package/src/markers/removeMarks.js

@@ -0,0 +1,11 @@
+/**
+ * Removes all marks from a text
+ *
+ * @param {string} text The marked text.
+ * @returns {string} The unmarked text.
+ */
+export default function( text ) {
+	return text
+		.replace( new RegExp( "<yoastmark[^>]*>", "g" ), "" )
+		.replace( new RegExp( "</yoastmark>", "g" ), "" );
+}

package/src/parse/build/build.js

@@ -0,0 +1,52 @@
+// External dependencies.
+import { parseFragment } from "parse5";
+// Internal dependencies.
+import adapt from "./private/adapt";
+import tokenize from "./private/tokenize";
+import filterTree from "./private/filterTree";
+import permanentFilters from "./private/alwaysFilterElements";
+import { filterBeforeTokenizing } from "./private/filterBeforeTokenizing";
+import parseBlocks from "./private/parseBlocks";
+import filterShortcodesFromTree from "../../languageProcessing/helpers/sanitize/filterShortcodesFromTree";
+import { htmlEntitiesRegex } from "../../helpers/htmlEntities";
+
+/**
+ * Parses the HTML string to a tree representation of the HTML document.
+ *
+ * @param {Paper} paper The paper to build the tree from.
+ * @param {LanguageProcessor} languageProcessor The language processor to use.
+ * @param {string[]} [shortcodes] An optional array of all active shortcodes.
+ *
+ * @returns {Node} The tree representation of the HTML string.
+ */
+export default function build( paper, languageProcessor, shortcodes ) {
+	let html = paper.getText();
+	// Change HTML entities like "&" to "#amp;" to prevent early conversion to "&" -- which would invalidate token positions.
+	html = html.replace( htmlEntitiesRegex, "#$1" );
+
+	let tree = adapt( parseFragment( html, { sourceCodeLocationInfo: true } ) );
+	if ( tree.childNodes && tree.childNodes.length > 0 ) {
+		parseBlocks( paper, tree );
+	}
+
+	/*
+	 * Filter out some content from the tree so that it can be correctly tokenized. We don't want to tokenize text in
+	 * between tags such as 'code' and 'script', but we do want to take into account the length of those elements when
+	 * calculating sentence and token positions.
+	 */
+	tree = filterBeforeTokenizing( tree );
+
+	// Add sentences and tokens to the tree's paragraph and heading nodes.
+	tree = tokenize( tree, languageProcessor );
+
+	// Filter out shortcodes from the tree.
+	if ( shortcodes ) {
+		filterShortcodesFromTree( tree, shortcodes );
+	}
+
+	/*
+	 * Filter out elements we don't want to include in the analysis. Only do this after tokenization as we need to
+	 * have all inline elements in the tree during tokenization to correctly calculate sentence and token positions.
+	 */
+	return filterTree( tree, permanentFilters );
+}

package/src/parse/build/index.js

@@ -0,0 +1,10 @@
+import build from "./build";
+
+/**
+ * @module parse/build
+ *
+ * Module containing functions to build a tree for the analysis.
+ */
+export {
+	build,
+};

package/src/parse/build/private/adapt.js

@@ -0,0 +1,113 @@
+import combineIntoImplicitParagraphs from "./combineIntoImplicitParagraphs";
+import adaptAttributes from "./adaptAttributes";
+import isPhrasingContent from "./isPhrasingContent";
+import { isEmpty } from "lodash";
+import { Paragraph, Text, Heading, Node } from "../../structure";
+
+/**
+ * Whether the element with the specified name is a paragraph.
+ *
+ * @param {string} nodeName The name/tag of the node.
+ *
+ * @returns {boolean} Whether the element is considered a paragraph.
+ */
+function isParagraph( nodeName ) {
+	return nodeName === "p";
+}
+
+/**
+ * Whether the element with the specified name is a heading.
+ *
+ * @param {string} nodeName The name/tag of the node.
+ *
+ * @returns {boolean} Whether the element is considered a heading.
+ */
+function isHeading( nodeName ) {
+	return [ "h1", "h2", "h3", "h4", "h5", "h6" ].includes( nodeName );
+}
+
+/**
+ * Whether the element is text.
+ *
+ * @param {string} nodeName The name/tag of the node.
+ *
+ * @returns {boolean} Whether the element is considered text.
+ */
+function isText( nodeName ) {
+	return nodeName === "#text";
+}
+
+/**
+ * Whether the element with the specified name is a block level element.
+ *
+ * @param {string} nodeName The name/tag of the node.
+ *
+ * @returns {boolean} Whether the element is considered a block level element.
+ */
+function isBlockElement( nodeName ) {
+	return ! (
+		isParagraph( nodeName ) ||
+		isPhrasingContent( nodeName ) ||
+		isHeading( nodeName )
+	);
+}
+
+/**
+ * Checks whether the current node is an overarching paragraph.
+ * Overarching paragraphs have double `<br>` nodes (line breaks) in their children.
+ * We consider those to be indicating the end and start of an implicit paragraph, similar to the `autop` function in WordPress.
+ *
+ * @param {string} nodeName The name of the current node.
+ * @param {Node[]} children The children of the current nodes.
+ *
+ * @returns {boolean} Whether the current node is an overarching paragraph.
+ */
+function isOverarchingParagraph( nodeName, children ) {
+	return isParagraph( nodeName ) && children.some( ( node, index, childNodes ) => {
+		const nextNode = childNodes.length - 1 !== index && childNodes[ index + 1 ];
+		return node.name === "br" && nextNode && nextNode.name === "br";
+	} );
+}
+
+/**
+ * Adapts the `parse5` tree to our own tree representation.
+ *
+ * By adapting the external `parse5` structure to our own tree representation
+ * we reduce the coupling between our code and theirs, which makes our code
+ * more robust against changes in the `parse5` library. [See also this blog post about coupling](https://mrpicky.dev/six-shades-of-coupling/)
+ *
+ * @param {Object} tree The parse5 tree representation.
+ *
+ * @returns {Node} The adapted tree.
+ */
+export default function adapt( tree ) {
+	if ( isText( tree.nodeName ) ) {
+		return new Text( tree );
+	}
+
+	let children = [];
+	let isOverarching = false;
+	if ( ! isEmpty( tree.childNodes ) ) {
+		children = tree.childNodes.map( adapt );
+		if ( isBlockElement( tree.nodeName ) ) {
+			children = combineIntoImplicitParagraphs( children, tree.sourceCodeLocation );
+		}
+		if ( isOverarchingParagraph( tree.nodeName, children ) ) {
+			isOverarching = true;
+			children = combineIntoImplicitParagraphs( children, tree.sourceCodeLocation );
+		}
+	}
+
+	const attributes = adaptAttributes( tree.attrs );
+
+	if ( isParagraph( tree.nodeName ) ) {
+		return new Paragraph( attributes, children, tree.sourceCodeLocation, false, isOverarching );
+	}
+
+	if ( isHeading( tree.nodeName ) ) {
+		const headingLevel = parseInt( tree.nodeName[ 1 ], 10 );
+		return new Heading( headingLevel, attributes, children, tree.sourceCodeLocation );
+	}
+
+	return new Node( tree.nodeName, attributes, children, tree.sourceCodeLocation );
+}

package/src/parse/build/private/adaptAttributes.js

@@ -0,0 +1,36 @@
+import parseClassAttribute from "./helpers/parseClassAttribute";
+
+/**
+ * @typedef Parse5Attribute
+ * @property {string} name The name of the attribute.
+ * @property {string} value The value of the attribute.
+ */
+
+/**
+ * Adapts name-value attribute pairs as output by `parse5`
+ * and transforms it into an easier to use object mapping attribute names
+ * to values.
+ *
+ * @param {Parse5Attribute[]|null} parse5attributes The attributes as name-value pairs.
+ *
+ * @returns {Object} The attributes as an object mapping attribute name to value.
+ */
+function adaptAttributes( parse5attributes ) {
+	if ( ! parse5attributes ) {
+		return {};
+	}
+
+	const attributes = {};
+
+	parse5attributes.forEach( ( { name, value } ) => {
+		if ( name === "class" ) {
+			value = parseClassAttribute( value );
+		}
+		attributes[ name ] = value;
+	} );
+
+
+	return attributes;
+}
+
+export default adaptAttributes;

package/src/parse/build/private/alwaysFilterElements.js

@@ -0,0 +1,75 @@
+/**
+ * A config file that contains filters that should always apply.
+ */
+
+import { elementHasName, elementHasClass, elementHasID } from "./filterHelpers";
+
+// These are elements that we don't want to include in the analysis and that can be child nodes of paragraphs or headings.
+export const canBeChildOfParagraph = [ "code", "kbd", "math", "q", "samp", "script", "var", "#comment", "cite", "form",
+	"map", "noscript", "output" ];
+
+const permanentFilters = [
+	// Filters out Yoast blocks that don't need to be part of the analysis.
+	// The only Yoast blocks that are not filtered out are the FAQ and the How-to block.
+	// The other Yoast blocks we want to filter (i.e. Yoast siblings, Yoast subpages, Yoast breadcrumbs) enter the Paper as HTML comments.
+	// Comments are filtered out in `filterBeforeTokenizing.js` step.
+	elementHasClass( "yoast-table-of-contents" ),
+	elementHasClass( "yoast-reading-time__wrapper" ),
+	// Filters for Elementor widgets
+	elementHasID( "breadcrumbs" ),
+	elementHasClass( "elementor-button-wrapper" ),
+	elementHasClass( "elementor-divider" ),
+	elementHasClass( "elementor-spacer" ),
+	elementHasClass( "elementor-custom-embed" ),
+	elementHasClass( "elementor-icon-wrapper" ),
+	elementHasClass( "elementor-icon-box-wrapper" ),
+	elementHasClass( "elementor-counter" ),
+	elementHasClass( "elementor-progress-wrapper" ),
+	// This element is used for the progress bar widget title.
+	elementHasClass( "elementor-title" ),
+	elementHasClass( "elementor-alert" ),
+	elementHasClass( "elementor-soundcloud-wrapper" ),
+	elementHasClass( "elementor-shortcode" ),
+	elementHasClass( "elementor-menu-anchor" ),
+	elementHasClass( "e-rating" ),
+	// Filters out HTML elements.
+	/* Elements are filtered out when: they contain content outside of the author's control (incl. quotes and embedded
+	content); their content isn't natural language (e.g. code); they contain metadata hidden from the page visitor
+	(e.g. <style>); they are used to accept input from the visitor. Deprecated HTML elements are not included.*/
+	elementHasName( "base" ),
+	elementHasName( "blockquote" ),
+	elementHasName( "canvas" ),
+	elementHasName( "code" ),
+	// It seems that the <head> element is filtered out by the parser we employ, but it's included here for completeness.
+	elementHasName( "head" ),
+	elementHasName( "iframe" ),
+	elementHasName( "input" ),
+	elementHasName( "kbd" ),
+	elementHasName( "link" ),
+	elementHasName( "math" ),
+	elementHasName( "meta" ),
+	elementHasName( "meter" ),
+	elementHasName( "noscript" ),
+	elementHasName( "object" ),
+	elementHasName( "portal" ),
+	elementHasName( "pre" ),
+	elementHasName( "progress" ),
+	elementHasName( "q" ),
+	elementHasName( "samp" ),
+	elementHasName( "script" ),
+	elementHasName( "slot" ),
+	elementHasName( "style" ),
+	elementHasName( "svg" ),
+	elementHasName( "template" ),
+	elementHasName( "textarea" ),
+	elementHasName( "title" ),
+	elementHasName( "var" ),
+	elementHasName( "#comment" ),
+	elementHasName( "cite" ),
+	elementHasName( "form" ),
+	elementHasName( "map" ),
+	elementHasName( "noscript" ),
+	elementHasName( "output" ),
+];
+
+export default permanentFilters;