axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/AbstractResearcher.js

@@ -0,0 +1,366 @@
+import {merge, isUndefined, isEmpty} from 'lodash';
+
+import InvalidTypeError from '../errors/invalidType';
+import MissingArgument from '../errors/missingArgument';
+
+// All researches in alphabetical order.
+import altTagCount from './researches/altTagCount.js';
+import countSentencesFromText from './researches/countSentencesFromText.js';
+import findKeywordInFirstParagraph from './researches/findKeywordInFirstParagraph.js';
+import findKeyphraseInSEOTitle from './researches/findKeyphraseInSEOTitle';
+import findTransitionWords from './researches/findTransitionWords';
+import functionWordsInKeyphrase from './researches/functionWordsInKeyphrase';
+import getAnchorsWithKeyphrase from './researches/getAnchorsWithKeyphrase';
+import getFleschReadingScore from './researches/getFleschReadingScore';
+import getKeyphraseDensity, {getKeywordDensity} from './researches/getKeywordDensity.js';
+import getLinks from './researches/getLinks.js';
+import getLinkStatistics from './researches/getLinkStatistics';
+import getParagraphs from './researches/getParagraphs';
+import getParagraphLength from './researches/getParagraphLength.js';
+import getPassiveVoiceResult from './researches/getPassiveVoiceResult';
+import getProminentWordsForInsights from './researches/getProminentWordsForInsights';
+import getProminentWordsForInternalLinking from './researches/getProminentWordsForInternalLinking';
+import getSentenceBeginnings from './researches/getSentenceBeginnings';
+import getSubheadingTextLengths from './researches/getSubheadingTextLengths.js';
+import h1s from './researches/h1s';
+import imageCount from './researches/imageCount.js';
+import keyphraseLength from './researches/keyphraseLength';
+import getKeyphraseCount, {keywordCount} from './researches/keywordCount';
+import {keywordCountInSlug, keywordCountInUrl} from './researches/keywordCountInUrl';
+import matchKeywordInSubheadings from './researches/matchKeywordInSubheadings';
+import metaDescriptionKeyword from './researches/metaDescriptionKeyword';
+import metaDescriptionLength from './researches/metaDescriptionLength.js';
+import morphology from './researches/getWordForms';
+import pageTitleWidth from './researches/pageTitleWidth.js';
+import readingTime from './researches/readingTime';
+import sentences from './researches/sentences';
+import videoCount from './researches/videoCount';
+import wordCountInText from './researches/wordCountInText.js';
+
+// All helpers.
+import memoizedTokenizer from './helpers/sentence/memoizedSentenceTokenizer';
+
+/**
+ * The researcher contains all the researches, helpers, data, and config.
+ */
+export default class AbstractResearcher {
+  /**
+   * Constructor
+   * @param {Paper} paper The Paper object that is needed within the researches.
+   *
+   * @constructor
+   */
+  constructor(paper) {
+    this.paper = paper;
+
+    // We expose the deprecated keywordCountInUrl for backwards compatibility.
+    this.defaultResearches = {
+      altTagCount,
+      countSentencesFromText,
+      findKeywordInFirstParagraph,
+      findKeyphraseInSEOTitle,
+      findTransitionWords,
+      functionWordsInKeyphrase,
+      getAnchorsWithKeyphrase,
+      getFleschReadingScore,
+      getKeyphraseCount,
+      getKeyphraseDensity,
+      getKeywordDensity,
+      getLinks,
+      getLinkStatistics,
+      getParagraphs,
+      getParagraphLength,
+      getProminentWordsForInsights,
+      getProminentWordsForInternalLinking,
+      getSentenceBeginnings,
+      getSubheadingTextLengths,
+      h1s,
+      imageCount,
+      keyphraseLength,
+      keywordCount,
+      keywordCountInSlug,
+      keywordCountInUrl,
+      matchKeywordInSubheadings,
+      metaDescriptionKeyword,
+      metaDescriptionLength,
+      morphology,
+      pageTitleWidth,
+      readingTime,
+      sentences,
+      wordCountInText,
+      videoCount,
+      getPassiveVoiceResult
+    };
+
+    this._data = {};
+
+    this.customResearches = {};
+
+    this.helpers = {
+      memoizedTokenizer
+    };
+
+    this.config = {
+      areHyphensWordBoundaries: true
+    };
+  }
+
+  /**
+   * Set the Paper associated with the Researcher.
+   *
+   * @param {Paper} paper The Paper to use within the Researcher.
+   *
+   * @throws {InvalidTypeError} Parameter needs to be an instance of the Paper object.
+   *
+   * @returns {void}
+   */
+  setPaper(paper) {
+    this.paper = paper;
+  }
+
+  /**
+   * Add a custom research that will be available within the Researcher.
+   *
+   * @param {string}   name     A name to reference the research by.
+   * @param {function} research The function to be added to the Researcher.
+   *
+   * @throws {MissingArgument}  Research name cannot be empty.
+   * @throws {InvalidTypeError} The research requires a valid Function callback.
+   *
+   * @returns {void}
+   */
+  addResearch(name, research) {
+    if (isUndefined(name) || isEmpty(name)) {
+      throw new MissingArgument('Research name cannot be empty');
+    }
+
+    if (!(research instanceof Function)) {
+      throw new InvalidTypeError('The research requires a Function callback.');
+    }
+
+    this.customResearches[name] = research;
+  }
+
+  /**
+   * Add research data to the researcher by the research name.
+   *
+   * @param {string} research The identifier of the research.
+   * @param {Object} data     The data object.
+   *
+   * @returns {void}.
+   */
+  addResearchData(research, data) {
+    this._data[research] = data;
+  }
+
+  /**
+   * Add a custom helper that will be available within the Researcher.
+   *
+   * @param {string}   name     A name to reference the helper by.
+   * @param {function} helper   The function to be added to the Researcher.
+   *
+   * @throws {MissingArgument}  Helper name cannot be empty.
+   * @throws {InvalidTypeError} The helper requires a valid Function callback.
+   *
+   * @returns {void}
+   */
+  addHelper(name, helper) {
+    if (isUndefined(name) || isEmpty(name)) {
+      throw new MissingArgument('Helper name cannot be empty');
+    }
+
+    if (!(helper instanceof Function)) {
+      throw new InvalidTypeError('The research requires a Function callback.');
+    }
+
+    this.helpers[name] = helper;
+  }
+
+  /**
+   * Add a custom configuration that will be available within the Researcher.
+   *
+   * @param {string}  name     A name to reference the helper by.
+   * @param {*}       config   The configuration to be added to the Researcher.
+   *
+   * @throws {MissingArgument}  Configuration name and the configuration itself cannot be empty.
+   *
+   * @returns {void}
+   */
+  addConfig(name, config) {
+    if (isUndefined(name) || isEmpty(name)) {
+      throw new MissingArgument(
+        'Failed to add the custom researcher config. Config name cannot be empty.'
+      );
+    }
+
+    if (isUndefined(config) || (isEmpty(config) && config === Object(config))) {
+      throw new MissingArgument(
+        'Failed to add the custom researcher config. Config cannot be empty.'
+      );
+    }
+
+    this.config[name] = config;
+  }
+
+  /**
+   * Check whether the research is known by the Researcher.
+   *
+   * @param {string} name The name to reference the research by.
+   *
+   * @returns {boolean} Whether or not the research is known by the Researcher.
+   */
+  hasResearch(name) {
+    return (
+      Object.keys(this.getAvailableResearches()).filter(function(research) {
+        return research === name;
+      }).length > 0
+    );
+  }
+
+  /**
+   * Check whether the helper is known by the Researcher.
+   *
+   * @param {string} name The name to reference the helper by.
+   *
+   * @returns {boolean} Whether or not the helper is known by the Researcher.
+   */
+  hasHelper(name) {
+    return (
+      Object.keys(this.getAvailableHelpers()).filter(function(helper) {
+        return helper === name;
+      }).length > 0
+    );
+  }
+
+  /**
+   * Check whether the config is known by the Researcher.
+   *
+   * @param {string} name The name to reference the config by.
+   *
+   * @returns {boolean} Whether or not the config is known by the Researcher.
+   */
+  hasConfig(name) {
+    return (
+      Object.keys(this.getAvailableConfig()).filter(function(config) {
+        return config === name;
+      }).length > 0
+    );
+  }
+
+  /**
+   * Check whether the research data is known by the Researcher.
+   *
+   * @param {string} name The name to reference the research data by.
+   *
+   * @returns {boolean} Whether or not the research data is known by the Researcher.
+   */
+  hasResearchData(name) {
+    return (
+      Object.keys(this.getAvailableResearchData()).filter(function(data) {
+        return data === name;
+      }).length > 0
+    );
+  }
+
+  /**
+   * Return all available researches.
+   *
+   * @returns {Object} An object containing all available researches.
+   */
+  getAvailableResearches() {
+    return merge(this.defaultResearches, this.customResearches);
+  }
+
+  /**
+   * Return all available helpers.
+   *
+   * @returns {Object} An object containing all available helpers.
+   */
+  getAvailableHelpers() {
+    return this.helpers;
+  }
+
+  /**
+   * Return all available configuration.
+   *
+   * @returns {Object} An object containing all available configuration.
+   */
+  getAvailableConfig() {
+    return this.config;
+  }
+
+  /**
+   * Return all available research data.
+   *
+   * @returns {Object} An object containing all available research data.
+   */
+  getAvailableResearchData() {
+    return this._data;
+  }
+
+  /**
+   * Return the Research by name.
+   *
+   * @param {string} name The name to reference the research by.
+   *
+   * @returns {*} Returns the result of the research or false if research does not exist.
+   *
+   * @throws {MissingArgument} Research name cannot be empty.
+   */
+  getResearch(name) {
+    if (isUndefined(name) || isEmpty(name)) {
+      throw new MissingArgument('Research name cannot be empty');
+    }
+
+    if (!this.hasResearch(name)) {
+      return false;
+    }
+
+    return this.getAvailableResearches()[name](this.paper, this);
+  }
+
+  /**
+   * Return the research data from a research data provider by research name.
+   *
+   * @param {string} research The identifier of the research.
+   *
+   * @returns {*} The data provided by the provider, false if the data do not exist
+   */
+  getData(research) {
+    if (this.hasResearchData(research)) {
+      return this._data[research];
+    }
+
+    return false;
+  }
+
+  /**
+   * Return language specific configuration by configuration name.
+   *
+   * @param {string} name The name of the configuration.
+   *
+   * @returns {*} The configuration, false if the configuration does not exist.
+   */
+  getConfig(name) {
+    if (this.hasConfig(name)) {
+      return this.config[name];
+    }
+
+    return false;
+  }
+
+  /**
+   * Return language specific helper by helper name.
+   *
+   * @param {string} name The name of the helper.
+   *
+   * @returns {*} The helper, false if the helper does not exist.
+   */
+  getHelper(name) {
+    if (this.hasHelper(name)) {
+      return this.helpers[name];
+    }
+
+    return false;
+  }
+}

package/src/languageProcessing/helpers/highlighting/getMarkingsInSentence.js

@@ -0,0 +1,125 @@
+import Mark from "../../../values/Mark";
+
+const markStart = "<yoastmark class='yoast-text-mark'>";
+const markEnd = "</yoastmark>";
+
+/**
+ * Adds `yoastmark` tags to the keyphrase matches in the sentence.
+ *
+ * This is a helper for search-based highlighting.
+ *
+ * @param {Sentence}	sentence	The sentence to add the `yoastmark` tags to.
+ * @param {Token[]}		matches		The array of the keyphrase matches.
+ *
+ * @returns {string} The sentence with the added `yoastmark` tags.
+ */
+const createMarksForSentence = ( sentence, matches  ) => {
+	const tokens = sentence.tokens;
+
+	const newTokens = [];
+	for ( let i = tokens.length - 1; i >= 0; i-- ) {
+		const token = tokens[ i ];
+		if ( matches.some( match => match.sourceCodeRange.startOffset === token.sourceCodeRange.startOffset ||
+			match.sourceCodeRange.endOffset === token.sourceCodeRange.endOffset ) ) {
+			newTokens.unshift( markStart, token.text, markEnd );
+		} else {
+			newTokens.unshift( token.text );
+		}
+	}
+	const markedSentence = newTokens.join( "" );
+	// Merge consecutive markings into one marking.
+	return markedSentence.replace( new RegExp( "</yoastmark>([ \u00A0]?)<yoastmark class='yoast-text-mark'>", "ig" ), "$1" );
+};
+
+/**
+ * Merges consecutive and overlapping markings into one marking.
+ *
+ * This is a helper for position-based highlighting.
+ *
+ * @param {Mark[]}	markings	An array of markings to merge.
+ *
+ * @returns {Mark[]} An array of markings where consecutive and overlapping markings are merged.
+ */
+const mergeConsecutiveAndOverlappingMarkings = ( markings ) => {
+	const newMarkings = [];
+
+	// Sort markings by start offset. This is probably redundant, but for extra safety.
+	markings.sort( function( a, b ) {
+		return a.getPositionStart() - b.getPositionStart();
+	} );
+
+	markings.forEach( ( marking ) => {
+		if ( newMarkings.length === 0 ) {
+			newMarkings.push( marking );
+			return;
+		}
+
+		const lastMarking = newMarkings[ newMarkings.length - 1 ];
+		// Adding 1 to the position end, with the assumption that the language uses spaces.
+		// When we adapt Japanese to use this helper, this check should also be adapted.
+		if ( ( lastMarking.getPositionEnd() + 1 === marking.getPositionStart() ) ||
+			( marking.getPositionStart() <= lastMarking.getPositionEnd() ) ) {
+			// The marking is consecutive to the last marking, so we extend the last marking to include the new marking.
+			lastMarking.setPositionEnd( marking.getPositionEnd() );
+			lastMarking.setBlockPositionEnd( marking.getBlockPositionEnd() );
+		} else {
+			// The marking is not consecutive to the last marking, so we add it to the array by itself.
+			newMarkings.push( marking );
+		}
+	} );
+
+	return newMarkings;
+};
+
+
+/**
+ * Gets the Mark objects of all keyphrase matches in the sentence.
+ * Currently, this function creates Mark objects compatible for both search-based and position-based highlighting.
+ * In a pure position-based highlighting, we don't need to provide 'marked' and 'original' when creating the Mark object.
+ *
+ * @param {Sentence}	sentence			The sentence to check.
+ * @param {Token[]}		matchesInSentence	An array containing the keyphrase matches in the sentence.
+ *
+ * @returns {Mark[]} The array of Mark objects of the keyphrase matches in the sentence.
+ */
+function getMarkingsInSentence( sentence, matchesInSentence ) {
+	if ( matchesInSentence.length === 0 ) {
+		return [];
+	}
+
+	// Create the marked sentence that is used for search-based highlighting.
+	const markedSentence = createMarksForSentence( sentence, matchesInSentence );
+
+	/*
+	 * Note that there is a paradigm shift:
+	 * With search-based highlighting, there would be one Mark object for the entire sentence.
+	 * With position-based highlighting, there is a Mark object for each match.
+	 * Hence, in order to be backwards compatible with search-based highlighting,
+	 * all Mark objects for a sentence have the same markedSentence.
+	 */
+	const markings = matchesInSentence.map( token => {
+		const startOffset = token.sourceCodeRange.startOffset;
+		const endOffset =  token.sourceCodeRange.endOffset;
+		return new Mark( {
+			position: {
+				startOffset: startOffset,
+				endOffset: endOffset,
+				// Relative to start of block positions.
+				startOffsetBlock: startOffset - ( sentence.parentStartOffset || 0 ),
+				endOffsetBlock: endOffset - ( sentence.parentStartOffset || 0 ),
+				// The client id of the block the match was found in.
+				clientId: sentence.parentClientId || "",
+				// The attribute id of the Yoast sub-block the match was found in.
+				attributeId: sentence.parentAttributeId || "",
+				// Whether the match was found in the first section of the Yoast sub-block.
+				isFirstSection: sentence.isParentFirstSectionOfBlock || false,
+			},
+			marked: markedSentence,
+			original: sentence.text,
+		} );
+	} );
+
+	return mergeConsecutiveAndOverlappingMarkings( markings );
+}
+
+export default getMarkingsInSentence;

package/src/languageProcessing/helpers/html/getFieldsToMark.js

@@ -0,0 +1,29 @@
+import { flatten, isUndefined, uniq } from "lodash";
+import { getSubheadings } from "./getSubheadings";
+
+/**
+ * Gets the part of the html that we want to apply the marking to.
+ *
+ * @param {array}   marks  The array of mark objects.
+ * @param {string}  html   The html of the page where we want to apply the marking to.
+ *
+ * @returns {{selectedHTML: *[], fieldsToMark: *}} The selected part of the html we want to apply the marking to.
+ */
+export function getFieldsToMark( marks, html ) {
+	const fieldsToMark = uniq( flatten( marks.map( mark => {
+		if ( ! isUndefined( mark.getFieldsToMark() ) ) {
+			return mark.getFieldsToMark();
+		}
+	} ) ) );
+
+	const selectedHTML = [];
+	fieldsToMark.forEach( field => {
+		if ( field === "heading" ) {
+			const subheadings = getSubheadings( html );
+			subheadings.forEach( subheading => {
+				selectedHTML.push( subheading[ 0 ] );
+			} );
+		}
+	} );
+	return { fieldsToMark, selectedHTML };
+}

package/src/languageProcessing/helpers/html/getSubheadingTexts.js

@@ -0,0 +1,47 @@
+import { isUndefined } from "lodash";
+
+/**
+ * Returns An array of found subheadings including the following text.
+ *
+ * @param {string} text The text to analyze from.
+ *
+ * @returns {Array} An array of found subheadings including the following text.
+ */
+export default function( text ) {
+	// Match all the subheadings.
+	const subheadings = [ ...text.matchAll( new RegExp( "<h([1-6])(?:[^>]+)?>(.*?)<\\/h\\1>", "ig" ) ) ];
+
+	const foundSubheadings = [];
+
+	subheadings.forEach( ( subheading, i ) => {
+		// Retrieve the current subheading string.
+		const subheadingString = subheading[ 0 ];
+		// Retrieve the current subheading index.
+		const currentMatchIndex = subheading.index;
+		// Retrieve the next subheading.
+		const nextSubheading = subheadings[ i + 1 ];
+
+		let nextMatchIndex;
+
+		// Check if there is a next subheading.
+		if ( isUndefined( nextSubheading ) ) {
+			// If there is no next subheading, the next match index is the index of the last character in the text.
+			nextMatchIndex = subheading.input.length;
+		} else {
+			// Retrieve the index of the next subheading.
+			nextMatchIndex = nextSubheading.index;
+		}
+
+		// Retrieve the text following the current subheading.
+		const textFollowingCurrentSubheading = subheading.input.slice( currentMatchIndex + subheadingString.length, nextMatchIndex );
+		foundSubheadings.push( {
+			subheading: subheadingString,
+			text: textFollowingCurrentSubheading,
+			index: currentMatchIndex,
+		} );
+	} );
+
+	return foundSubheadings;
+}
+
+

package/src/languageProcessing/helpers/html/getSubheadings.js

@@ -0,0 +1,95 @@
+/**
+ * Gets all subheadings from the text and returns these in an array.
+ *
+ * @param {string} text The text to return the headings from.
+ *
+ * @returns {Array<string[]>} Matches of subheadings in the text, first key is everything including tags,
+ *                            second is the heading level, third is the content of the subheading.
+ */
+function getSubheadings( text ) {
+	const subheadings = [];
+	const regex = /<h([1-6])(?:[^>]+)?>(.*?)<\/h\1>/ig;
+	let match;
+
+	while ( ( match = regex.exec( text ) ) !== null ) {
+		subheadings.push( match );
+	}
+
+	return subheadings;
+}
+
+/**
+ * Gets all the level 2 and 3 subheadings from the text and returns these in an array.
+ *
+ * @param {string} text The text to return the headings from.
+ *
+ * @returns {Array<string[]>} Matches of subheadings in the text, first key is everything including tags,
+ *                            second is the heading level, third is the content of the subheading.
+ */
+function getSubheadingsTopLevel( text ) {
+	const subheadings = [];
+	const regex = /<h([2-3])(?:[^>]+)?>(.*?)<\/h\1>/ig;
+	let match;
+
+	while ( ( match = regex.exec( text ) ) !== null ) {
+		subheadings.push( match );
+	}
+
+	return subheadings;
+}
+
+/**
+ * Gets the content of subheadings in the text.
+ *
+ * @param {string} text The text to get the subheading contents from.
+ *
+ * @returns {string[]} A list of all the subheadings with their content.
+ */
+function getSubheadingContents( text ) {
+	const subheadings = getSubheadings( text );
+
+	return subheadings.map( subheading => subheading[ 0 ] );
+}
+
+/**
+ * Gets the content of subheadings h2 and h3 in the text.
+ *
+ * @param {string} text The text to get the subheading contents from.
+ *
+ * @returns {string[]} A list of all the subheadings with their content.
+ */
+function getSubheadingContentsTopLevel( text ) {
+	const subheadings = getSubheadingsTopLevel( text );
+
+	// Only return the entire string matched, not the rest of the outputs of the regex.exec function.
+	return subheadings.map( subheading => subheading[ 0 ] );
+}
+
+/**
+ * Removes all level 2 and 3 subheadings from the text.
+ *
+ * @param {string} text The text to remove the headings from.
+ *
+ * @returns {string} The text with removed subheadings.
+ */
+function removeSubheadingsTopLevel( text ) {
+	const regex = /<h([2-3])(?:[^>]+)?>(.*?)<\/h\1>/ig;
+
+	return text.replace( regex, "" );
+}
+
+export {
+	getSubheadings,
+	getSubheadingsTopLevel,
+	getSubheadingContents,
+	getSubheadingContentsTopLevel,
+	removeSubheadingsTopLevel,
+};
+
+export default {
+	getSubheadings: getSubheadings,
+	getSubheadingsTopLevel: getSubheadingsTopLevel,
+	getSubheadingContents: getSubheadingContents,
+	getSubheadingContentsTopLevel: getSubheadingContentsTopLevel,
+	removeSubheadingsTopLevel: removeSubheadingsTopLevel,
+};