axyseo 2.0.0-alpha.0.0.4 → 2.0.0-alpha.0.0.5

package/src/languageProcessing/helpers/sanitize/removePunctuation.js

@@ -1,64 +0,0 @@
-import { unifyNonBreakingSpace } from "./unifyWhitespace";
-
-/**
- * String containing all the characters that we consider punctuation.
- * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
- * @type {string}
- */
-export const punctuationRegexString = "\\–\\-\\(\\)_\\[\\]’‘“”〝〞〟‟„\"'.?!:;,¿¡«»‹›\u2014\u00d7\u002b\u0026\u06d4\u061f\u060C\u061B\u3002\uff61" +
-"\uff01\u203c\uff1f\u2047\u2049\u2048\u2025\u2026\u30fb\u30fc\u3001\u3003\u3004\u3006\u3007\u3008\u3009\u300a\u300b\u300c\u300d\u300e" +
-"\u300f\u3010\u3011\u3012\u3013\u3014\u3015\u3016\u3017\u3018\u3019\u301a\u301b\u301c\u301d\u301e\u301f\u3020\u3036\u303c\u303d\uff5b" +
-"\uff5d\uff5c\uff5e\uff5f\uff60\uff62\uff63\uff64\uff3b\uff3d\uff65\uffe5\uff04\uff05\uff20\uff06\uff07\uff08\uff09\uff0a\uff0f\uff1a" +
-"\uff1b\uff1c\uff1e\uff3c\\<>";
-
-/**
- * Array containing all the characters that we consider punctuation.
- * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
- * @type {string[]}
- */
-export const punctuationList = punctuationRegexString.split( "" );
-
-export const punctuationRegexStart = new RegExp( "^[" + punctuationRegexString + "]+" );
-export const punctuationRegexEnd = new RegExp( "[" + punctuationRegexString +  "]+$" );
-
-/*
- * \u2014 - em-dash
- * \u00d7 - multiplication sign
- * \u002b - plus sign
- * \u0026 - ampersand
- * \u06d4 - Urdu full stop
- * \u061f - Arabic question mark
- * \u060C - Arabic comma
- * \u061B - Arabic semicolon
- */
-
-/**
- * Replaces punctuation characters from the given text string.
- *
- * @param {String} text The text to remove the punctuation characters for.
- *
- * @returns {String} The sanitized text.
- */
-export default function( text ) {
-	// Unify whitespaces and non-breaking spaces.
-	text = unifyNonBreakingSpace( text );
-
-	// Remove &amp from the string. In some editors (a.o. Block and Elementor) the ampersand (&) is transformed into &amp.
-	// If it is not removed, then it is returned as "amp" and counted as a word in assessments downstream.
-	text = text.replace( "\u0026amp", "" );
-
-	/*
-	 * Remove backslash from the beginning and end of a word/text.
-	 * When a string such as `This is a \"calico\" cat` enters the Paper,
-	 * the Paper adds two extra backslash in front of the original backslash.
-	 * After the text is split into words, we also need to remove those backslashes from the word.
-	 * Otherwise, it will be problematic when word boundary regex is added to the word.
-	 */
-	const backslashRegex = new RegExp( "(\\\\)", "g" );
-
-	text = text.replace( backslashRegex, "" );
-	text = text.replace( punctuationRegexStart, "" );
-	text = text.replace( punctuationRegexEnd, "" );
-
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/removePunctuationExceptQuotes.js

@@ -1,18 +0,0 @@
-// Replace all other punctuation characters at the beginning or at the end of a word.
-const punctuationRegexString = "[\\–\\-\\(\\)_\\[\\]’'.?!:;,¿¡«»‹›\u2014\u00d7\u002b\u0026<>]+";
-const punctuationRegexStart = new RegExp( "^" + punctuationRegexString );
-const punctuationRegexEnd = new RegExp( punctuationRegexString + "$" );
-
-/**
- * Replaces punctuation characters at the beginning and end of a given text string.
- *
- * @param {String} text The text to remove the punctuation characters for.
- *
- * @returns {String} The sanitized text.
- */
-export default function( text ) {
-	text = text.replace( punctuationRegexStart, "" );
-	text = text.replace( punctuationRegexEnd, "" );
-
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/removeSentenceTerminators.js

@@ -1,13 +0,0 @@
-// These are sentence terminators, that never should be in the middle of a word.
-const sentenceTerminators = /[.?!:;,]/g;
-
-/**
- * Replaces sentence terminators from the text.
- *
- * @param {String} text The text to remove the terminators from.
- *
- * @returns {String} The sanitized text.
- */
-export default function( text ) {
-	return text.replace( sentenceTerminators, "" );
-}

package/src/languageProcessing/helpers/sanitize/removeURLs.js

@@ -1,13 +0,0 @@
-const urlRegex = new RegExp( "(ftp|http(s)?:\\/\\/.)(www\\\\.)?[-a-zA-Z0-9@:%._\\/+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:;%_\\/+.~#?&()=]*)" +
-	"|www\\.[-a-zA-Z0-9@:%._\\/+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:;%_\\/+.~#?&()=]*)", "igm" );
-/**
- * This regex is used to match URLs, whether they are embedded in tags or not.
- * It doesn't match domain names (e.g. "yoast.com" in "We got so much traffic on yoast.com after the latest release").
- *
- * @param {string} text The text to remove URLs from.
- *
- * @returns {string} The text without URLs.
- */
-export default function( text ) {
-	return text.replace( urlRegex, "" );
-}

package/src/languageProcessing/helpers/sanitize/sanitizeLineBreakTag.js

@@ -1,11 +0,0 @@
-/**
- * Replaces line break tag that contains an attribute with paragraph tag.
- * Line break tags with attribute aren't properly sanitized using wordpress `autop` function.
- *
- * @param {string} text The text to sanitize.
- *
- * @returns {string} The sanitized text.
- */
-export default function( text ) {
-	return text.replace( /<br.*?>\s*<.*?br.*?>/gm, "</p><p>" );
-}

package/src/languageProcessing/helpers/sanitize/sanitizeString.js

@@ -1,18 +0,0 @@
-import { stripFullTags as stripTags } from "./stripHTMLTags.js";
-import { unifyAllSpaces } from "./unifyWhitespace";
-
-/**
- * Sanitizes the text before we use the text for the analysis.
- *
- * @param {String} text The text to be sanitized.
- *
- * @returns {String} The sanitized text.
- */
-export default function( text ) {
-	// Unify whitespaces and non-breaking spaces.
-	text = unifyAllSpaces( text );
-	// Strip the tags and multiple spaces.
-	text = stripTags( text );
-
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/stripHTMLTags.js

@@ -1,57 +0,0 @@
-/** @module stringProcessing/stripHTMLTags */
-
-import stripSpaces from "./stripSpaces.js";
-
-import { blockElements } from "../html/html.js";
-
-const blockElementStartRegex = new RegExp( "^<(" + blockElements.join( "|" ) + ")[^>]*?>", "i" );
-const blockElementEndRegex = new RegExp( "</(" + blockElements.join( "|" ) + ")[^>]*?>$", "i" );
-
-/**
- * Strip incomplete tags within a text. Strips an end tag at the beginning of a string and the start tag at the end of a
- * start of a string.
- *
- * @param {String} text The text to strip the HTML-tags from at the begin and end.
- * @returns {String} The text without HTML-tags at the begin and end.
- */
-const stripIncompleteTags = function( text ) {
-	text = text.replace( /^(<\/([^>]+)>)+/i, "" );
-	text = text.replace( /(<([^/>]+)>)+$/i, "" );
-	return text;
-};
-
-/**
- * Removes the block element tags at the beginning and end of a string and returns this string.
- *
- * @param {string} text The unformatted string.
- * @returns {string} The text with removed HTML begin and end block elements
- */
-const stripBlockTagsAtStartEnd = function( text ) {
-	text = text.replace( blockElementStartRegex, "" );
-	text = text.replace( blockElementEndRegex, "" );
-	return text;
-};
-
-/**
- * Strip HTML-tags from text
- *
- * @param {String} text The text to strip the HTML-tags from.
- * @returns {String} The text without HTML-tags.
- */
-const stripFullTags = function( text ) {
-	text = text.replace( /(<([^>]+)>)/ig, " " );
-	text = stripSpaces( text );
-	return text;
-};
-
-export {
-	stripFullTags,
-	stripIncompleteTags,
-	stripBlockTagsAtStartEnd,
-};
-
-export default {
-	stripFullTags: stripFullTags,
-	stripIncompleteTags: stripIncompleteTags,
-	stripBlockTagsAtStartEnd: stripBlockTagsAtStartEnd,
-};

package/src/languageProcessing/helpers/sanitize/stripNonTextTags.js

@@ -1,15 +0,0 @@
-/** @module stringProcessing/stripNonTextTags */
-
-import stripSpaces from "./stripSpaces.js";
-
-/**
- * Strips all tags from the text, except li, p, dd and h1-h6 tags from the text that contain content to check.
- *
- * @param {string} text The text to strip tags from
- * @returns {string} The text stripped of tags, except for li, p, dd and h1-h6 tags.
- */
-export default function( text ) {
-	text = text.replace( /<(?!li|\/li|p|\/p|h1|\/h1|h2|\/h2|h3|\/h3|h4|\/h4|h5|\/h5|h6|\/h6|dd).*?>/g, "" );
-	text = stripSpaces( text );
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/stripNumbers.js

@@ -1,21 +0,0 @@
-/** @module stringProcessing/stripNumbers */
-
-import stripSpaces from "./stripSpaces.js";
-
-/**
- * Removes all words comprised only of numbers.
- *
- * @param {string} text to remove words
- * @returns {string} The text with 'numberonly' words removed.
- */
-export default function( text ) {
-	// Remove "words" comprised only of numbers
-	text = text.replace( /\b[0-9]+\b/g, "" );
-
-	text = stripSpaces( text );
-
-	if ( text === "." ) {
-		text = "";
-	}
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/stripSpaces.js

@@ -1,23 +0,0 @@
-/** @module stringProcessing/stripSpaces */
-
-/**
- * Replaces multiple spaces with a single space.
- * Removes spaces followed by a period (if there's no text after the period) and spaces in the beginning or end of a string.
- *
- * @param {String} text The text to strip spaces from.
- * @returns {String} The text with stripped spaces.
- */
-export default function( text ) {
-	// Replaces multiple spaces with a single space.
-	text = text.replace( /\s{2,}/g, " " );
-	// Replaces space(s) followed by a period, if the period is the last character, with only the period.
-	text = text.replace( /\s\.$/, "." );
-	// Removes first/last character if it's a space.
-	text = text.replace( /^\s+|\s+$/g, "" );
-	// Replaces spaces followed by a Japanese period with only the period.
-	text = text.replace( /\s。/g, "。" );
-	// Replaces spaces after a Japanese period with only the period.
-	text = text.replace( /。\s/g, "。" );
-
-	return text;
-}

package/src/languageProcessing/helpers/sanitize/stripWordBoundaries.js

@@ -1,65 +0,0 @@
-/*
- * \u00a0 - No-break space
- * \u06d4 - Urdu full stop
- * \u061f - Arabic question mark
- * \u060C - Arabic comma
- * \u061B - Arabic semicolon
- */
-const wordBoundary = "[ \\u00a0\\u06d4\\u061f\\u060C\\u061B \\n\\r\\t.,'()\"+\\-;!?:/»«‹›<>]";
-const wordBoundaryStart = new RegExp( "^(" + wordBoundary + "+)", "ig" );
-const wordBoundaryEnd = new RegExp( "(" + wordBoundary + "+$)", "ig" );
-
-/**
- * Strip word boundary markers from text in the beginning
- *
- * @param {String} text The text to strip word boundary markers from.
- *
- * @returns {String} The text without double word boundary markers.
- */
-const stripWordBoundariesStart = function( text ) {
-	// Remove first character if word boundary
-	text = text.replace( wordBoundaryStart, "" );
-
-	return text;
-};
-
-/**
- * Strip word boundary markers from text in the end
- *
- * @param {String} text The text to strip word boundary markers from.
- *
- * @returns {String} The text without double word boundary markers.
- */
-const stripWordBoundariesEnd = function( text ) {
-	// Remove last character if word boundary
-	text = text.replace( wordBoundaryEnd, "" );
-
-	return text;
-};
-
-/**
- * Strip word boundary markers from text in the beginning and in the end
- *
- * @param {String} text The text to strip word boundary markers from.
- *
- * @returns {String} The text without word boundary markers.
- */
-const stripWordBoundariesEverywhere = function( text ) {
-	// Remove first/last character if word boundary
-	text = text.replace( wordBoundaryStart, "" );
-	text = text.replace( wordBoundaryEnd, "" );
-
-	return text;
-};
-
-export {
-	stripWordBoundariesStart,
-	stripWordBoundariesEnd,
-	stripWordBoundariesEverywhere,
-};
-
-export default {
-	stripWordBoundariesStart: stripWordBoundariesStart,
-	stripWordBoundariesEnd: stripWordBoundariesEnd,
-	stripWordBoundariesEverywhere: stripWordBoundariesEverywhere,
-};

package/src/languageProcessing/helpers/sanitize/unifyWhitespace.js

@@ -1,61 +0,0 @@
-/** @module stringProcessing/unifyWhitespace */
-
-/**
- * Replaces a non-breaking space with a normal space.
- *
- * @param {string} text The string to replace the non-breaking space in.
- *
- * @returns {string} The text with unified spaces.
- */
-const unifyNonBreakingSpace = function( text ) {
-	return text.replace( / /g, " " );
-};
-
-/**
- * Replaces an em dash with a normal space.
- *
- * @param {string} text The string to replace the em dash in.
- *
- * @returns {string} The text with unified spaces.
- */
-const unifyEmDash = function( text ) {
-	return text.replace( /\u2014/g, " " );
-};
-
-/**
- * Replaces all whitespace characters with a normal space.
- *
- * @param {string} text The string to replace the whitespace characters in.
- *
- * @returns {string} The text with unified spaces.
- */
-const unifyWhiteSpace = function( text ) {
-	return text.replace( /\s/g, " " );
-};
-
-/**
- * Converts all whitespace to spaces.
- *
- * @param {string} text The text to replace spaces.
- *
- * @returns {string} The text with unified spaces.
- */
-const unifyAllSpaces = function( text ) {
-	text = unifyNonBreakingSpace( text );
-	text = unifyEmDash( text );
-	return unifyWhiteSpace( text );
-};
-
-export {
-	unifyNonBreakingSpace,
-	unifyEmDash,
-	unifyWhiteSpace,
-	unifyAllSpaces,
-};
-
-export default {
-	unifyNonBreakingSpace: unifyNonBreakingSpace,
-	unifyEmDash: unifyEmDash,
-	unifyWhiteSpace: unifyWhiteSpace,
-	unifyAllSpaces: unifyAllSpaces,
-};