axyseo 2.0.0-alpha.1

package/build/languageProcessing/helpers/regex/createRegexFromArray.js

@@ -0,0 +1,38 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+var _replaceDiacritics = _interopRequireDefault(require("../transliterate/replaceDiacritics"));
+var _sanitizeString = _interopRequireDefault(require("../sanitize/sanitizeString"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/createRegexFromArray */
+
+/**
+ * Creates a regex of combined strings from the input array.
+ *
+ * @param {string[]} array                      The array with strings
+ * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.
+ * @param {string} [extraBoundary=""]           A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.
+ *
+ * @returns {RegExp} regex                              The regex created from the array.
+ */
+function _default(array, disableWordBoundary = false, extraBoundary = "", doReplaceDiacritics = false) {
+  array = (0, _lodash.map)(array, function (string) {
+    if (doReplaceDiacritics) {
+      string = (0, _replaceDiacritics.default)(string);
+    }
+    string = (0, _sanitizeString.default)(string);
+    if (disableWordBoundary) {
+      return string;
+    }
+    return (0, _addWordboundary.default)(string, true, extraBoundary);
+  });
+  const regexString = "(" + array.join(")|(") + ")";
+  return new RegExp(regexString, "ig");
+}
+//# sourceMappingURL=createRegexFromArray.js.map

package/build/languageProcessing/helpers/regex/createRegexFromArray.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRegexFromArray.js","names":["_lodash","require","_addWordboundary","_interopRequireDefault","_replaceDiacritics","_sanitizeString","e","__esModule","default","_default","array","disableWordBoundary","extraBoundary","doReplaceDiacritics","map","string","replaceDiacritics","sanitizeString","addWordBoundary","regexString","join","RegExp"],"sources":["../../../../src/languageProcessing/helpers/regex/createRegexFromArray.js"],"sourcesContent":["/** @module stringProcessing/createRegexFromArray */\n\nimport { map } from \"lodash\";\nimport addWordBoundary from \"../word/addWordboundary.js\";\nimport replaceDiacritics from \"../transliterate/replaceDiacritics\";\nimport sanitizeString from \"../sanitize/sanitizeString\";\n\n/**\n * Creates a regex of combined strings from the input array.\n *\n * @param {string[]} array                      The array with strings\n * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.\n * @param {string} [extraBoundary=\"\"]           A string that is used as extra boundary for the regex.\n * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.\n *\n * @returns {RegExp} regex                              The regex created from the array.\n */\nexport default function( array, disableWordBoundary = false, extraBoundary = \"\", doReplaceDiacritics = false ) {\n\tarray = map( array, function( string ) {\n\t\tif ( doReplaceDiacritics ) {\n\t\t\tstring = replaceDiacritics( string );\n\t\t}\n\n\t\tstring = sanitizeString( string );\n\n\t\tif ( disableWordBoundary ) {\n\t\t\treturn string;\n\t\t}\n\t\treturn addWordBoundary( string, true, extraBoundary );\n\t} );\n\n\tconst regexString = \"(\" + array.join( \")|(\" ) + \")\";\n\n\treturn new RegExp( regexString, \"ig\" );\n}\n"],"mappings":";;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,gBAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,kBAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,eAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAwD,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AALxD;;AAOA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,KAAK,EAAEC,mBAAmB,GAAG,KAAK,EAAEC,aAAa,GAAG,EAAE,EAAEC,mBAAmB,GAAG,KAAK,EAAG;EAC9GH,KAAK,GAAG,IAAAI,WAAG,EAAEJ,KAAK,EAAE,UAAUK,MAAM,EAAG;IACtC,IAAKF,mBAAmB,EAAG;MAC1BE,MAAM,GAAG,IAAAC,0BAAiB,EAAED,MAAO,CAAC;IACrC;IAEAA,MAAM,GAAG,IAAAE,uBAAc,EAAEF,MAAO,CAAC;IAEjC,IAAKJ,mBAAmB,EAAG;MAC1B,OAAOI,MAAM;IACd;IACA,OAAO,IAAAG,wBAAe,EAAEH,MAAM,EAAE,IAAI,EAAEH,aAAc,CAAC;EACtD,CAAE,CAAC;EAEH,MAAMO,WAAW,GAAG,GAAG,GAAGT,KAAK,CAACU,IAAI,CAAE,KAAM,CAAC,GAAG,GAAG;EAEnD,OAAO,IAAIC,MAAM,CAAEF,WAAW,EAAE,IAAK,CAAC;AACvC","ignoreList":[]}

package/build/languageProcessing/helpers/regex/createRegexFromDoubleArray.js

@@ -0,0 +1,39 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/createRegexFromDoubleArray */
+
+/**
+ * Creates a regex string of combined strings from the input array.
+ *
+ * @param {array} array The array containing the various parts of a transition word combination.
+ *
+ * @returns {array} The array with replaced entries.
+ */
+const wordCombinationToRegexString = function (array) {
+  array = array.map(function (word) {
+    return (0, _addWordboundary.default)(word);
+  });
+  return array.join("(.*?)");
+};
+
+/**
+ * Creates a regex of combined strings from the input array, containing arrays with two entries.
+ *
+ * @param {array} array The array containing arrays.
+ *
+ * @returns {RegExp} The regex created from the array.
+ */
+function _default(array) {
+  array = array.map(function (wordCombination) {
+    return wordCombinationToRegexString(wordCombination);
+  });
+  const regexString = "(" + array.join(")|(") + ")";
+  return new RegExp(regexString, "ig");
+}
+//# sourceMappingURL=createRegexFromDoubleArray.js.map

package/build/languageProcessing/helpers/regex/createRegexFromDoubleArray.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRegexFromDoubleArray.js","names":["_addWordboundary","_interopRequireDefault","require","e","__esModule","default","wordCombinationToRegexString","array","map","word","addWordBoundary","join","_default","wordCombination","regexString","RegExp"],"sources":["../../../../src/languageProcessing/helpers/regex/createRegexFromDoubleArray.js"],"sourcesContent":["/** @module stringProcessing/createRegexFromDoubleArray */\n\nimport addWordBoundary from \"../word/addWordboundary.js\";\n\n/**\n * Creates a regex string of combined strings from the input array.\n *\n * @param {array} array The array containing the various parts of a transition word combination.\n *\n * @returns {array} The array with replaced entries.\n */\nconst wordCombinationToRegexString = function( array ) {\n\tarray = array.map( function( word ) {\n\t\treturn addWordBoundary( word );\n\t} );\n\treturn array.join( \"(.*?)\" );\n};\n\n/**\n * Creates a regex of combined strings from the input array, containing arrays with two entries.\n *\n * @param {array} array The array containing arrays.\n *\n * @returns {RegExp} The regex created from the array.\n */\nexport default function( array ) {\n\tarray = array.map( function( wordCombination ) {\n\t\treturn wordCombinationToRegexString( wordCombination );\n\t} );\n\n\tconst regexString = \"(\" + array.join( \")|(\" ) + \")\";\n\n\treturn new RegExp( regexString, \"ig\" );\n}\n"],"mappings":";;;;;;AAEA,IAAAA,gBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAyD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAFzD;;AAIA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,4BAA4B,GAAG,SAAAA,CAAUC,KAAK,EAAG;EACtDA,KAAK,GAAGA,KAAK,CAACC,GAAG,CAAE,UAAUC,IAAI,EAAG;IACnC,OAAO,IAAAC,wBAAe,EAAED,IAAK,CAAC;EAC/B,CAAE,CAAC;EACH,OAAOF,KAAK,CAACI,IAAI,CAAE,OAAQ,CAAC;AAC7B,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAAUL,KAAK,EAAG;EAChCA,KAAK,GAAGA,KAAK,CAACC,GAAG,CAAE,UAAUK,eAAe,EAAG;IAC9C,OAAOP,4BAA4B,CAAEO,eAAgB,CAAC;EACvD,CAAE,CAAC;EAEH,MAAMC,WAAW,GAAG,GAAG,GAAGP,KAAK,CAACI,IAAI,CAAE,KAAM,CAAC,GAAG,GAAG;EAEnD,OAAO,IAAII,MAAM,CAAED,WAAW,EAAE,IAAK,CAAC;AACvC","ignoreList":[]}

package/build/languageProcessing/helpers/regex/createWordRegex.js

@@ -0,0 +1,34 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _sanitizeString = _interopRequireDefault(require("../sanitize/sanitizeString"));
+var _replaceDiacritics = _interopRequireDefault(require("../transliterate/replaceDiacritics.js"));
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/stringToRegex */
+/**
+ * Creates a regex from a string so it can be matched everywhere in the same way.
+ *
+ * @param {string}  string                     The string to make a regex from.
+ * @param {string}  [extraBoundary=""]         A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.
+ *
+ * @returns {RegExp} regex The regex made from the keyword
+ */
+var _default = exports.default = (0, _lodash.memoize)(function (string, extraBoundary, doReplaceDiacritics) {
+  if ((0, _lodash.isUndefined)(extraBoundary)) {
+    extraBoundary = "";
+  }
+  if ((0, _lodash.isUndefined)(doReplaceDiacritics) || doReplaceDiacritics === true) {
+    string = (0, _replaceDiacritics.default)(string);
+  }
+  string = (0, _sanitizeString.default)(string);
+  string = (0, _lodash.escapeRegExp)(string);
+  string = (0, _addWordboundary.default)(string, false, extraBoundary);
+  return new RegExp(string, "ig");
+});
+//# sourceMappingURL=createWordRegex.js.map

package/build/languageProcessing/helpers/regex/createWordRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createWordRegex.js","names":["_lodash","require","_sanitizeString","_interopRequireDefault","_replaceDiacritics","_addWordboundary","e","__esModule","default","_default","exports","memoize","string","extraBoundary","doReplaceDiacritics","isUndefined","replaceDiacritics","sanitizeString","escapeRegExp","addWordBoundary","RegExp"],"sources":["../../../../src/languageProcessing/helpers/regex/createWordRegex.js"],"sourcesContent":["/** @module stringProcessing/stringToRegex */\nimport { escapeRegExp, isUndefined, memoize } from \"lodash\";\nimport sanitizeString from \"../sanitize/sanitizeString\";\nimport replaceDiacritics from \"../transliterate/replaceDiacritics.js\";\nimport addWordBoundary from \"../word/addWordboundary.js\";\n\n/**\n * Creates a regex from a string so it can be matched everywhere in the same way.\n *\n * @param {string}  string                     The string to make a regex from.\n * @param {string}  [extraBoundary=\"\"]         A string that is used as extra boundary for the regex.\n * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.\n *\n * @returns {RegExp} regex The regex made from the keyword\n */\nexport default memoize( function( string, extraBoundary, doReplaceDiacritics ) {\n\tif ( isUndefined( extraBoundary ) ) {\n\t\textraBoundary = \"\";\n\t}\n\n\tif ( isUndefined( doReplaceDiacritics ) || doReplaceDiacritics === true ) {\n\t\tstring = replaceDiacritics( string );\n\t}\n\n\tstring = sanitizeString( string );\n\tstring = escapeRegExp( string );\n\tstring = addWordBoundary( string, false, extraBoundary );\n\n\treturn new RegExp( string, \"ig\" );\n} );\n"],"mappings":";;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,eAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,kBAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,gBAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAyD,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAJzD;AAMA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AARA,IAAAG,QAAA,GAAAC,OAAA,CAAAF,OAAA,GASe,IAAAG,eAAO,EAAE,UAAUC,MAAM,EAAEC,aAAa,EAAEC,mBAAmB,EAAG;EAC9E,IAAK,IAAAC,mBAAW,EAAEF,aAAc,CAAC,EAAG;IACnCA,aAAa,GAAG,EAAE;EACnB;EAEA,IAAK,IAAAE,mBAAW,EAAED,mBAAoB,CAAC,IAAIA,mBAAmB,KAAK,IAAI,EAAG;IACzEF,MAAM,GAAG,IAAAI,0BAAiB,EAAEJ,MAAO,CAAC;EACrC;EAEAA,MAAM,GAAG,IAAAK,uBAAc,EAAEL,MAAO,CAAC;EACjCA,MAAM,GAAG,IAAAM,oBAAY,EAAEN,MAAO,CAAC;EAC/BA,MAAM,GAAG,IAAAO,wBAAe,EAAEP,MAAM,EAAE,KAAK,EAAEC,aAAc,CAAC;EAExD,OAAO,IAAIO,MAAM,CAAER,MAAM,EAAE,IAAK,CAAC;AAClC,CAAE,CAAC","ignoreList":[]}

package/build/languageProcessing/helpers/regex/matchStringWithRegex.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+/** @module stringProcessing/matchStringWithRegex */
+
+/**
+ * Checks a string with a regex, return all matches found with that regex.
+ *
+ * @param {String} 		text        The text to match.
+ * @param {object} 		regex  		A compiled regular expression.
+ *
+ * @returns {Array} Array with matches, empty array if no matches found.
+ */
+function _default(text, regex) {
+  let matches = text.match(regex);
+  if (matches === null) {
+    matches = [];
+  }
+  return matches;
+}
+//# sourceMappingURL=matchStringWithRegex.js.map

package/build/languageProcessing/helpers/regex/matchStringWithRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchStringWithRegex.js","names":["_default","text","regex","matches","match"],"sources":["../../../../src/languageProcessing/helpers/regex/matchStringWithRegex.js"],"sourcesContent":["/** @module stringProcessing/matchStringWithRegex */\n\n/**\n * Checks a string with a regex, return all matches found with that regex.\n *\n * @param {String} \t\ttext        The text to match.\n * @param {object} \t\tregex  \t\tA compiled regular expression.\n *\n * @returns {Array} Array with matches, empty array if no matches found.\n */\nexport default function( text, regex ) {\n\tlet matches = text.match( regex );\n\n\tif ( matches === null ) {\n\t\tmatches = [];\n\t}\n\n\treturn matches;\n}\n"],"mappings":";;;;;;AAAA;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAA,SAAUC,IAAI,EAAEC,KAAK,EAAG;EACtC,IAAIC,OAAO,GAAGF,IAAI,CAACG,KAAK,CAAEF,KAAM,CAAC;EAEjC,IAAKC,OAAO,KAAK,IAAI,EAAG;IACvBA,OAAO,GAAG,EAAE;EACb;EAEA,OAAOA,OAAO;AACf","ignoreList":[]}

package/build/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js

@@ -0,0 +1,21 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = searchAndReplaceWithOneRegex;
+/**
+ * Checks whether a given regex matches a given word and if so, performs the replacement.
+ *
+ * @param {string}      word 					The word that may need to be modified.
+ * @param {string[]}    regexAndReplacement     A regex and the required replacement.
+ *
+ * @returns {?string} The modified stem or undefined if no match was found.
+ */
+function searchAndReplaceWithOneRegex(word, regexAndReplacement) {
+  if (word.search(new RegExp(regexAndReplacement[0])) !== -1) {
+    word = word.replace(new RegExp(regexAndReplacement[0]), regexAndReplacement[1]);
+    return word;
+  }
+}
+//# sourceMappingURL=searchAndReplaceWithOneRegex.js.map

package/build/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"searchAndReplaceWithOneRegex.js","names":["searchAndReplaceWithOneRegex","word","regexAndReplacement","search","RegExp","replace"],"sources":["../../../../src/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js"],"sourcesContent":["/**\n * Checks whether a given regex matches a given word and if so, performs the replacement.\n *\n * @param {string}      word \t\t\t\t\tThe word that may need to be modified.\n * @param {string[]}    regexAndReplacement     A regex and the required replacement.\n *\n * @returns {?string} The modified stem or undefined if no match was found.\n */\nexport default function searchAndReplaceWithOneRegex( word, regexAndReplacement ) {\n\tif ( word.search( new RegExp( regexAndReplacement[ 0 ] ) ) !== -1 ) {\n\t\tword = word.replace( new RegExp( regexAndReplacement[ 0 ] ), regexAndReplacement[ 1 ] );\n\t\treturn word;\n\t}\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASA,4BAA4BA,CAAEC,IAAI,EAAEC,mBAAmB,EAAG;EACjF,IAAKD,IAAI,CAACE,MAAM,CAAE,IAAIC,MAAM,CAAEF,mBAAmB,CAAE,CAAC,CAAG,CAAE,CAAC,KAAK,CAAC,CAAC,EAAG;IACnED,IAAI,GAAGA,IAAI,CAACI,OAAO,CAAE,IAAID,MAAM,CAAEF,mBAAmB,CAAE,CAAC,CAAG,CAAC,EAAEA,mBAAmB,CAAE,CAAC,CAAG,CAAC;IACvF,OAAOD,IAAI;EACZ;AACD","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/doubleQuotes.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+/**
+ * Returns an array with double quotes.
+ *
+ * Japanese-specific quotes:
+ * U300c - left corner bracket.
+ * U300d - right corner bracket.
+ * U300e - left white corner bracket.
+ * U300f - right white corner bracket.
+ *
+ * @returns {string[]} The array with double quotes..
+ */
+var _default = exports.default = ["“", "”", "〝", "〞", "〟", "‟", "„", "\"", "\u300c", "\u300d", "\u300e", "\u300f"];
+//# sourceMappingURL=doubleQuotes.js.map

package/build/languageProcessing/helpers/sanitize/doubleQuotes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"doubleQuotes.js","names":["_default","exports","default"],"sources":["../../../../src/languageProcessing/helpers/sanitize/doubleQuotes.js"],"sourcesContent":["/**\n * Returns an array with double quotes.\n *\n * Japanese-specific quotes:\n * U300c - left corner bracket.\n * U300d - right corner bracket.\n * U300e - left white corner bracket.\n * U300f - right white corner bracket.\n *\n * @returns {string[]} The array with double quotes..\n */\nexport default [ \"“\", \"”\", \"〝\", \"〞\", \"〟\", \"‟\", \"„\", \"\\\"\", \"\\u300c\", \"\\u300d\", \"\\u300e\", \"\\u300f\" ];\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAVA,IAAAA,QAAA,GAAAC,OAAA,CAAAC,OAAA,GAWe,CAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAE","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js

@@ -0,0 +1,131 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.filterShortcodesFromHTML = exports.default = exports.createShortcodeTagsRegex = void 0;
+/**
+ * Creates a regex to filter shortcodes from HTML.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {RegExp} The regex to recognize the shortcodes.
+ */
+const createShortcodeTagsRegex = shortcodeTags => {
+  const shortcodeTagsRegexString = `\\[\\/?(${shortcodeTags.join("|")})[^\\]]*\\]`;
+  return new RegExp(shortcodeTagsRegexString, "g");
+};
+
+/**
+ * Filters shortcodes from HTML string.
+ * @param {string} html The HTML to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {string} The filtered HTML.
+ */
+exports.createShortcodeTagsRegex = createShortcodeTagsRegex;
+const filterShortcodesFromHTML = (html, shortcodeTags) => {
+  if (!shortcodeTags || shortcodeTags.length === 0) {
+    return html;
+  }
+  const shortcodeTagsRegex = createShortcodeTagsRegex(shortcodeTags);
+  return html.replace(shortcodeTagsRegex, "");
+};
+
+/**
+ * Checks if a token is part of an opening shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of an opening shortcode.
+ */
+exports.filterShortcodesFromHTML = filterShortcodesFromHTML;
+const tokenIsPartOfOpeningShortcode = (tokens, index) => {
+  return tokens[index - 1] && tokens[index - 1].text === "[";
+};
+
+/**
+ * Checks if a token is part of a closing shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of a closing shortcode.
+ */
+const tokenIsPartOfClosingShortcode = (tokens, index) => {
+  return tokens[index - 1] && tokens[index - 1].text === "/" && tokens[index - 2] && tokens[index - 2].text === "[";
+};
+
+/**
+ * Checks if a token is part of a shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @param {string[]} shortcodeTags An array of active shortcode tags.
+ * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).
+ * @returns {boolean} Whether the token is part of a shortcode.
+ */
+const tokenIsShortcode = (tokens, index, shortcodeTags, encounteredClosingBracket) => {
+  return encounteredClosingBracket && shortcodeTags.includes(tokens[index].text) && (tokenIsPartOfOpeningShortcode(tokens, index) || tokenIsPartOfClosingShortcode(tokens, index));
+};
+
+/**
+ * Filters shortcodes from a sentence.
+ * @param {Sentence} sentence The sentence to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ *
+ */
+const filterShortcodesFromSentence = (sentence, shortcodeTags, shortcodeTagsRegex) => {
+  const tokens = sentence.tokens;
+  let encounteredClosingBracket = false;
+  // traverse through tokens backwards
+  for (let i = tokens.length - 1; i >= 0; i--) {
+    if (tokens[i].text === "]") {
+      encounteredClosingBracket = true;
+    }
+    if (tokenIsShortcode(tokens, i, shortcodeTags, encounteredClosingBracket)) {
+      while (tokens[i].text !== "]") {
+        tokens.splice(i, 1);
+      }
+      tokens.splice(i, 1);
+      encounteredClosingBracket = false;
+      while (tokens[i - 1] && "[/".includes(tokens[i - 1].text)) {
+        tokens.splice(i - 1, 1);
+        i--;
+      }
+    }
+  }
+  sentence.tokens = tokens;
+  sentence.text = sentence.text.replace(shortcodeTagsRegex, "");
+};
+
+/**
+ * A recursive function that filters shortcodes from sentences.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ */
+const filterShortcodesFromSentences = (tree, shortcodeTags, shortcodeTagsRegex) => {
+  if (tree.sentences) {
+    tree.sentences.forEach(sentence => {
+      filterShortcodesFromSentence(sentence, shortcodeTags, shortcodeTagsRegex);
+    });
+  }
+  if (tree.childNodes) {
+    tree.childNodes.forEach(childNode => {
+      filterShortcodesFromSentences(childNode, shortcodeTags, shortcodeTagsRegex);
+    });
+  }
+};
+
+/**
+ * Filters shortcodes from the tree.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {void}
+ */
+const filterShortcodesFromTree = (tree, shortcodeTags) => {
+  if (!shortcodeTags || shortcodeTags.length === 0) {
+    return;
+  }
+  const shortcodeTagsRegex = createShortcodeTagsRegex(shortcodeTags);
+  filterShortcodesFromSentences(tree, shortcodeTags, shortcodeTagsRegex);
+};
+var _default = exports.default = filterShortcodesFromTree;
+//# sourceMappingURL=filterShortcodesFromTree.js.map

package/build/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterShortcodesFromTree.js","names":["createShortcodeTagsRegex","shortcodeTags","shortcodeTagsRegexString","join","RegExp","exports","filterShortcodesFromHTML","html","length","shortcodeTagsRegex","replace","tokenIsPartOfOpeningShortcode","tokens","index","text","tokenIsPartOfClosingShortcode","tokenIsShortcode","encounteredClosingBracket","includes","filterShortcodesFromSentence","sentence","i","splice","filterShortcodesFromSentences","tree","sentences","forEach","childNodes","childNode","filterShortcodesFromTree","_default","default"],"sources":["../../../../src/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js"],"sourcesContent":["\n/**\n * Creates a regex to filter shortcodes from HTML.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {RegExp} The regex to recognize the shortcodes.\n */\nexport const createShortcodeTagsRegex = shortcodeTags => {\n\tconst shortcodeTagsRegexString = `\\\\[\\\\/?(${ shortcodeTags.join( \"|\" ) })[^\\\\]]*\\\\]`;\n\treturn new RegExp( shortcodeTagsRegexString, \"g\" );\n};\n\n/**\n * Filters shortcodes from HTML string.\n * @param {string} html The HTML to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {string} The filtered HTML.\n */\nexport const filterShortcodesFromHTML = ( html, shortcodeTags ) => {\n\tif ( ! shortcodeTags || shortcodeTags.length === 0 ) {\n\t\treturn html;\n\t}\n\tconst shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );\n\treturn html.replace( shortcodeTagsRegex, \"\" );\n};\n\n/**\n * Checks if a token is part of an opening shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @returns {boolean} Whether the token is part of an opening shortcode.\n */\nconst tokenIsPartOfOpeningShortcode = ( tokens, index ) => {\n\treturn tokens[ index - 1 ] && tokens[ index - 1 ].text === \"[\";\n};\n\n/**\n * Checks if a token is part of a closing shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @returns {boolean} Whether the token is part of a closing shortcode.\n */\nconst tokenIsPartOfClosingShortcode = ( tokens, index ) => {\n\treturn  tokens[ index - 1 ] && tokens[ index - 1 ].text === \"/\"  && tokens[ index - 2 ]  && tokens[ index - 2 ].text === \"[\";\n};\n\n/**\n * Checks if a token is part of a shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @param {string[]} shortcodeTags An array of active shortcode tags.\n * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).\n * @returns {boolean} Whether the token is part of a shortcode.\n */\nconst tokenIsShortcode = ( tokens, index, shortcodeTags, encounteredClosingBracket ) => {\n\treturn encounteredClosingBracket && shortcodeTags.includes( tokens[ index ].text ) &&\n\t\t( tokenIsPartOfOpeningShortcode( tokens, index ) || tokenIsPartOfClosingShortcode( tokens, index )  );\n};\n\n/**\n * Filters shortcodes from a sentence.\n * @param {Sentence} sentence The sentence to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.\n * @returns {void}\n *\n */\nconst filterShortcodesFromSentence = ( sentence, shortcodeTags, shortcodeTagsRegex ) => {\n\tconst tokens = sentence.tokens;\n\n\tlet encounteredClosingBracket = false;\n\t// traverse through tokens backwards\n\tfor ( let i = tokens.length - 1; i >= 0; i-- ) {\n\t\tif ( tokens[ i ].text === \"]\" ) {\n\t\t\tencounteredClosingBracket = true;\n\t\t}\n\n\t\tif ( tokenIsShortcode( tokens, i, shortcodeTags, encounteredClosingBracket ) ) {\n\t\t\twhile ( tokens[ i ].text !== \"]\" ) {\n\t\t\t\ttokens.splice( i, 1 );\n\t\t\t}\n\t\t\ttokens.splice( i, 1 );\n\t\t\tencounteredClosingBracket = false;\n\n\t\t\twhile ( tokens[ i - 1 ] && \"[/\".includes( tokens[ i - 1 ].text ) ) {\n\t\t\t\ttokens.splice( i - 1, 1 );\n\t\t\t\ti--;\n\t\t\t}\n\t\t}\n\t}\n\tsentence.tokens = tokens;\n\n\tsentence.text = sentence.text.replace( shortcodeTagsRegex, \"\" );\n};\n\n/**\n * A recursive function that filters shortcodes from sentences.\n * @param {Node} tree The tree to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.\n * @returns {void}\n */\nconst filterShortcodesFromSentences = ( tree, shortcodeTags, shortcodeTagsRegex ) => {\n\tif ( tree.sentences ) {\n\t\ttree.sentences.forEach( sentence => {\n\t\t\tfilterShortcodesFromSentence( sentence, shortcodeTags, shortcodeTagsRegex );\n\t\t} );\n\t}\n\n\tif ( tree.childNodes ) {\n\t\ttree.childNodes.forEach( childNode => {\n\t\t\tfilterShortcodesFromSentences( childNode, shortcodeTags, shortcodeTagsRegex );\n\t\t} );\n\t}\n};\n\n/**\n * Filters shortcodes from the tree.\n * @param {Node} tree The tree to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {void}\n */\nconst filterShortcodesFromTree = ( tree, shortcodeTags ) => {\n\tif ( ! shortcodeTags || shortcodeTags.length === 0 ) {\n\t\treturn;\n\t}\n\tconst shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );\n\n\tfilterShortcodesFromSentences( tree, shortcodeTags, shortcodeTagsRegex );\n};\n\nexport default filterShortcodesFromTree;\n"],"mappings":";;;;;;AACA;AACA;AACA;AACA;AACA;AACO,MAAMA,wBAAwB,GAAGC,aAAa,IAAI;EACxD,MAAMC,wBAAwB,GAAG,WAAYD,aAAa,CAACE,IAAI,CAAE,GAAI,CAAC,aAAc;EACpF,OAAO,IAAIC,MAAM,CAAEF,wBAAwB,EAAE,GAAI,CAAC;AACnD,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AALAG,OAAA,CAAAL,wBAAA,GAAAA,wBAAA;AAMO,MAAMM,wBAAwB,GAAGA,CAAEC,IAAI,EAAEN,aAAa,KAAM;EAClE,IAAK,CAAEA,aAAa,IAAIA,aAAa,CAACO,MAAM,KAAK,CAAC,EAAG;IACpD,OAAOD,IAAI;EACZ;EACA,MAAME,kBAAkB,GAAGT,wBAAwB,CAAEC,aAAc,CAAC;EACpE,OAAOM,IAAI,CAACG,OAAO,CAAED,kBAAkB,EAAE,EAAG,CAAC;AAC9C,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AALAJ,OAAA,CAAAC,wBAAA,GAAAA,wBAAA;AAMA,MAAMK,6BAA6B,GAAGA,CAAEC,MAAM,EAAEC,KAAK,KAAM;EAC1D,OAAOD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAID,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG;AAC/D,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,6BAA6B,GAAGA,CAAEH,MAAM,EAAEC,KAAK,KAAM;EAC1D,OAAQD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAID,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG,IAAKF,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAKD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG;AAC7H,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAME,gBAAgB,GAAGA,CAAEJ,MAAM,EAAEC,KAAK,EAAEZ,aAAa,EAAEgB,yBAAyB,KAAM;EACvF,OAAOA,yBAAyB,IAAIhB,aAAa,CAACiB,QAAQ,CAAEN,MAAM,CAAEC,KAAK,CAAE,CAACC,IAAK,CAAC,KAC/EH,6BAA6B,CAAEC,MAAM,EAAEC,KAAM,CAAC,IAAIE,6BAA6B,CAAEH,MAAM,EAAEC,KAAM,CAAC,CAAG;AACvG,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMM,4BAA4B,GAAGA,CAAEC,QAAQ,EAAEnB,aAAa,EAAEQ,kBAAkB,KAAM;EACvF,MAAMG,MAAM,GAAGQ,QAAQ,CAACR,MAAM;EAE9B,IAAIK,yBAAyB,GAAG,KAAK;EACrC;EACA,KAAM,IAAII,CAAC,GAAGT,MAAM,CAACJ,MAAM,GAAG,CAAC,EAAEa,CAAC,IAAI,CAAC,EAAEA,CAAC,EAAE,EAAG;IAC9C,IAAKT,MAAM,CAAES,CAAC,CAAE,CAACP,IAAI,KAAK,GAAG,EAAG;MAC/BG,yBAAyB,GAAG,IAAI;IACjC;IAEA,IAAKD,gBAAgB,CAAEJ,MAAM,EAAES,CAAC,EAAEpB,aAAa,EAAEgB,yBAA0B,CAAC,EAAG;MAC9E,OAAQL,MAAM,CAAES,CAAC,CAAE,CAACP,IAAI,KAAK,GAAG,EAAG;QAClCF,MAAM,CAACU,MAAM,CAAED,CAAC,EAAE,CAAE,CAAC;MACtB;MACAT,MAAM,CAACU,MAAM,CAAED,CAAC,EAAE,CAAE,CAAC;MACrBJ,yBAAyB,GAAG,KAAK;MAEjC,OAAQL,MAAM,CAAES,CAAC,GAAG,CAAC,CAAE,IAAI,IAAI,CAACH,QAAQ,CAAEN,MAAM,CAAES,CAAC,GAAG,CAAC,CAAE,CAACP,IAAK,CAAC,EAAG;QAClEF,MAAM,CAACU,MAAM,CAAED,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACzBA,CAAC,EAAE;MACJ;IACD;EACD;EACAD,QAAQ,CAACR,MAAM,GAAGA,MAAM;EAExBQ,QAAQ,CAACN,IAAI,GAAGM,QAAQ,CAACN,IAAI,CAACJ,OAAO,CAAED,kBAAkB,EAAE,EAAG,CAAC;AAChE,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMc,6BAA6B,GAAGA,CAAEC,IAAI,EAAEvB,aAAa,EAAEQ,kBAAkB,KAAM;EACpF,IAAKe,IAAI,CAACC,SAAS,EAAG;IACrBD,IAAI,CAACC,SAAS,CAACC,OAAO,CAAEN,QAAQ,IAAI;MACnCD,4BAA4B,CAAEC,QAAQ,EAAEnB,aAAa,EAAEQ,kBAAmB,CAAC;IAC5E,CAAE,CAAC;EACJ;EAEA,IAAKe,IAAI,CAACG,UAAU,EAAG;IACtBH,IAAI,CAACG,UAAU,CAACD,OAAO,CAAEE,SAAS,IAAI;MACrCL,6BAA6B,CAAEK,SAAS,EAAE3B,aAAa,EAAEQ,kBAAmB,CAAC;IAC9E,CAAE,CAAC;EACJ;AACD,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA,MAAMoB,wBAAwB,GAAGA,CAAEL,IAAI,EAAEvB,aAAa,KAAM;EAC3D,IAAK,CAAEA,aAAa,IAAIA,aAAa,CAACO,MAAM,KAAK,CAAC,EAAG;IACpD;EACD;EACA,MAAMC,kBAAkB,GAAGT,wBAAwB,CAAEC,aAAc,CAAC;EAEpEsB,6BAA6B,CAAEC,IAAI,EAAEvB,aAAa,EAAEQ,kBAAmB,CAAC;AACzE,CAAC;AAAC,IAAAqB,QAAA,GAAAzB,OAAA,CAAA0B,OAAA,GAEaF,wBAAwB","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/mergeListItems.js

@@ -0,0 +1,30 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.mergeListItems = mergeListItems;
+/**
+ * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.
+ * That way, lists with single words don't result in a skewed keyphrase distribution result.
+ *
+ * @param {string}  text    The text in which to remove the list structures.
+ *
+ * @returns {string} The text with all list structures removed.
+ */
+function mergeListItems(text) {
+  const listTags = /<\/?(o|ul)(?:[^>]+)?>/g;
+  const listItemTags = /\s?<\/?li(?:[^>]+)?>\s?/g;
+
+  /*
+   * Steps:
+   * 1) Remove all ul and ol tags.
+   * 2) Replace all li tags with spaces to avoid list items being stuck together.
+   * 3) Replace multiple instances of whitespace with a single instance.
+   */
+  text = text.replace(listTags, "");
+  text = text.replace(listItemTags, " ");
+  text = text.replace(/\s+/g, " ");
+  return text;
+}
+//# sourceMappingURL=mergeListItems.js.map

package/build/languageProcessing/helpers/sanitize/mergeListItems.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mergeListItems.js","names":["mergeListItems","text","listTags","listItemTags","replace"],"sources":["../../../../src/languageProcessing/helpers/sanitize/mergeListItems.js"],"sourcesContent":["/**\n * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.\n * That way, lists with single words don't result in a skewed keyphrase distribution result.\n *\n * @param {string}  text    The text in which to remove the list structures.\n *\n * @returns {string} The text with all list structures removed.\n */\nexport function mergeListItems( text ) {\n\tconst listTags = /<\\/?(o|ul)(?:[^>]+)?>/g;\n\tconst listItemTags = /\\s?<\\/?li(?:[^>]+)?>\\s?/g;\n\n\t/*\n\t * Steps:\n\t * 1) Remove all ul and ol tags.\n\t * 2) Replace all li tags with spaces to avoid list items being stuck together.\n\t * 3) Replace multiple instances of whitespace with a single instance.\n\t */\n\ttext = text.replace( listTags, \"\" );\n\ttext = text.replace( listItemTags, \" \" );\n\ttext = text.replace( /\\s+/g, \" \" );\n\n\treturn text;\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASA,cAAcA,CAAEC,IAAI,EAAG;EACtC,MAAMC,QAAQ,GAAG,wBAAwB;EACzC,MAAMC,YAAY,GAAG,0BAA0B;;EAE/C;AACD;AACA;AACA;AACA;AACA;EACCF,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAEF,QAAQ,EAAE,EAAG,CAAC;EACnCD,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAED,YAAY,EAAE,GAAI,CAAC;EACxCF,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAE,MAAM,EAAE,GAAI,CAAC;EAElC,OAAOH,IAAI;AACZ","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/parseSynonyms.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _stripSpaces = _interopRequireDefault(require("./stripSpaces.js"));
+var _removePunctuationExceptQuotes = _interopRequireDefault(require("./removePunctuationExceptQuotes.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/parseSynonyms */
+
+/**
+ * Parses synonyms from a comma-separated string into an array.
+ *
+ * @param {String} synonyms The text to match
+ *
+ * @returns {Array} An array with all synonyms.
+ */
+function _default(synonyms) {
+  let synonymsSplit = synonyms.split(",");
+  synonymsSplit = synonymsSplit.map(synonym => (0, _removePunctuationExceptQuotes.default)((0, _stripSpaces.default)(synonym))).filter(synonym => synonym);
+  return synonymsSplit;
+}
+//# sourceMappingURL=parseSynonyms.js.map

package/build/languageProcessing/helpers/sanitize/parseSynonyms.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseSynonyms.js","names":["_stripSpaces","_interopRequireDefault","require","_removePunctuationExceptQuotes","e","__esModule","default","_default","synonyms","synonymsSplit","split","map","synonym","removePunctuationExceptQuotes","stripSpaces","filter"],"sources":["../../../../src/languageProcessing/helpers/sanitize/parseSynonyms.js"],"sourcesContent":["/** @module stringProcessing/parseSynonyms */\n\nimport stripSpaces from \"./stripSpaces.js\";\nimport removePunctuationExceptQuotes from \"./removePunctuationExceptQuotes.js\";\n\n/**\n * Parses synonyms from a comma-separated string into an array.\n *\n * @param {String} synonyms The text to match\n *\n * @returns {Array} An array with all synonyms.\n */\nexport default function( synonyms ) {\n\tlet synonymsSplit = synonyms.split( \",\" );\n\n\tsynonymsSplit = synonymsSplit.map( synonym =>\n\t\tremovePunctuationExceptQuotes( stripSpaces( synonym ) ) ).filter( synonym => synonym );\n\n\treturn synonymsSplit;\n}\n"],"mappings":";;;;;;AAEA,IAAAA,YAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,8BAAA,GAAAF,sBAAA,CAAAC,OAAA;AAA+E,SAAAD,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAH/E;;AAKA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,QAAQ,EAAG;EACnC,IAAIC,aAAa,GAAGD,QAAQ,CAACE,KAAK,CAAE,GAAI,CAAC;EAEzCD,aAAa,GAAGA,aAAa,CAACE,GAAG,CAAEC,OAAO,IACzC,IAAAC,sCAA6B,EAAE,IAAAC,oBAAW,EAAEF,OAAQ,CAAE,CAAE,CAAC,CAACG,MAAM,CAAEH,OAAO,IAAIA,OAAQ,CAAC;EAEvF,OAAOH,aAAa;AACrB","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/quotes.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = exports.SINGLE_QUOTES_REGEX = exports.SINGLE_QUOTES_ARRAY = void 0;
+exports.normalize = normalizeQuotes;
+exports.normalizeDouble = normalizeDoubleQuotes;
+exports.normalizeSingle = normalizeSingleQuotes;
+const SINGLE_QUOTES_ARRAY = exports.SINGLE_QUOTES_ARRAY = ["'", "‘", "’", "‛", "`", "‹", "›"];
+const SINGLE_QUOTES_REGEX = exports.SINGLE_QUOTES_REGEX = new RegExp("[" + SINGLE_QUOTES_ARRAY.join("") + "]", "g");
+
+/**
+ * Normalizes single quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeSingleQuotes(text) {
+  return text.replace(SINGLE_QUOTES_REGEX, "'");
+}
+
+/**
+ * Normalizes double quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeDoubleQuotes(text) {
+  return text.replace(/[“”〝〞〟‟„『』«»]/g, "\"");
+}
+
+/**
+ * Normalizes quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeQuotes(text) {
+  return normalizeDoubleQuotes(normalizeSingleQuotes(text));
+}
+var _default = exports.default = {
+  normalizeSingle: normalizeSingleQuotes,
+  normalizeDouble: normalizeDoubleQuotes,
+  normalize: normalizeQuotes
+};
+//# sourceMappingURL=quotes.js.map

package/build/languageProcessing/helpers/sanitize/quotes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quotes.js","names":["SINGLE_QUOTES_ARRAY","exports","SINGLE_QUOTES_REGEX","RegExp","join","normalizeSingleQuotes","text","replace","normalizeDoubleQuotes","normalizeQuotes","_default","default","normalizeSingle","normalizeDouble","normalize"],"sources":["../../../../src/languageProcessing/helpers/sanitize/quotes.js"],"sourcesContent":["const SINGLE_QUOTES_ARRAY = [ \"'\", \"‘\", \"’\", \"‛\", \"`\", \"‹\", \"›\" ];\nconst SINGLE_QUOTES_REGEX = new RegExp( \"[\" + SINGLE_QUOTES_ARRAY.join( \"\" ) + \"]\", \"g\" );\n\n/**\n * Normalizes single quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeSingleQuotes( text ) {\n\treturn text.replace( SINGLE_QUOTES_REGEX, \"'\" );\n}\n\n/**\n * Normalizes double quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeDoubleQuotes( text ) {\n\treturn text.replace( /[“”〝〞〟‟„『』«»]/g, \"\\\"\" );\n}\n\n/**\n * Normalizes quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeQuotes( text ) {\n\treturn normalizeDoubleQuotes( normalizeSingleQuotes( text ) );\n}\n\nexport {\n\tnormalizeSingleQuotes as normalizeSingle,\n\tnormalizeDoubleQuotes as normalizeDouble,\n\tnormalizeQuotes as normalize,\n\tSINGLE_QUOTES_REGEX,\n\tSINGLE_QUOTES_ARRAY,\n};\n\nexport default {\n\tnormalizeSingle: normalizeSingleQuotes,\n\tnormalizeDouble: normalizeDoubleQuotes,\n\tnormalize: normalizeQuotes,\n};\n"],"mappings":";;;;;;;;;AAAA,MAAMA,mBAAmB,GAAAC,OAAA,CAAAD,mBAAA,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAE;AACjE,MAAME,mBAAmB,GAAAD,OAAA,CAAAC,mBAAA,GAAG,IAAIC,MAAM,CAAE,GAAG,GAAGH,mBAAmB,CAACI,IAAI,CAAE,EAAG,CAAC,GAAG,GAAG,EAAE,GAAI,CAAC;;AAEzF;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,qBAAqBA,CAAEC,IAAI,EAAG;EACtC,OAAOA,IAAI,CAACC,OAAO,CAAEL,mBAAmB,EAAE,GAAI,CAAC;AAChD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,SAASM,qBAAqBA,CAAEF,IAAI,EAAG;EACtC,OAAOA,IAAI,CAACC,OAAO,CAAE,gBAAgB,EAAE,IAAK,CAAC;AAC9C;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,SAASE,eAAeA,CAAEH,IAAI,EAAG;EAChC,OAAOE,qBAAqB,CAAEH,qBAAqB,CAAEC,IAAK,CAAE,CAAC;AAC9D;AAAC,IAAAI,QAAA,GAAAT,OAAA,CAAAU,OAAA,GAUc;EACdC,eAAe,EAAEP,qBAAqB;EACtCQ,eAAe,EAAEL,qBAAqB;EACtCM,SAAS,EAAEL;AACZ,CAAC","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/removeEmailAddresses.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+const emailRegex = new RegExp("[^\\s@]+@[^\\s@]+\\.[^\\s@]+", "igm");
+
+/**
+ * Removes email addresses from a text.
+ *
+ * @param {string} text The text to remove emails from.
+ *
+ * @returns {string} The text without email addresses.
+ */
+function _default(text) {
+  return text.replace(emailRegex, "");
+}
+//# sourceMappingURL=removeEmailAddresses.js.map

package/build/languageProcessing/helpers/sanitize/removeEmailAddresses.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removeEmailAddresses.js","names":["emailRegex","RegExp","_default","text","replace"],"sources":["../../../../src/languageProcessing/helpers/sanitize/removeEmailAddresses.js"],"sourcesContent":["const emailRegex = new RegExp( \"[^\\\\s@]+@[^\\\\s@]+\\\\.[^\\\\s@]+\", \"igm\" );\n\n/**\n * Removes email addresses from a text.\n *\n * @param {string} text The text to remove emails from.\n *\n * @returns {string} The text without email addresses.\n */\nexport default function( text ) {\n\treturn text.replace( emailRegex, \"\" );\n}\n"],"mappings":";;;;;;AAAA,MAAMA,UAAU,GAAG,IAAIC,MAAM,CAAE,8BAA8B,EAAE,KAAM,CAAC;;AAEtE;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAAUC,IAAI,EAAG;EAC/B,OAAOA,IAAI,CAACC,OAAO,CAAEJ,UAAU,EAAE,EAAG,CAAC;AACtC","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/removePunctuation.js

@@ -0,0 +1,64 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+exports.punctuationRegexString = exports.punctuationRegexStart = exports.punctuationRegexEnd = exports.punctuationList = void 0;
+var _unifyWhitespace = require("./unifyWhitespace");
+/**
+ * String containing all the characters that we consider punctuation.
+ * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
+ * @type {string}
+ */
+const punctuationRegexString = exports.punctuationRegexString = "\\–\\-\\(\\)_\\[\\]’‘“”〝〞〟‟„\"'.?!:;,¿¡«»‹›\u2014\u00d7\u002b\u0026\u06d4\u061f\u060C\u061B\u3002\uff61" + "\uff01\u203c\uff1f\u2047\u2049\u2048\u2025\u2026\u30fb\u30fc\u3001\u3003\u3004\u3006\u3007\u3008\u3009\u300a\u300b\u300c\u300d\u300e" + "\u300f\u3010\u3011\u3012\u3013\u3014\u3015\u3016\u3017\u3018\u3019\u301a\u301b\u301c\u301d\u301e\u301f\u3020\u3036\u303c\u303d\uff5b" + "\uff5d\uff5c\uff5e\uff5f\uff60\uff62\uff63\uff64\uff3b\uff3d\uff65\uffe5\uff04\uff05\uff20\uff06\uff07\uff08\uff09\uff0a\uff0f\uff1a" + "\uff1b\uff1c\uff1e\uff3c\\<>";
+
+/**
+ * Array containing all the characters that we consider punctuation.
+ * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
+ * @type {string[]}
+ */
+const punctuationList = exports.punctuationList = punctuationRegexString.split("");
+const punctuationRegexStart = exports.punctuationRegexStart = new RegExp("^[" + punctuationRegexString + "]+");
+const punctuationRegexEnd = exports.punctuationRegexEnd = new RegExp("[" + punctuationRegexString + "]+$");
+
+/*
+ * \u2014 - em-dash
+ * \u00d7 - multiplication sign
+ * \u002b - plus sign
+ * \u0026 - ampersand
+ * \u06d4 - Urdu full stop
+ * \u061f - Arabic question mark
+ * \u060C - Arabic comma
+ * \u061B - Arabic semicolon
+ */
+
+/**
+ * Replaces punctuation characters from the given text string.
+ *
+ * @param {String} text The text to remove the punctuation characters for.
+ *
+ * @returns {String} The sanitized text.
+ */
+function _default(text) {
+  // Unify whitespaces and non-breaking spaces.
+  text = (0, _unifyWhitespace.unifyNonBreakingSpace)(text);
+
+  // Remove &amp from the string. In some editors (a.o. Block and Elementor) the ampersand (&) is transformed into &amp.
+  // If it is not removed, then it is returned as "amp" and counted as a word in assessments downstream.
+  text = text.replace("\u0026amp", "");
+
+  /*
+   * Remove backslash from the beginning and end of a word/text.
+   * When a string such as `This is a \"calico\" cat` enters the Paper,
+   * the Paper adds two extra backslash in front of the original backslash.
+   * After the text is split into words, we also need to remove those backslashes from the word.
+   * Otherwise, it will be problematic when word boundary regex is added to the word.
+   */
+  const backslashRegex = new RegExp("(\\\\)", "g");
+  text = text.replace(backslashRegex, "");
+  text = text.replace(punctuationRegexStart, "");
+  text = text.replace(punctuationRegexEnd, "");
+  return text;
+}
+//# sourceMappingURL=removePunctuation.js.map

package/build/languageProcessing/helpers/sanitize/removePunctuation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removePunctuation.js","names":["_unifyWhitespace","require","punctuationRegexString","exports","punctuationList","split","punctuationRegexStart","RegExp","punctuationRegexEnd","_default","text","unifyNonBreakingSpace","replace","backslashRegex"],"sources":["../../../../src/languageProcessing/helpers/sanitize/removePunctuation.js"],"sourcesContent":["import { unifyNonBreakingSpace } from \"./unifyWhitespace\";\n\n/**\n * String containing all the characters that we consider punctuation.\n * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.\n * @type {string}\n */\nexport const punctuationRegexString = \"\\\\–\\\\-\\\\(\\\\)_\\\\[\\\\]’‘“”〝〞〟‟„\\\"'.?!:;,¿¡«»‹›\\u2014\\u00d7\\u002b\\u0026\\u06d4\\u061f\\u060C\\u061B\\u3002\\uff61\" +\n\"\\uff01\\u203c\\uff1f\\u2047\\u2049\\u2048\\u2025\\u2026\\u30fb\\u30fc\\u3001\\u3003\\u3004\\u3006\\u3007\\u3008\\u3009\\u300a\\u300b\\u300c\\u300d\\u300e\" +\n\"\\u300f\\u3010\\u3011\\u3012\\u3013\\u3014\\u3015\\u3016\\u3017\\u3018\\u3019\\u301a\\u301b\\u301c\\u301d\\u301e\\u301f\\u3020\\u3036\\u303c\\u303d\\uff5b\" +\n\"\\uff5d\\uff5c\\uff5e\\uff5f\\uff60\\uff62\\uff63\\uff64\\uff3b\\uff3d\\uff65\\uffe5\\uff04\\uff05\\uff20\\uff06\\uff07\\uff08\\uff09\\uff0a\\uff0f\\uff1a\" +\n\"\\uff1b\\uff1c\\uff1e\\uff3c\\\\<>\";\n\n/**\n * Array containing all the characters that we consider punctuation.\n * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.\n * @type {string[]}\n */\nexport const punctuationList = punctuationRegexString.split( \"\" );\n\nexport const punctuationRegexStart = new RegExp( \"^[\" + punctuationRegexString + \"]+\" );\nexport const punctuationRegexEnd = new RegExp( \"[\" + punctuationRegexString +  \"]+$\" );\n\n/*\n * \\u2014 - em-dash\n * \\u00d7 - multiplication sign\n * \\u002b - plus sign\n * \\u0026 - ampersand\n * \\u06d4 - Urdu full stop\n * \\u061f - Arabic question mark\n * \\u060C - Arabic comma\n * \\u061B - Arabic semicolon\n */\n\n/**\n * Replaces punctuation characters from the given text string.\n *\n * @param {String} text The text to remove the punctuation characters for.\n *\n * @returns {String} The sanitized text.\n */\nexport default function( text ) {\n\t// Unify whitespaces and non-breaking spaces.\n\ttext = unifyNonBreakingSpace( text );\n\n\t// Remove &amp from the string. In some editors (a.o. Block and Elementor) the ampersand (&) is transformed into &amp.\n\t// If it is not removed, then it is returned as \"amp\" and counted as a word in assessments downstream.\n\ttext = text.replace( \"\\u0026amp\", \"\" );\n\n\t/*\n\t * Remove backslash from the beginning and end of a word/text.\n\t * When a string such as `This is a \\\"calico\\\" cat` enters the Paper,\n\t * the Paper adds two extra backslash in front of the original backslash.\n\t * After the text is split into words, we also need to remove those backslashes from the word.\n\t * Otherwise, it will be problematic when word boundary regex is added to the word.\n\t */\n\tconst backslashRegex = new RegExp( \"(\\\\\\\\)\", \"g\" );\n\n\ttext = text.replace( backslashRegex, \"\" );\n\ttext = text.replace( punctuationRegexStart, \"\" );\n\ttext = text.replace( punctuationRegexEnd, \"\" );\n\n\treturn text;\n}\n"],"mappings":";;;;;;;AAAA,IAAAA,gBAAA,GAAAC,OAAA;AAEA;AACA;AACA;AACA;AACA;AACO,MAAMC,sBAAsB,GAAAC,OAAA,CAAAD,sBAAA,GAAG,yGAAyG,GAC/I,sIAAsI,GACtI,sIAAsI,GACtI,sIAAsI,GACtI,8BAA8B;;AAE9B;AACA;AACA;AACA;AACA;AACO,MAAME,eAAe,GAAAD,OAAA,CAAAC,eAAA,GAAGF,sBAAsB,CAACG,KAAK,CAAE,EAAG,CAAC;AAE1D,MAAMC,qBAAqB,GAAAH,OAAA,CAAAG,qBAAA,GAAG,IAAIC,MAAM,CAAE,IAAI,GAAGL,sBAAsB,GAAG,IAAK,CAAC;AAChF,MAAMM,mBAAmB,GAAAL,OAAA,CAAAK,mBAAA,GAAG,IAAID,MAAM,CAAE,GAAG,GAAGL,sBAAsB,GAAI,KAAM,CAAC;;AAEtF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAO,SAAUC,IAAI,EAAG;EAC/B;EACAA,IAAI,GAAG,IAAAC,sCAAqB,EAAED,IAAK,CAAC;;EAEpC;EACA;EACAA,IAAI,GAAGA,IAAI,CAACE,OAAO,CAAE,WAAW,EAAE,EAAG,CAAC;;EAEtC;AACD;AACA;AACA;AACA;AACA;AACA;EACC,MAAMC,cAAc,GAAG,IAAIN,MAAM,CAAE,QAAQ,EAAE,GAAI,CAAC;EAElDG,IAAI,GAAGA,IAAI,CAACE,OAAO,CAAEC,cAAc,EAAE,EAAG,CAAC;EACzCH,IAAI,GAAGA,IAAI,CAACE,OAAO,CAAEN,qBAAqB,EAAE,EAAG,CAAC;EAChDI,IAAI,GAAGA,IAAI,CAACE,OAAO,CAAEJ,mBAAmB,EAAE,EAAG,CAAC;EAE9C,OAAOE,IAAI;AACZ","ignoreList":[]}

package/build/languageProcessing/helpers/sanitize/removePunctuationExceptQuotes.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+// Replace all other punctuation characters at the beginning or at the end of a word.
+const punctuationRegexString = "[\\–\\-\\(\\)_\\[\\]’'.?!:;,¿¡«»‹›\u2014\u00d7\u002b\u0026<>]+";
+const punctuationRegexStart = new RegExp("^" + punctuationRegexString);
+const punctuationRegexEnd = new RegExp(punctuationRegexString + "$");
+
+/**
+ * Replaces punctuation characters at the beginning and end of a given text string.
+ *
+ * @param {String} text The text to remove the punctuation characters for.
+ *
+ * @returns {String} The sanitized text.
+ */
+function _default(text) {
+  text = text.replace(punctuationRegexStart, "");
+  text = text.replace(punctuationRegexEnd, "");
+  return text;
+}
+//# sourceMappingURL=removePunctuationExceptQuotes.js.map

package/build/languageProcessing/helpers/sanitize/removePunctuationExceptQuotes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removePunctuationExceptQuotes.js","names":["punctuationRegexString","punctuationRegexStart","RegExp","punctuationRegexEnd","_default","text","replace"],"sources":["../../../../src/languageProcessing/helpers/sanitize/removePunctuationExceptQuotes.js"],"sourcesContent":["// Replace all other punctuation characters at the beginning or at the end of a word.\nconst punctuationRegexString = \"[\\\\–\\\\-\\\\(\\\\)_\\\\[\\\\]’'.?!:;,¿¡«»‹›\\u2014\\u00d7\\u002b\\u0026<>]+\";\nconst punctuationRegexStart = new RegExp( \"^\" + punctuationRegexString );\nconst punctuationRegexEnd = new RegExp( punctuationRegexString + \"$\" );\n\n/**\n * Replaces punctuation characters at the beginning and end of a given text string.\n *\n * @param {String} text The text to remove the punctuation characters for.\n *\n * @returns {String} The sanitized text.\n */\nexport default function( text ) {\n\ttext = text.replace( punctuationRegexStart, \"\" );\n\ttext = text.replace( punctuationRegexEnd, \"\" );\n\n\treturn text;\n}\n"],"mappings":";;;;;;AAAA;AACA,MAAMA,sBAAsB,GAAG,gEAAgE;AAC/F,MAAMC,qBAAqB,GAAG,IAAIC,MAAM,CAAE,GAAG,GAAGF,sBAAuB,CAAC;AACxE,MAAMG,mBAAmB,GAAG,IAAID,MAAM,CAAEF,sBAAsB,GAAG,GAAI,CAAC;;AAEtE;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAI,SAAUC,IAAI,EAAG;EAC/BA,IAAI,GAAGA,IAAI,CAACC,OAAO,CAAEL,qBAAqB,EAAE,EAAG,CAAC;EAChDI,IAAI,GAAGA,IAAI,CAACC,OAAO,CAAEH,mBAAmB,EAAE,EAAG,CAAC;EAE9C,OAAOE,IAAI;AACZ","ignoreList":[]}