axyseo 2.1.12 → 2.1.13

package/build/cjs/languageProcessing/helpers/prominentWords/determineProminentWords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"determineProminentWords.js","names":["_lodash","require","_getWords","_interopRequireDefault","_quotes","_ProminentWord","e","__esModule","default","specialCharacters","filterProminentWords","prominentWords","minimalNumberOfOccurrences","filter","word","getOccurrences","getWord","replace","sortProminentWords","sort","wordA","wordB","difference","getStem","localeCompare","collapseProminentWordsOnStem","length","collapsedProminentWords","previousWord","ProminentWord","i","currentWord","setOccurrences","toLocaleLowerCase","setWord","push","retrieveAbbreviations","text","words","getWords","normalizeSingle","abbreviations","forEach","toLocaleUpperCase","uniq","computeProminentWords","stemmer","functionWords","uniqueContentWords","includes","trim","element","computeProminentWordsMemoized","memoize","join","getProminentWords","getWordsCustomHelper","getProminentWordsFromPaperAttributes","attributes","wordsFromAttributes","_default","exports"],"sources":["../../../../../src/languageProcessing/helpers/prominentWords/determineProminentWords.js"],"sourcesContent":["import { memoize, uniq } from \"lodash\";\n\nimport getWords from \"../word/getWords\";\nimport { normalizeSingle } from \"../sanitize/quotes\";\nimport ProminentWord from \"../../values/ProminentWord\";\n\nconst specialCharacters = /[1234567890‘’“”\"'.…?!:;,¿¡«»&*@#±^%$|~=+§`[\\](){}⟨⟩<>/\\\\–\\-\\u2014\\u00d7\\s]/g;\n\n/**\n * Returns only those prominent words that occur more than a certain number of times and do not consist of special characters.\n *\n * @param {ProminentWord[]} prominentWords A list of prominent words.\n * @param {int} [minimalNumberOfOccurrences] A minimal number of occurrences that is needed for a relevant prominentWord, default 2.\n *\n * @returns {ProminentWord[]} Only relevant words.\n */\nfunction filterProminentWords( prominentWords, minimalNumberOfOccurrences = 2 ) {\n\tprominentWords = prominentWords.filter( function( word ) {\n\t\treturn (\n\t\t\tword.getOccurrences() >= minimalNumberOfOccurrences &&\n\t\t\tword.getWord().replace( specialCharacters, \"\" ) !== \"\"\n\t\t);\n\t} );\n\treturn prominentWords;\n}\n\n/**\n * Sorts prominent words based on their number of occurrences and length.\n *\n * @param {ProminentWord[]} prominentWords The prominent words to sort.\n *\n * @returns {void}\n */\nfunction sortProminentWords( prominentWords ) {\n\tprominentWords.sort( function( wordA, wordB ) {\n\t\tconst difference = wordB.getOccurrences() - wordA.getOccurrences();\n\t\t// The word with the highest number of occurrences comes first.\n\t\tif ( difference !== 0 ) {\n\t\t\treturn difference;\n\t\t}\n\n\t\t// In case of a tie on occurrence number, the alphabetically first word comes first.\n\t\treturn wordA.getStem().localeCompare( wordB.getStem() );\n\t} );\n}\n\n/**\n * Collapses prominent words that have the same stem.\n *\n * @param {ProminentWord[]} prominentWords All prominentWords.\n *\n * @returns {ProminentWord[]} The original array with collapsed duplicates.\n */\nfunction collapseProminentWordsOnStem( prominentWords ) {\n\tif ( prominentWords.length === 0 ) {\n\t\treturn [];\n\t}\n\n\t// Sort the input array by stem\n\tprominentWords.sort( function( wordA, wordB ) {\n\t\treturn wordA.getStem().localeCompare( wordB.getStem() );\n\t} );\n\n\tconst collapsedProminentWords = [];\n\tlet previousWord = new ProminentWord(\n\t\tprominentWords[ 0 ].getWord(),\n\t\tprominentWords[ 0 ].getStem(),\n\t\tprominentWords[ 0 ].getOccurrences()\n\t);\n\n\tfor ( let i = 1; i < prominentWords.length; i++ ) {\n\t\tconst currentWord = new ProminentWord(\n\t\t\tprominentWords[ i ].getWord(),\n\t\t\tprominentWords[ i ].getStem(),\n\t\t\tprominentWords[ i ].getOccurrences()\n\t\t);\n\n\t\t/*\n\t\t * Compare the stem of the current word in the loop with the previously available stem.\n\t\t * If they are equal, the word should be collapsed.\n\t\t * When collapsing, the numbers of occurrences get summed.\n\t\t * If the stem happens to equal the real word that occurred in the text, we can be sure it's ok to display it\n\t\t * to the customer. So, the stem reassigns the word.\n\t\t */\n\t\tif ( currentWord.getStem() ===  previousWord.getStem() ) {\n\t\t\tpreviousWord.setOccurrences( previousWord.getOccurrences() + currentWord.getOccurrences() );\n\n\t\t\tif ( currentWord.getWord() === previousWord.getStem() || currentWord.getWord().toLocaleLowerCase() === previousWord.getStem() ) {\n\t\t\t\tpreviousWord.setWord( currentWord.getWord() );\n\t\t\t}\n\t\t} else {\n\t\t\tcollapsedProminentWords.push( previousWord );\n\t\t\tpreviousWord = currentWord;\n\t\t}\n\t}\n\n\tcollapsedProminentWords.push( previousWord );\n\n\treturn collapsedProminentWords;\n}\n\n/**\n * Retrieves a list of all abbreviations from the text. Returns an empty array if the input text is empty.\n *\n * @param {string} text A text.\n *\n * @returns {string[]} A list of abbreviations from the list.\n */\nfunction retrieveAbbreviations( text ) {\n\tconst words = getWords( normalizeSingle( text ) );\n\tconst abbreviations = [];\n\n\twords.forEach( function( word ) {\n\t\tif ( word.length > 1 && word.length < 5 && word === word.toLocaleUpperCase() ) {\n\t\t\tabbreviations.push( word.toLocaleLowerCase() );\n\t\t}\n\t} );\n\n\treturn uniq( abbreviations );\n}\n\n/**\n * Computes prominent words from an array of words. In order to do so, checks whether the word is included in the list of\n * function words and determines the number of occurrences for every word. Then checks if any two words have the same stem\n * and if so collapses over them.\n *\n * @param {string[]} words          The words to determine relevance for.\n * @param {string[]} abbreviations  Abbreviations that should not be stemmed.\n * @param {Function} stemmer        The available stemmer.\n * @param {Array} functionWords     The available function words list.\n *\n * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.\n */\nfunction computeProminentWords( words, abbreviations, stemmer, functionWords ) {\n\tif ( words.length === 0 ) {\n\t\treturn [];\n\t}\n\n\tconst uniqueContentWords = uniq( words.filter( word => ! functionWords.includes( word.trim() ) ) );\n\tconst prominentWords = [];\n\n\tuniqueContentWords.forEach( function( word ) {\n\t\tif ( abbreviations.includes( word ) ) {\n\t\t\tprominentWords.push( new ProminentWord(\n\t\t\t\tword.toLocaleUpperCase(),\n\t\t\t\tword,\n\t\t\t\twords.filter( element => element === word ).length\n\t\t\t) );\n\t\t} else {\n\t\t\tprominentWords.push( new ProminentWord(\n\t\t\t\tword,\n\t\t\t\tstemmer( word ),\n\t\t\t\twords.filter( element => element === word ).length\n\t\t\t) );\n\t\t}\n\t} );\n\n\treturn collapseProminentWordsOnStem( prominentWords );\n}\n\n/**\n * Caches prominent words depending on text words.\n * Only the words and abbreviations are used as the cache key as the stemmer and function words\n * are config and shouldn't change in the scope of one request.\n *\n * @param {string[]} words          The words to determine relevance for.\n * @param {string[]} abbreviations  Abbreviations that should not be stemmed.\n * @param {Function} stemmer        The available stemmer.\n * @param {Array} functionWords     The available function words list.\n *\n * @returns {function} The function that collects prominent words for a given set of text words, language and morphologyData.\n */\nconst computeProminentWordsMemoized = memoize( ( words, abbreviations, stemmer, functionWords ) => {\n\treturn computeProminentWords( words, abbreviations, stemmer, functionWords );\n}, ( words, abbreviations ) => {\n\treturn words.join( \",\" ) + \",\" + abbreviations.join( \",\" );\n} );\n\n\n/**\n * Gets prominent words from the paper text.\n *\n * @param {string} text             The text to retrieve the prominent words from.\n * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.\n * @param {Function} stemmer        The available stemmer.\n * @param {Array} functionWords     The available function words list.\n * @param {function} getWordsCustomHelper   The custom helper to get words.\n *\n * @returns {ProminentWord[]} All prominent words sorted and filtered for this text.\n */\nfunction getProminentWords( text, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {\n\tif ( text === \"\" ) {\n\t\treturn [];\n\t}\n\n\tconst words = getWordsCustomHelper\n\t\t? getWordsCustomHelper( normalizeSingle( text ).toLocaleLowerCase() )\n\t\t: getWords( normalizeSingle( text ).toLocaleLowerCase() );\n\n\treturn computeProminentWordsMemoized( words, abbreviations, stemmer, functionWords );\n}\n\n/**\n * Gets prominent words from keyphrase and synonyms, metadescription, title, and subheadings.\n *\n * @param {string[]} attributes     The array with attributes to process.\n * @param {string[]} abbreviations  The abbreviations that occur in the text and attributes of the paper.\n * @param {Function} stemmer        The available stemmer.\n * @param {Array} functionWords     The available function words list.\n * @param {function} getWordsCustomHelper   The custom helper to get words.\n *\n * @returns {ProminentWord[]} Prominent words from the paper attributes.\n */\nfunction getProminentWordsFromPaperAttributes( attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper ) {\n\tconst wordsFromAttributes = getWordsCustomHelper\n\t\t? getWordsCustomHelper( attributes.join( \" \" ).toLocaleLowerCase() )\n\t\t: getWords( attributes.join( \" \" ).toLocaleLowerCase() );\n\n\treturn computeProminentWords( wordsFromAttributes, abbreviations, stemmer, functionWords );\n}\n\nexport {\n\tgetProminentWords,\n\tgetProminentWordsFromPaperAttributes,\n\tfilterProminentWords,\n\tsortProminentWords,\n\tcollapseProminentWordsOnStem,\n\tretrieveAbbreviations,\n};\n\nexport default {\n\tgetProminentWords,\n\tgetProminentWordsFromPaperAttributes,\n\tfilterProminentWords,\n\tsortProminentWords,\n\tcollapseProminentWordsOnStem,\n\tretrieveAbbreviations,\n};\n"],"mappings":";;;;;;;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA,IAAAC,SAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AACA,IAAAI,cAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAuD,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEvD,MAAMG,iBAAiB,GAAG,6EAA6E;;AAEvG;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,oBAAoBA,CAAEC,cAAc,EAAEC,0BAA0B,GAAG,CAAC,EAAG;EAC/ED,cAAc,GAAGA,cAAc,CAACE,MAAM,CAAE,UAAUC,IAAI,EAAG;IACxD,OACCA,IAAI,CAACC,cAAc,CAAC,CAAC,IAAIH,0BAA0B,IACnDE,IAAI,CAACE,OAAO,CAAC,CAAC,CAACC,OAAO,CAAER,iBAAiB,EAAE,EAAG,CAAC,KAAK,EAAE;EAExD,CAAE,CAAC;EACH,OAAOE,cAAc;AACtB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASO,kBAAkBA,CAAEP,cAAc,EAAG;EAC7CA,cAAc,CAACQ,IAAI,CAAE,UAAUC,KAAK,EAAEC,KAAK,EAAG;IAC7C,MAAMC,UAAU,GAAGD,KAAK,CAACN,cAAc,CAAC,CAAC,GAAGK,KAAK,CAACL,cAAc,CAAC,CAAC;IAClE;IACA,IAAKO,UAAU,KAAK,CAAC,EAAG;MACvB,OAAOA,UAAU;IAClB;;IAEA;IACA,OAAOF,KAAK,CAACG,OAAO,CAAC,CAAC,CAACC,aAAa,CAAEH,KAAK,CAACE,OAAO,CAAC,CAAE,CAAC;EACxD,CAAE,CAAC;AACJ;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASE,4BAA4BA,CAAEd,cAAc,EAAG;EACvD,IAAKA,cAAc,CAACe,MAAM,KAAK,CAAC,EAAG;IAClC,OAAO,EAAE;EACV;;EAEA;EACAf,cAAc,CAACQ,IAAI,CAAE,UAAUC,KAAK,EAAEC,KAAK,EAAG;IAC7C,OAAOD,KAAK,CAACG,OAAO,CAAC,CAAC,CAACC,aAAa,CAAEH,KAAK,CAACE,OAAO,CAAC,CAAE,CAAC;EACxD,CAAE,CAAC;EAEH,MAAMI,uBAAuB,GAAG,EAAE;EAClC,IAAIC,YAAY,GAAG,IAAIC,sBAAa,CACnClB,cAAc,CAAE,CAAC,CAAE,CAACK,OAAO,CAAC,CAAC,EAC7BL,cAAc,CAAE,CAAC,CAAE,CAACY,OAAO,CAAC,CAAC,EAC7BZ,cAAc,CAAE,CAAC,CAAE,CAACI,cAAc,CAAC,CACpC,CAAC;EAED,KAAM,IAAIe,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnB,cAAc,CAACe,MAAM,EAAEI,CAAC,EAAE,EAAG;IACjD,MAAMC,WAAW,GAAG,IAAIF,sBAAa,CACpClB,cAAc,CAAEmB,CAAC,CAAE,CAACd,OAAO,CAAC,CAAC,EAC7BL,cAAc,CAAEmB,CAAC,CAAE,CAACP,OAAO,CAAC,CAAC,EAC7BZ,cAAc,CAAEmB,CAAC,CAAE,CAACf,cAAc,CAAC,CACpC,CAAC;;IAED;AACF;AACA;AACA;AACA;AACA;AACA;IACE,IAAKgB,WAAW,CAACR,OAAO,CAAC,CAAC,KAAMK,YAAY,CAACL,OAAO,CAAC,CAAC,EAAG;MACxDK,YAAY,CAACI,cAAc,CAAEJ,YAAY,CAACb,cAAc,CAAC,CAAC,GAAGgB,WAAW,CAAChB,cAAc,CAAC,CAAE,CAAC;MAE3F,IAAKgB,WAAW,CAACf,OAAO,CAAC,CAAC,KAAKY,YAAY,CAACL,OAAO,CAAC,CAAC,IAAIQ,WAAW,CAACf,OAAO,CAAC,CAAC,CAACiB,iBAAiB,CAAC,CAAC,KAAKL,YAAY,CAACL,OAAO,CAAC,CAAC,EAAG;QAC/HK,YAAY,CAACM,OAAO,CAAEH,WAAW,CAACf,OAAO,CAAC,CAAE,CAAC;MAC9C;IACD,CAAC,MAAM;MACNW,uBAAuB,CAACQ,IAAI,CAAEP,YAAa,CAAC;MAC5CA,YAAY,GAAGG,WAAW;IAC3B;EACD;EAEAJ,uBAAuB,CAACQ,IAAI,CAAEP,YAAa,CAAC;EAE5C,OAAOD,uBAAuB;AAC/B;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASS,qBAAqBA,CAAEC,IAAI,EAAG;EACtC,MAAMC,KAAK,GAAG,IAAAC,iBAAQ,EAAE,IAAAC,uBAAe,EAAEH,IAAK,CAAE,CAAC;EACjD,MAAMI,aAAa,GAAG,EAAE;EAExBH,KAAK,CAACI,OAAO,CAAE,UAAU5B,IAAI,EAAG;IAC/B,IAAKA,IAAI,CAACY,MAAM,GAAG,CAAC,IAAIZ,IAAI,CAACY,MAAM,GAAG,CAAC,IAAIZ,IAAI,KAAKA,IAAI,CAAC6B,iBAAiB,CAAC,CAAC,EAAG;MAC9EF,aAAa,CAACN,IAAI,CAAErB,IAAI,CAACmB,iBAAiB,CAAC,CAAE,CAAC;IAC/C;EACD,CAAE,CAAC;EAEH,OAAO,IAAAW,YAAI,EAAEH,aAAc,CAAC;AAC7B;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASI,qBAAqBA,CAAEP,KAAK,EAAEG,aAAa,EAAEK,OAAO,EAAEC,aAAa,EAAG;EAC9E,IAAKT,KAAK,CAACZ,MAAM,KAAK,CAAC,EAAG;IACzB,OAAO,EAAE;EACV;EAEA,MAAMsB,kBAAkB,GAAG,IAAAJ,YAAI,EAAEN,KAAK,CAACzB,MAAM,CAAEC,IAAI,IAAI,CAAEiC,aAAa,CAACE,QAAQ,CAAEnC,IAAI,CAACoC,IAAI,CAAC,CAAE,CAAE,CAAE,CAAC;EAClG,MAAMvC,cAAc,GAAG,EAAE;EAEzBqC,kBAAkB,CAACN,OAAO,CAAE,UAAU5B,IAAI,EAAG;IAC5C,IAAK2B,aAAa,CAACQ,QAAQ,CAAEnC,IAAK,CAAC,EAAG;MACrCH,cAAc,CAACwB,IAAI,CAAE,IAAIN,sBAAa,CACrCf,IAAI,CAAC6B,iBAAiB,CAAC,CAAC,EACxB7B,IAAI,EACJwB,KAAK,CAACzB,MAAM,CAAEsC,OAAO,IAAIA,OAAO,KAAKrC,IAAK,CAAC,CAACY,MAC7C,CAAE,CAAC;IACJ,CAAC,MAAM;MACNf,cAAc,CAACwB,IAAI,CAAE,IAAIN,sBAAa,CACrCf,IAAI,EACJgC,OAAO,CAAEhC,IAAK,CAAC,EACfwB,KAAK,CAACzB,MAAM,CAAEsC,OAAO,IAAIA,OAAO,KAAKrC,IAAK,CAAC,CAACY,MAC7C,CAAE,CAAC;IACJ;EACD,CAAE,CAAC;EAEH,OAAOD,4BAA4B,CAAEd,cAAe,CAAC;AACtD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMyC,6BAA6B,GAAG,IAAAC,eAAO,EAAE,CAAEf,KAAK,EAAEG,aAAa,EAAEK,OAAO,EAAEC,aAAa,KAAM;EAClG,OAAOF,qBAAqB,CAAEP,KAAK,EAAEG,aAAa,EAAEK,OAAO,EAAEC,aAAc,CAAC;AAC7E,CAAC,EAAE,CAAET,KAAK,EAAEG,aAAa,KAAM;EAC9B,OAAOH,KAAK,CAACgB,IAAI,CAAE,GAAI,CAAC,GAAG,GAAG,GAAGb,aAAa,CAACa,IAAI,CAAE,GAAI,CAAC;AAC3D,CAAE,CAAC;;AAGH;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAAElB,IAAI,EAAEI,aAAa,EAAEK,OAAO,EAAEC,aAAa,EAAES,oBAAoB,EAAG;EAC/F,IAAKnB,IAAI,KAAK,EAAE,EAAG;IAClB,OAAO,EAAE;EACV;EAEA,MAAMC,KAAK,GAAGkB,oBAAoB,GAC/BA,oBAAoB,CAAE,IAAAhB,uBAAe,EAAEH,IAAK,CAAC,CAACJ,iBAAiB,CAAC,CAAE,CAAC,GACnE,IAAAM,iBAAQ,EAAE,IAAAC,uBAAe,EAAEH,IAAK,CAAC,CAACJ,iBAAiB,CAAC,CAAE,CAAC;EAE1D,OAAOmB,6BAA6B,CAAEd,KAAK,EAAEG,aAAa,EAAEK,OAAO,EAAEC,aAAc,CAAC;AACrF;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASU,oCAAoCA,CAAEC,UAAU,EAAEjB,aAAa,EAAEK,OAAO,EAAEC,aAAa,EAAES,oBAAoB,EAAG;EACxH,MAAMG,mBAAmB,GAAGH,oBAAoB,GAC7CA,oBAAoB,CAAEE,UAAU,CAACJ,IAAI,CAAE,GAAI,CAAC,CAACrB,iBAAiB,CAAC,CAAE,CAAC,GAClE,IAAAM,iBAAQ,EAAEmB,UAAU,CAACJ,IAAI,CAAE,GAAI,CAAC,CAACrB,iBAAiB,CAAC,CAAE,CAAC;EAEzD,OAAOY,qBAAqB,CAAEc,mBAAmB,EAAElB,aAAa,EAAEK,OAAO,EAAEC,aAAc,CAAC;AAC3F;AAAC,IAAAa,QAAA,GAAAC,OAAA,CAAArD,OAAA,GAWc;EACd+C,iBAAiB;EACjBE,oCAAoC;EACpC/C,oBAAoB;EACpBQ,kBAAkB;EAClBO,4BAA4B;EAC5BW;AACD,CAAC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/regex/createRegexFromArray.js

@@ -0,0 +1,38 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+var _replaceDiacritics = _interopRequireDefault(require("../transliterate/replaceDiacritics"));
+var _sanitizeString = _interopRequireDefault(require("../sanitize/sanitizeString"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/createRegexFromArray */
+
+/**
+ * Creates a regex of combined strings from the input array.
+ *
+ * @param {string[]} array                      The array with strings
+ * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.
+ * @param {string} [extraBoundary=""]           A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.
+ *
+ * @returns {RegExp} regex                              The regex created from the array.
+ */
+function _default(array, disableWordBoundary = false, extraBoundary = "", doReplaceDiacritics = false) {
+  array = (0, _lodash.map)(array, function (string) {
+    if (doReplaceDiacritics) {
+      string = (0, _replaceDiacritics.default)(string);
+    }
+    string = (0, _sanitizeString.default)(string);
+    if (disableWordBoundary) {
+      return string;
+    }
+    return (0, _addWordboundary.default)(string, true, extraBoundary);
+  });
+  const regexString = "(" + array.join(")|(") + ")";
+  return new RegExp(regexString, "ig");
+}
+//# sourceMappingURL=createRegexFromArray.js.map

package/build/cjs/languageProcessing/helpers/regex/createRegexFromArray.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRegexFromArray.js","names":["_lodash","require","_addWordboundary","_interopRequireDefault","_replaceDiacritics","_sanitizeString","e","__esModule","default","_default","array","disableWordBoundary","extraBoundary","doReplaceDiacritics","map","string","replaceDiacritics","sanitizeString","addWordBoundary","regexString","join","RegExp"],"sources":["../../../../../src/languageProcessing/helpers/regex/createRegexFromArray.js"],"sourcesContent":["/** @module stringProcessing/createRegexFromArray */\n\nimport { map } from \"lodash\";\nimport addWordBoundary from \"../word/addWordboundary.js\";\nimport replaceDiacritics from \"../transliterate/replaceDiacritics\";\nimport sanitizeString from \"../sanitize/sanitizeString\";\n\n/**\n * Creates a regex of combined strings from the input array.\n *\n * @param {string[]} array                      The array with strings\n * @param {boolean} [disableWordBoundary=false] Boolean indicating whether or not to disable word boundaries.\n * @param {string} [extraBoundary=\"\"]           A string that is used as extra boundary for the regex.\n * @param {boolean} [doReplaceDiacritics=false] If set to true, it replaces diacritics. Defaults to false.\n *\n * @returns {RegExp} regex                              The regex created from the array.\n */\nexport default function( array, disableWordBoundary = false, extraBoundary = \"\", doReplaceDiacritics = false ) {\n\tarray = map( array, function( string ) {\n\t\tif ( doReplaceDiacritics ) {\n\t\t\tstring = replaceDiacritics( string );\n\t\t}\n\n\t\tstring = sanitizeString( string );\n\n\t\tif ( disableWordBoundary ) {\n\t\t\treturn string;\n\t\t}\n\t\treturn addWordBoundary( string, true, extraBoundary );\n\t} );\n\n\tconst regexString = \"(\" + array.join( \")|(\" ) + \")\";\n\n\treturn new RegExp( regexString, \"ig\" );\n}\n"],"mappings":";;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,gBAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,kBAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,eAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAwD,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AALxD;;AAOA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,KAAK,EAAEC,mBAAmB,GAAG,KAAK,EAAEC,aAAa,GAAG,EAAE,EAAEC,mBAAmB,GAAG,KAAK,EAAG;EAC9GH,KAAK,GAAG,IAAAI,WAAG,EAAEJ,KAAK,EAAE,UAAUK,MAAM,EAAG;IACtC,IAAKF,mBAAmB,EAAG;MAC1BE,MAAM,GAAG,IAAAC,0BAAiB,EAAED,MAAO,CAAC;IACrC;IAEAA,MAAM,GAAG,IAAAE,uBAAc,EAAEF,MAAO,CAAC;IAEjC,IAAKJ,mBAAmB,EAAG;MAC1B,OAAOI,MAAM;IACd;IACA,OAAO,IAAAG,wBAAe,EAAEH,MAAM,EAAE,IAAI,EAAEH,aAAc,CAAC;EACtD,CAAE,CAAC;EAEH,MAAMO,WAAW,GAAG,GAAG,GAAGT,KAAK,CAACU,IAAI,CAAE,KAAM,CAAC,GAAG,GAAG;EAEnD,OAAO,IAAIC,MAAM,CAAEF,WAAW,EAAE,IAAK,CAAC;AACvC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/regex/createRegexFromDoubleArray.js

@@ -0,0 +1,39 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/createRegexFromDoubleArray */
+
+/**
+ * Creates a regex string of combined strings from the input array.
+ *
+ * @param {array} array The array containing the various parts of a transition word combination.
+ *
+ * @returns {array} The array with replaced entries.
+ */
+const wordCombinationToRegexString = function (array) {
+  array = array.map(function (word) {
+    return (0, _addWordboundary.default)(word);
+  });
+  return array.join("(.*?)");
+};
+
+/**
+ * Creates a regex of combined strings from the input array, containing arrays with two entries.
+ *
+ * @param {array} array The array containing arrays.
+ *
+ * @returns {RegExp} The regex created from the array.
+ */
+function _default(array) {
+  array = array.map(function (wordCombination) {
+    return wordCombinationToRegexString(wordCombination);
+  });
+  const regexString = "(" + array.join(")|(") + ")";
+  return new RegExp(regexString, "ig");
+}
+//# sourceMappingURL=createRegexFromDoubleArray.js.map

package/build/cjs/languageProcessing/helpers/regex/createRegexFromDoubleArray.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createRegexFromDoubleArray.js","names":["_addWordboundary","_interopRequireDefault","require","e","__esModule","default","wordCombinationToRegexString","array","map","word","addWordBoundary","join","_default","wordCombination","regexString","RegExp"],"sources":["../../../../../src/languageProcessing/helpers/regex/createRegexFromDoubleArray.js"],"sourcesContent":["/** @module stringProcessing/createRegexFromDoubleArray */\n\nimport addWordBoundary from \"../word/addWordboundary.js\";\n\n/**\n * Creates a regex string of combined strings from the input array.\n *\n * @param {array} array The array containing the various parts of a transition word combination.\n *\n * @returns {array} The array with replaced entries.\n */\nconst wordCombinationToRegexString = function( array ) {\n\tarray = array.map( function( word ) {\n\t\treturn addWordBoundary( word );\n\t} );\n\treturn array.join( \"(.*?)\" );\n};\n\n/**\n * Creates a regex of combined strings from the input array, containing arrays with two entries.\n *\n * @param {array} array The array containing arrays.\n *\n * @returns {RegExp} The regex created from the array.\n */\nexport default function( array ) {\n\tarray = array.map( function( wordCombination ) {\n\t\treturn wordCombinationToRegexString( wordCombination );\n\t} );\n\n\tconst regexString = \"(\" + array.join( \")|(\" ) + \")\";\n\n\treturn new RegExp( regexString, \"ig\" );\n}\n"],"mappings":";;;;;;AAEA,IAAAA,gBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAyD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAFzD;;AAIA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,4BAA4B,GAAG,SAAAA,CAAUC,KAAK,EAAG;EACtDA,KAAK,GAAGA,KAAK,CAACC,GAAG,CAAE,UAAUC,IAAI,EAAG;IACnC,OAAO,IAAAC,wBAAe,EAAED,IAAK,CAAC;EAC/B,CAAE,CAAC;EACH,OAAOF,KAAK,CAACI,IAAI,CAAE,OAAQ,CAAC;AAC7B,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAAUL,KAAK,EAAG;EAChCA,KAAK,GAAGA,KAAK,CAACC,GAAG,CAAE,UAAUK,eAAe,EAAG;IAC9C,OAAOP,4BAA4B,CAAEO,eAAgB,CAAC;EACvD,CAAE,CAAC;EAEH,MAAMC,WAAW,GAAG,GAAG,GAAGP,KAAK,CAACI,IAAI,CAAE,KAAM,CAAC,GAAG,GAAG;EAEnD,OAAO,IAAII,MAAM,CAAED,WAAW,EAAE,IAAK,CAAC;AACvC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/regex/createWordRegex.js

@@ -0,0 +1,34 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _sanitizeString = _interopRequireDefault(require("../sanitize/sanitizeString"));
+var _replaceDiacritics = _interopRequireDefault(require("../transliterate/replaceDiacritics.js"));
+var _addWordboundary = _interopRequireDefault(require("../word/addWordboundary.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/stringToRegex */
+/**
+ * Creates a regex from a string so it can be matched everywhere in the same way.
+ *
+ * @param {string}  string                     The string to make a regex from.
+ * @param {string}  [extraBoundary=""]         A string that is used as extra boundary for the regex.
+ * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.
+ *
+ * @returns {RegExp} regex The regex made from the keyword
+ */
+var _default = exports.default = (0, _lodash.memoize)(function (string, extraBoundary, doReplaceDiacritics) {
+  if ((0, _lodash.isUndefined)(extraBoundary)) {
+    extraBoundary = "";
+  }
+  if ((0, _lodash.isUndefined)(doReplaceDiacritics) || doReplaceDiacritics === true) {
+    string = (0, _replaceDiacritics.default)(string);
+  }
+  string = (0, _sanitizeString.default)(string);
+  string = (0, _lodash.escapeRegExp)(string);
+  string = (0, _addWordboundary.default)(string, false, extraBoundary);
+  return new RegExp(string, "ig");
+});
+//# sourceMappingURL=createWordRegex.js.map

package/build/cjs/languageProcessing/helpers/regex/createWordRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createWordRegex.js","names":["_lodash","require","_sanitizeString","_interopRequireDefault","_replaceDiacritics","_addWordboundary","e","__esModule","default","_default","exports","memoize","string","extraBoundary","doReplaceDiacritics","isUndefined","replaceDiacritics","sanitizeString","escapeRegExp","addWordBoundary","RegExp"],"sources":["../../../../../src/languageProcessing/helpers/regex/createWordRegex.js"],"sourcesContent":["/** @module stringProcessing/stringToRegex */\nimport { escapeRegExp, isUndefined, memoize } from \"lodash\";\nimport sanitizeString from \"../sanitize/sanitizeString\";\nimport replaceDiacritics from \"../transliterate/replaceDiacritics.js\";\nimport addWordBoundary from \"../word/addWordboundary.js\";\n\n/**\n * Creates a regex from a string so it can be matched everywhere in the same way.\n *\n * @param {string}  string                     The string to make a regex from.\n * @param {string}  [extraBoundary=\"\"]         A string that is used as extra boundary for the regex.\n * @param {boolean} [doReplaceDiacritics=true] If set to false, it doesn't replace diacritics. Defaults to true.\n *\n * @returns {RegExp} regex The regex made from the keyword\n */\nexport default memoize( function( string, extraBoundary, doReplaceDiacritics ) {\n\tif ( isUndefined( extraBoundary ) ) {\n\t\textraBoundary = \"\";\n\t}\n\n\tif ( isUndefined( doReplaceDiacritics ) || doReplaceDiacritics === true ) {\n\t\tstring = replaceDiacritics( string );\n\t}\n\n\tstring = sanitizeString( string );\n\tstring = escapeRegExp( string );\n\tstring = addWordBoundary( string, false, extraBoundary );\n\n\treturn new RegExp( string, \"ig\" );\n} );\n"],"mappings":";;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,eAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,kBAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,gBAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAyD,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAJzD;AAMA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AARA,IAAAG,QAAA,GAAAC,OAAA,CAAAF,OAAA,GASe,IAAAG,eAAO,EAAE,UAAUC,MAAM,EAAEC,aAAa,EAAEC,mBAAmB,EAAG;EAC9E,IAAK,IAAAC,mBAAW,EAAEF,aAAc,CAAC,EAAG;IACnCA,aAAa,GAAG,EAAE;EACnB;EAEA,IAAK,IAAAE,mBAAW,EAAED,mBAAoB,CAAC,IAAIA,mBAAmB,KAAK,IAAI,EAAG;IACzEF,MAAM,GAAG,IAAAI,0BAAiB,EAAEJ,MAAO,CAAC;EACrC;EAEAA,MAAM,GAAG,IAAAK,uBAAc,EAAEL,MAAO,CAAC;EACjCA,MAAM,GAAG,IAAAM,oBAAY,EAAEN,MAAO,CAAC;EAC/BA,MAAM,GAAG,IAAAO,wBAAe,EAAEP,MAAM,EAAE,KAAK,EAAEC,aAAc,CAAC;EAExD,OAAO,IAAIO,MAAM,CAAER,MAAM,EAAE,IAAK,CAAC;AAClC,CAAE,CAAC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/regex/matchStringWithRegex.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+/** @module stringProcessing/matchStringWithRegex */
+
+/**
+ * Checks a string with a regex, return all matches found with that regex.
+ *
+ * @param {String} 		text        The text to match.
+ * @param {object} 		regex  		A compiled regular expression.
+ *
+ * @returns {Array} Array with matches, empty array if no matches found.
+ */
+function _default(text, regex) {
+  let matches = text.match(regex);
+  if (matches === null) {
+    matches = [];
+  }
+  return matches;
+}
+//# sourceMappingURL=matchStringWithRegex.js.map

package/build/cjs/languageProcessing/helpers/regex/matchStringWithRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchStringWithRegex.js","names":["_default","text","regex","matches","match"],"sources":["../../../../../src/languageProcessing/helpers/regex/matchStringWithRegex.js"],"sourcesContent":["/** @module stringProcessing/matchStringWithRegex */\n\n/**\n * Checks a string with a regex, return all matches found with that regex.\n *\n * @param {String} \t\ttext        The text to match.\n * @param {object} \t\tregex  \t\tA compiled regular expression.\n *\n * @returns {Array} Array with matches, empty array if no matches found.\n */\nexport default function( text, regex ) {\n\tlet matches = text.match( regex );\n\n\tif ( matches === null ) {\n\t\tmatches = [];\n\t}\n\n\treturn matches;\n}\n"],"mappings":";;;;;;AAAA;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAA,SAAUC,IAAI,EAAEC,KAAK,EAAG;EACtC,IAAIC,OAAO,GAAGF,IAAI,CAACG,KAAK,CAAEF,KAAM,CAAC;EAEjC,IAAKC,OAAO,KAAK,IAAI,EAAG;IACvBA,OAAO,GAAG,EAAE;EACb;EAEA,OAAOA,OAAO;AACf","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js

@@ -0,0 +1,21 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = searchAndReplaceWithOneRegex;
+/**
+ * Checks whether a given regex matches a given word and if so, performs the replacement.
+ *
+ * @param {string}      word 					The word that may need to be modified.
+ * @param {string[]}    regexAndReplacement     A regex and the required replacement.
+ *
+ * @returns {?string} The modified stem or undefined if no match was found.
+ */
+function searchAndReplaceWithOneRegex(word, regexAndReplacement) {
+  if (word.search(new RegExp(regexAndReplacement[0])) !== -1) {
+    word = word.replace(new RegExp(regexAndReplacement[0]), regexAndReplacement[1]);
+    return word;
+  }
+}
+//# sourceMappingURL=searchAndReplaceWithOneRegex.js.map

package/build/cjs/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"searchAndReplaceWithOneRegex.js","names":["searchAndReplaceWithOneRegex","word","regexAndReplacement","search","RegExp","replace"],"sources":["../../../../../src/languageProcessing/helpers/regex/searchAndReplaceWithOneRegex.js"],"sourcesContent":["/**\n * Checks whether a given regex matches a given word and if so, performs the replacement.\n *\n * @param {string}      word \t\t\t\t\tThe word that may need to be modified.\n * @param {string[]}    regexAndReplacement     A regex and the required replacement.\n *\n * @returns {?string} The modified stem or undefined if no match was found.\n */\nexport default function searchAndReplaceWithOneRegex( word, regexAndReplacement ) {\n\tif ( word.search( new RegExp( regexAndReplacement[ 0 ] ) ) !== -1 ) {\n\t\tword = word.replace( new RegExp( regexAndReplacement[ 0 ] ), regexAndReplacement[ 1 ] );\n\t\treturn word;\n\t}\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASA,4BAA4BA,CAAEC,IAAI,EAAEC,mBAAmB,EAAG;EACjF,IAAKD,IAAI,CAACE,MAAM,CAAE,IAAIC,MAAM,CAAEF,mBAAmB,CAAE,CAAC,CAAG,CAAE,CAAC,KAAK,CAAC,CAAC,EAAG;IACnED,IAAI,GAAGA,IAAI,CAACI,OAAO,CAAE,IAAID,MAAM,CAAEF,mBAAmB,CAAE,CAAC,CAAG,CAAC,EAAEA,mBAAmB,CAAE,CAAC,CAAG,CAAC;IACvF,OAAOD,IAAI;EACZ;AACD","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/doubleQuotes.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+/**
+ * Returns an array with double quotes.
+ *
+ * Japanese-specific quotes:
+ * U300c - left corner bracket.
+ * U300d - right corner bracket.
+ * U300e - left white corner bracket.
+ * U300f - right white corner bracket.
+ *
+ * @returns {string[]} The array with double quotes..
+ */
+var _default = exports.default = ["“", "”", "〝", "〞", "〟", "‟", "„", "\"", "\u300c", "\u300d", "\u300e", "\u300f"];
+//# sourceMappingURL=doubleQuotes.js.map

package/build/cjs/languageProcessing/helpers/sanitize/doubleQuotes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"doubleQuotes.js","names":["_default","exports","default"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/doubleQuotes.js"],"sourcesContent":["/**\n * Returns an array with double quotes.\n *\n * Japanese-specific quotes:\n * U300c - left corner bracket.\n * U300d - right corner bracket.\n * U300e - left white corner bracket.\n * U300f - right white corner bracket.\n *\n * @returns {string[]} The array with double quotes..\n */\nexport default [ \"“\", \"”\", \"〝\", \"〞\", \"〟\", \"‟\", \"„\", \"\\\"\", \"\\u300c\", \"\\u300d\", \"\\u300e\", \"\\u300f\" ];\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAVA,IAAAA,QAAA,GAAAC,OAAA,CAAAC,OAAA,GAWe,CAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAE","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js

@@ -0,0 +1,131 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.filterShortcodesFromHTML = exports.default = exports.createShortcodeTagsRegex = void 0;
+/**
+ * Creates a regex to filter shortcodes from HTML.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {RegExp} The regex to recognize the shortcodes.
+ */
+const createShortcodeTagsRegex = shortcodeTags => {
+  const shortcodeTagsRegexString = `\\[\\/?(${shortcodeTags.join("|")})[^\\]]*\\]`;
+  return new RegExp(shortcodeTagsRegexString, "g");
+};
+
+/**
+ * Filters shortcodes from HTML string.
+ * @param {string} html The HTML to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {string} The filtered HTML.
+ */
+exports.createShortcodeTagsRegex = createShortcodeTagsRegex;
+const filterShortcodesFromHTML = (html, shortcodeTags) => {
+  if (!shortcodeTags || shortcodeTags.length === 0) {
+    return html;
+  }
+  const shortcodeTagsRegex = createShortcodeTagsRegex(shortcodeTags);
+  return html.replace(shortcodeTagsRegex, "");
+};
+
+/**
+ * Checks if a token is part of an opening shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of an opening shortcode.
+ */
+exports.filterShortcodesFromHTML = filterShortcodesFromHTML;
+const tokenIsPartOfOpeningShortcode = (tokens, index) => {
+  return tokens[index - 1] && tokens[index - 1].text === "[";
+};
+
+/**
+ * Checks if a token is part of a closing shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @returns {boolean} Whether the token is part of a closing shortcode.
+ */
+const tokenIsPartOfClosingShortcode = (tokens, index) => {
+  return tokens[index - 1] && tokens[index - 1].text === "/" && tokens[index - 2] && tokens[index - 2].text === "[";
+};
+
+/**
+ * Checks if a token is part of a shortcode.
+ * @param {Token[]} tokens The tokens to check.
+ * @param {number} index The index of the current token.
+ * @param {string[]} shortcodeTags An array of active shortcode tags.
+ * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).
+ * @returns {boolean} Whether the token is part of a shortcode.
+ */
+const tokenIsShortcode = (tokens, index, shortcodeTags, encounteredClosingBracket) => {
+  return encounteredClosingBracket && shortcodeTags.includes(tokens[index].text) && (tokenIsPartOfOpeningShortcode(tokens, index) || tokenIsPartOfClosingShortcode(tokens, index));
+};
+
+/**
+ * Filters shortcodes from a sentence.
+ * @param {Sentence} sentence The sentence to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ *
+ */
+const filterShortcodesFromSentence = (sentence, shortcodeTags, shortcodeTagsRegex) => {
+  const tokens = sentence.tokens;
+  let encounteredClosingBracket = false;
+  // traverse through tokens backwards
+  for (let i = tokens.length - 1; i >= 0; i--) {
+    if (tokens[i].text === "]") {
+      encounteredClosingBracket = true;
+    }
+    if (tokenIsShortcode(tokens, i, shortcodeTags, encounteredClosingBracket)) {
+      while (tokens[i].text !== "]") {
+        tokens.splice(i, 1);
+      }
+      tokens.splice(i, 1);
+      encounteredClosingBracket = false;
+      while (tokens[i - 1] && "[/".includes(tokens[i - 1].text)) {
+        tokens.splice(i - 1, 1);
+        i--;
+      }
+    }
+  }
+  sentence.tokens = tokens;
+  sentence.text = sentence.text.replace(shortcodeTagsRegex, "");
+};
+
+/**
+ * A recursive function that filters shortcodes from sentences.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.
+ * @returns {void}
+ */
+const filterShortcodesFromSentences = (tree, shortcodeTags, shortcodeTagsRegex) => {
+  if (tree.sentences) {
+    tree.sentences.forEach(sentence => {
+      filterShortcodesFromSentence(sentence, shortcodeTags, shortcodeTagsRegex);
+    });
+  }
+  if (tree.childNodes) {
+    tree.childNodes.forEach(childNode => {
+      filterShortcodesFromSentences(childNode, shortcodeTags, shortcodeTagsRegex);
+    });
+  }
+};
+
+/**
+ * Filters shortcodes from the tree.
+ * @param {Node} tree The tree to filter.
+ * @param {string[]} shortcodeTags The tags of the shortcodes to filter.
+ * @returns {void}
+ */
+const filterShortcodesFromTree = (tree, shortcodeTags) => {
+  if (!shortcodeTags || shortcodeTags.length === 0) {
+    return;
+  }
+  const shortcodeTagsRegex = createShortcodeTagsRegex(shortcodeTags);
+  filterShortcodesFromSentences(tree, shortcodeTags, shortcodeTagsRegex);
+};
+var _default = exports.default = filterShortcodesFromTree;
+//# sourceMappingURL=filterShortcodesFromTree.js.map

package/build/cjs/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterShortcodesFromTree.js","names":["createShortcodeTagsRegex","shortcodeTags","shortcodeTagsRegexString","join","RegExp","exports","filterShortcodesFromHTML","html","length","shortcodeTagsRegex","replace","tokenIsPartOfOpeningShortcode","tokens","index","text","tokenIsPartOfClosingShortcode","tokenIsShortcode","encounteredClosingBracket","includes","filterShortcodesFromSentence","sentence","i","splice","filterShortcodesFromSentences","tree","sentences","forEach","childNodes","childNode","filterShortcodesFromTree","_default","default"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/filterShortcodesFromTree.js"],"sourcesContent":["\n/**\n * Creates a regex to filter shortcodes from HTML.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {RegExp} The regex to recognize the shortcodes.\n */\nexport const createShortcodeTagsRegex = shortcodeTags => {\n\tconst shortcodeTagsRegexString = `\\\\[\\\\/?(${ shortcodeTags.join( \"|\" ) })[^\\\\]]*\\\\]`;\n\treturn new RegExp( shortcodeTagsRegexString, \"g\" );\n};\n\n/**\n * Filters shortcodes from HTML string.\n * @param {string} html The HTML to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {string} The filtered HTML.\n */\nexport const filterShortcodesFromHTML = ( html, shortcodeTags ) => {\n\tif ( ! shortcodeTags || shortcodeTags.length === 0 ) {\n\t\treturn html;\n\t}\n\tconst shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );\n\treturn html.replace( shortcodeTagsRegex, \"\" );\n};\n\n/**\n * Checks if a token is part of an opening shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @returns {boolean} Whether the token is part of an opening shortcode.\n */\nconst tokenIsPartOfOpeningShortcode = ( tokens, index ) => {\n\treturn tokens[ index - 1 ] && tokens[ index - 1 ].text === \"[\";\n};\n\n/**\n * Checks if a token is part of a closing shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @returns {boolean} Whether the token is part of a closing shortcode.\n */\nconst tokenIsPartOfClosingShortcode = ( tokens, index ) => {\n\treturn  tokens[ index - 1 ] && tokens[ index - 1 ].text === \"/\"  && tokens[ index - 2 ]  && tokens[ index - 2 ].text === \"[\";\n};\n\n/**\n * Checks if a token is part of a shortcode.\n * @param {Token[]} tokens The tokens to check.\n * @param {number} index The index of the current token.\n * @param {string[]} shortcodeTags An array of active shortcode tags.\n * @param {boolean} encounteredClosingBracket Whether a closing bracket has been encountered (while looping backwards through the tokens).\n * @returns {boolean} Whether the token is part of a shortcode.\n */\nconst tokenIsShortcode = ( tokens, index, shortcodeTags, encounteredClosingBracket ) => {\n\treturn encounteredClosingBracket && shortcodeTags.includes( tokens[ index ].text ) &&\n\t\t( tokenIsPartOfOpeningShortcode( tokens, index ) || tokenIsPartOfClosingShortcode( tokens, index )  );\n};\n\n/**\n * Filters shortcodes from a sentence.\n * @param {Sentence} sentence The sentence to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.\n * @returns {void}\n *\n */\nconst filterShortcodesFromSentence = ( sentence, shortcodeTags, shortcodeTagsRegex ) => {\n\tconst tokens = sentence.tokens;\n\n\tlet encounteredClosingBracket = false;\n\t// traverse through tokens backwards\n\tfor ( let i = tokens.length - 1; i >= 0; i-- ) {\n\t\tif ( tokens[ i ].text === \"]\" ) {\n\t\t\tencounteredClosingBracket = true;\n\t\t}\n\n\t\tif ( tokenIsShortcode( tokens, i, shortcodeTags, encounteredClosingBracket ) ) {\n\t\t\twhile ( tokens[ i ].text !== \"]\" ) {\n\t\t\t\ttokens.splice( i, 1 );\n\t\t\t}\n\t\t\ttokens.splice( i, 1 );\n\t\t\tencounteredClosingBracket = false;\n\n\t\t\twhile ( tokens[ i - 1 ] && \"[/\".includes( tokens[ i - 1 ].text ) ) {\n\t\t\t\ttokens.splice( i - 1, 1 );\n\t\t\t\ti--;\n\t\t\t}\n\t\t}\n\t}\n\tsentence.tokens = tokens;\n\n\tsentence.text = sentence.text.replace( shortcodeTagsRegex, \"\" );\n};\n\n/**\n * A recursive function that filters shortcodes from sentences.\n * @param {Node} tree The tree to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @param {RegExp} shortcodeTagsRegex The regex to filter the shortcodes.\n * @returns {void}\n */\nconst filterShortcodesFromSentences = ( tree, shortcodeTags, shortcodeTagsRegex ) => {\n\tif ( tree.sentences ) {\n\t\ttree.sentences.forEach( sentence => {\n\t\t\tfilterShortcodesFromSentence( sentence, shortcodeTags, shortcodeTagsRegex );\n\t\t} );\n\t}\n\n\tif ( tree.childNodes ) {\n\t\ttree.childNodes.forEach( childNode => {\n\t\t\tfilterShortcodesFromSentences( childNode, shortcodeTags, shortcodeTagsRegex );\n\t\t} );\n\t}\n};\n\n/**\n * Filters shortcodes from the tree.\n * @param {Node} tree The tree to filter.\n * @param {string[]} shortcodeTags The tags of the shortcodes to filter.\n * @returns {void}\n */\nconst filterShortcodesFromTree = ( tree, shortcodeTags ) => {\n\tif ( ! shortcodeTags || shortcodeTags.length === 0 ) {\n\t\treturn;\n\t}\n\tconst shortcodeTagsRegex = createShortcodeTagsRegex( shortcodeTags );\n\n\tfilterShortcodesFromSentences( tree, shortcodeTags, shortcodeTagsRegex );\n};\n\nexport default filterShortcodesFromTree;\n"],"mappings":";;;;;;AACA;AACA;AACA;AACA;AACA;AACO,MAAMA,wBAAwB,GAAGC,aAAa,IAAI;EACxD,MAAMC,wBAAwB,GAAG,WAAYD,aAAa,CAACE,IAAI,CAAE,GAAI,CAAC,aAAc;EACpF,OAAO,IAAIC,MAAM,CAAEF,wBAAwB,EAAE,GAAI,CAAC;AACnD,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AALAG,OAAA,CAAAL,wBAAA,GAAAA,wBAAA;AAMO,MAAMM,wBAAwB,GAAGA,CAAEC,IAAI,EAAEN,aAAa,KAAM;EAClE,IAAK,CAAEA,aAAa,IAAIA,aAAa,CAACO,MAAM,KAAK,CAAC,EAAG;IACpD,OAAOD,IAAI;EACZ;EACA,MAAME,kBAAkB,GAAGT,wBAAwB,CAAEC,aAAc,CAAC;EACpE,OAAOM,IAAI,CAACG,OAAO,CAAED,kBAAkB,EAAE,EAAG,CAAC;AAC9C,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AALAJ,OAAA,CAAAC,wBAAA,GAAAA,wBAAA;AAMA,MAAMK,6BAA6B,GAAGA,CAAEC,MAAM,EAAEC,KAAK,KAAM;EAC1D,OAAOD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAID,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG;AAC/D,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,6BAA6B,GAAGA,CAAEH,MAAM,EAAEC,KAAK,KAAM;EAC1D,OAAQD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAID,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG,IAAKF,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,IAAKD,MAAM,CAAEC,KAAK,GAAG,CAAC,CAAE,CAACC,IAAI,KAAK,GAAG;AAC7H,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAME,gBAAgB,GAAGA,CAAEJ,MAAM,EAAEC,KAAK,EAAEZ,aAAa,EAAEgB,yBAAyB,KAAM;EACvF,OAAOA,yBAAyB,IAAIhB,aAAa,CAACiB,QAAQ,CAAEN,MAAM,CAAEC,KAAK,CAAE,CAACC,IAAK,CAAC,KAC/EH,6BAA6B,CAAEC,MAAM,EAAEC,KAAM,CAAC,IAAIE,6BAA6B,CAAEH,MAAM,EAAEC,KAAM,CAAC,CAAG;AACvG,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMM,4BAA4B,GAAGA,CAAEC,QAAQ,EAAEnB,aAAa,EAAEQ,kBAAkB,KAAM;EACvF,MAAMG,MAAM,GAAGQ,QAAQ,CAACR,MAAM;EAE9B,IAAIK,yBAAyB,GAAG,KAAK;EACrC;EACA,KAAM,IAAII,CAAC,GAAGT,MAAM,CAACJ,MAAM,GAAG,CAAC,EAAEa,CAAC,IAAI,CAAC,EAAEA,CAAC,EAAE,EAAG;IAC9C,IAAKT,MAAM,CAAES,CAAC,CAAE,CAACP,IAAI,KAAK,GAAG,EAAG;MAC/BG,yBAAyB,GAAG,IAAI;IACjC;IAEA,IAAKD,gBAAgB,CAAEJ,MAAM,EAAES,CAAC,EAAEpB,aAAa,EAAEgB,yBAA0B,CAAC,EAAG;MAC9E,OAAQL,MAAM,CAAES,CAAC,CAAE,CAACP,IAAI,KAAK,GAAG,EAAG;QAClCF,MAAM,CAACU,MAAM,CAAED,CAAC,EAAE,CAAE,CAAC;MACtB;MACAT,MAAM,CAACU,MAAM,CAAED,CAAC,EAAE,CAAE,CAAC;MACrBJ,yBAAyB,GAAG,KAAK;MAEjC,OAAQL,MAAM,CAAES,CAAC,GAAG,CAAC,CAAE,IAAI,IAAI,CAACH,QAAQ,CAAEN,MAAM,CAAES,CAAC,GAAG,CAAC,CAAE,CAACP,IAAK,CAAC,EAAG;QAClEF,MAAM,CAACU,MAAM,CAAED,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACzBA,CAAC,EAAE;MACJ;IACD;EACD;EACAD,QAAQ,CAACR,MAAM,GAAGA,MAAM;EAExBQ,QAAQ,CAACN,IAAI,GAAGM,QAAQ,CAACN,IAAI,CAACJ,OAAO,CAAED,kBAAkB,EAAE,EAAG,CAAC;AAChE,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMc,6BAA6B,GAAGA,CAAEC,IAAI,EAAEvB,aAAa,EAAEQ,kBAAkB,KAAM;EACpF,IAAKe,IAAI,CAACC,SAAS,EAAG;IACrBD,IAAI,CAACC,SAAS,CAACC,OAAO,CAAEN,QAAQ,IAAI;MACnCD,4BAA4B,CAAEC,QAAQ,EAAEnB,aAAa,EAAEQ,kBAAmB,CAAC;IAC5E,CAAE,CAAC;EACJ;EAEA,IAAKe,IAAI,CAACG,UAAU,EAAG;IACtBH,IAAI,CAACG,UAAU,CAACD,OAAO,CAAEE,SAAS,IAAI;MACrCL,6BAA6B,CAAEK,SAAS,EAAE3B,aAAa,EAAEQ,kBAAmB,CAAC;IAC9E,CAAE,CAAC;EACJ;AACD,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA,MAAMoB,wBAAwB,GAAGA,CAAEL,IAAI,EAAEvB,aAAa,KAAM;EAC3D,IAAK,CAAEA,aAAa,IAAIA,aAAa,CAACO,MAAM,KAAK,CAAC,EAAG;IACpD;EACD;EACA,MAAMC,kBAAkB,GAAGT,wBAAwB,CAAEC,aAAc,CAAC;EAEpEsB,6BAA6B,CAAEC,IAAI,EAAEvB,aAAa,EAAEQ,kBAAmB,CAAC;AACzE,CAAC;AAAC,IAAAqB,QAAA,GAAAzB,OAAA,CAAA0B,OAAA,GAEaF,wBAAwB","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/mergeListItems.js

@@ -0,0 +1,30 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.mergeListItems = mergeListItems;
+/**
+ * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.
+ * That way, lists with single words don't result in a skewed keyphrase distribution result.
+ *
+ * @param {string}  text    The text in which to remove the list structures.
+ *
+ * @returns {string} The text with all list structures removed.
+ */
+function mergeListItems(text) {
+  const listTags = /<\/?(o|ul)(?:[^>]+)?>/g;
+  const listItemTags = /\s?<\/?li(?:[^>]+)?>\s?/g;
+
+  /*
+   * Steps:
+   * 1) Remove all ul and ol tags.
+   * 2) Replace all li tags with spaces to avoid list items being stuck together.
+   * 3) Replace multiple instances of whitespace with a single instance.
+   */
+  text = text.replace(listTags, "");
+  text = text.replace(listItemTags, " ");
+  text = text.replace(/\s+/g, " ");
+  return text;
+}
+//# sourceMappingURL=mergeListItems.js.map

package/build/cjs/languageProcessing/helpers/sanitize/mergeListItems.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mergeListItems.js","names":["mergeListItems","text","listTags","listItemTags","replace"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/mergeListItems.js"],"sourcesContent":["/**\n * Removes list-related html tags in a text to be used for the keyphrase distribution assessment.\n * That way, lists with single words don't result in a skewed keyphrase distribution result.\n *\n * @param {string}  text    The text in which to remove the list structures.\n *\n * @returns {string} The text with all list structures removed.\n */\nexport function mergeListItems( text ) {\n\tconst listTags = /<\\/?(o|ul)(?:[^>]+)?>/g;\n\tconst listItemTags = /\\s?<\\/?li(?:[^>]+)?>\\s?/g;\n\n\t/*\n\t * Steps:\n\t * 1) Remove all ul and ol tags.\n\t * 2) Replace all li tags with spaces to avoid list items being stuck together.\n\t * 3) Replace multiple instances of whitespace with a single instance.\n\t */\n\ttext = text.replace( listTags, \"\" );\n\ttext = text.replace( listItemTags, \" \" );\n\ttext = text.replace( /\\s+/g, \" \" );\n\n\treturn text;\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASA,cAAcA,CAAEC,IAAI,EAAG;EACtC,MAAMC,QAAQ,GAAG,wBAAwB;EACzC,MAAMC,YAAY,GAAG,0BAA0B;;EAE/C;AACD;AACA;AACA;AACA;AACA;EACCF,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAEF,QAAQ,EAAE,EAAG,CAAC;EACnCD,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAED,YAAY,EAAE,GAAI,CAAC;EACxCF,IAAI,GAAGA,IAAI,CAACG,OAAO,CAAE,MAAM,EAAE,GAAI,CAAC;EAElC,OAAOH,IAAI;AACZ","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/parseSynonyms.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _stripSpaces = _interopRequireDefault(require("./stripSpaces.js"));
+var _removePunctuationExceptQuotes = _interopRequireDefault(require("./removePunctuationExceptQuotes.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/** @module stringProcessing/parseSynonyms */
+
+/**
+ * Parses synonyms from a comma-separated string into an array.
+ *
+ * @param {String} synonyms The text to match
+ *
+ * @returns {Array} An array with all synonyms.
+ */
+function _default(synonyms) {
+  let synonymsSplit = synonyms.split(",");
+  synonymsSplit = synonymsSplit.map(synonym => (0, _removePunctuationExceptQuotes.default)((0, _stripSpaces.default)(synonym))).filter(synonym => synonym);
+  return synonymsSplit;
+}
+//# sourceMappingURL=parseSynonyms.js.map

package/build/cjs/languageProcessing/helpers/sanitize/parseSynonyms.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseSynonyms.js","names":["_stripSpaces","_interopRequireDefault","require","_removePunctuationExceptQuotes","e","__esModule","default","_default","synonyms","synonymsSplit","split","map","synonym","removePunctuationExceptQuotes","stripSpaces","filter"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/parseSynonyms.js"],"sourcesContent":["/** @module stringProcessing/parseSynonyms */\n\nimport stripSpaces from \"./stripSpaces.js\";\nimport removePunctuationExceptQuotes from \"./removePunctuationExceptQuotes.js\";\n\n/**\n * Parses synonyms from a comma-separated string into an array.\n *\n * @param {String} synonyms The text to match\n *\n * @returns {Array} An array with all synonyms.\n */\nexport default function( synonyms ) {\n\tlet synonymsSplit = synonyms.split( \",\" );\n\n\tsynonymsSplit = synonymsSplit.map( synonym =>\n\t\tremovePunctuationExceptQuotes( stripSpaces( synonym ) ) ).filter( synonym => synonym );\n\n\treturn synonymsSplit;\n}\n"],"mappings":";;;;;;AAEA,IAAAA,YAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,8BAAA,GAAAF,sBAAA,CAAAC,OAAA;AAA+E,SAAAD,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAH/E;;AAKA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,QAAQ,EAAG;EACnC,IAAIC,aAAa,GAAGD,QAAQ,CAACE,KAAK,CAAE,GAAI,CAAC;EAEzCD,aAAa,GAAGA,aAAa,CAACE,GAAG,CAAEC,OAAO,IACzC,IAAAC,sCAA6B,EAAE,IAAAC,oBAAW,EAAEF,OAAQ,CAAE,CAAE,CAAC,CAACG,MAAM,CAAEH,OAAO,IAAIA,OAAQ,CAAC;EAEvF,OAAOH,aAAa;AACrB","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/quotes.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = exports.SINGLE_QUOTES_REGEX = exports.SINGLE_QUOTES_ARRAY = void 0;
+exports.normalize = normalizeQuotes;
+exports.normalizeDouble = normalizeDoubleQuotes;
+exports.normalizeSingle = normalizeSingleQuotes;
+const SINGLE_QUOTES_ARRAY = exports.SINGLE_QUOTES_ARRAY = ["'", "‘", "’", "‛", "`", "‹", "›"];
+const SINGLE_QUOTES_REGEX = exports.SINGLE_QUOTES_REGEX = new RegExp("[" + SINGLE_QUOTES_ARRAY.join("") + "]", "g");
+
+/**
+ * Normalizes single quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeSingleQuotes(text) {
+  return text.replace(SINGLE_QUOTES_REGEX, "'");
+}
+
+/**
+ * Normalizes double quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeDoubleQuotes(text) {
+  return text.replace(/[“”〝〞〟‟„『』«»]/g, "\"");
+}
+
+/**
+ * Normalizes quotes to 'regular' quotes.
+ *
+ * @param {string} text Text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeQuotes(text) {
+  return normalizeDoubleQuotes(normalizeSingleQuotes(text));
+}
+var _default = exports.default = {
+  normalizeSingle: normalizeSingleQuotes,
+  normalizeDouble: normalizeDoubleQuotes,
+  normalize: normalizeQuotes
+};
+//# sourceMappingURL=quotes.js.map

package/build/cjs/languageProcessing/helpers/sanitize/quotes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quotes.js","names":["SINGLE_QUOTES_ARRAY","exports","SINGLE_QUOTES_REGEX","RegExp","join","normalizeSingleQuotes","text","replace","normalizeDoubleQuotes","normalizeQuotes","_default","default","normalizeSingle","normalizeDouble","normalize"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/quotes.js"],"sourcesContent":["const SINGLE_QUOTES_ARRAY = [ \"'\", \"‘\", \"’\", \"‛\", \"`\", \"‹\", \"›\" ];\nconst SINGLE_QUOTES_REGEX = new RegExp( \"[\" + SINGLE_QUOTES_ARRAY.join( \"\" ) + \"]\", \"g\" );\n\n/**\n * Normalizes single quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeSingleQuotes( text ) {\n\treturn text.replace( SINGLE_QUOTES_REGEX, \"'\" );\n}\n\n/**\n * Normalizes double quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeDoubleQuotes( text ) {\n\treturn text.replace( /[“”〝〞〟‟„『』«»]/g, \"\\\"\" );\n}\n\n/**\n * Normalizes quotes to 'regular' quotes.\n *\n * @param {string} text Text to normalize.\n * @returns {string} The normalized text.\n */\nfunction normalizeQuotes( text ) {\n\treturn normalizeDoubleQuotes( normalizeSingleQuotes( text ) );\n}\n\nexport {\n\tnormalizeSingleQuotes as normalizeSingle,\n\tnormalizeDoubleQuotes as normalizeDouble,\n\tnormalizeQuotes as normalize,\n\tSINGLE_QUOTES_REGEX,\n\tSINGLE_QUOTES_ARRAY,\n};\n\nexport default {\n\tnormalizeSingle: normalizeSingleQuotes,\n\tnormalizeDouble: normalizeDoubleQuotes,\n\tnormalize: normalizeQuotes,\n};\n"],"mappings":";;;;;;;;;AAAA,MAAMA,mBAAmB,GAAAC,OAAA,CAAAD,mBAAA,GAAG,CAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAE;AACjE,MAAME,mBAAmB,GAAAD,OAAA,CAAAC,mBAAA,GAAG,IAAIC,MAAM,CAAE,GAAG,GAAGH,mBAAmB,CAACI,IAAI,CAAE,EAAG,CAAC,GAAG,GAAG,EAAE,GAAI,CAAC;;AAEzF;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,qBAAqBA,CAAEC,IAAI,EAAG;EACtC,OAAOA,IAAI,CAACC,OAAO,CAAEL,mBAAmB,EAAE,GAAI,CAAC;AAChD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,SAASM,qBAAqBA,CAAEF,IAAI,EAAG;EACtC,OAAOA,IAAI,CAACC,OAAO,CAAE,gBAAgB,EAAE,IAAK,CAAC;AAC9C;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,SAASE,eAAeA,CAAEH,IAAI,EAAG;EAChC,OAAOE,qBAAqB,CAAEH,qBAAqB,CAAEC,IAAK,CAAE,CAAC;AAC9D;AAAC,IAAAI,QAAA,GAAAT,OAAA,CAAAU,OAAA,GAUc;EACdC,eAAe,EAAEP,qBAAqB;EACtCQ,eAAe,EAAEL,qBAAqB;EACtCM,SAAS,EAAEL;AACZ,CAAC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/removeEmailAddresses.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+const emailRegex = new RegExp("[^\\s@]+@[^\\s@]+\\.[^\\s@]+", "igm");
+
+/**
+ * Removes email addresses from a text.
+ *
+ * @param {string} text The text to remove emails from.
+ *
+ * @returns {string} The text without email addresses.
+ */
+function _default(text) {
+  return text.replace(emailRegex, "");
+}
+//# sourceMappingURL=removeEmailAddresses.js.map

package/build/cjs/languageProcessing/helpers/sanitize/removeEmailAddresses.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removeEmailAddresses.js","names":["emailRegex","RegExp","_default","text","replace"],"sources":["../../../../../src/languageProcessing/helpers/sanitize/removeEmailAddresses.js"],"sourcesContent":["const emailRegex = new RegExp( \"[^\\\\s@]+@[^\\\\s@]+\\\\.[^\\\\s@]+\", \"igm\" );\n\n/**\n * Removes email addresses from a text.\n *\n * @param {string} text The text to remove emails from.\n *\n * @returns {string} The text without email addresses.\n */\nexport default function( text ) {\n\treturn text.replace( emailRegex, \"\" );\n}\n"],"mappings":";;;;;;AAAA,MAAMA,UAAU,GAAG,IAAIC,MAAM,CAAE,8BAA8B,EAAE,KAAM,CAAC;;AAEtE;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAAUC,IAAI,EAAG;EAC/B,OAAOA,IAAI,CAACC,OAAO,CAAEJ,UAAU,EAAE,EAAG,CAAC;AACtC","ignoreList":[]}

package/build/cjs/languageProcessing/helpers/sanitize/removePunctuation.js

@@ -0,0 +1,64 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+exports.punctuationRegexString = exports.punctuationRegexStart = exports.punctuationRegexEnd = exports.punctuationList = void 0;
+var _unifyWhitespace = require("./unifyWhitespace");
+/**
+ * String containing all the characters that we consider punctuation.
+ * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
+ * @type {string}
+ */
+const punctuationRegexString = exports.punctuationRegexString = "\\–\\-\\(\\)_\\[\\]’‘“”〝〞〟‟„\"'.?!:;,¿¡«»‹›\u2014\u00d7\u002b\u0026\u06d4\u061f\u060C\u061B\u3002\uff61" + "\uff01\u203c\uff1f\u2047\u2049\u2048\u2025\u2026\u30fb\u30fc\u3001\u3003\u3004\u3006\u3007\u3008\u3009\u300a\u300b\u300c\u300d\u300e" + "\u300f\u3010\u3011\u3012\u3013\u3014\u3015\u3016\u3017\u3018\u3019\u301a\u301b\u301c\u301d\u301e\u301f\u3020\u3036\u303c\u303d\uff5b" + "\uff5d\uff5c\uff5e\uff5f\uff60\uff62\uff63\uff64\uff3b\uff3d\uff65\uffe5\uff04\uff05\uff20\uff06\uff07\uff08\uff09\uff0a\uff0f\uff1a" + "\uff1b\uff1c\uff1e\uff3c\\<>";
+
+/**
+ * Array containing all the characters that we consider punctuation.
+ * Characters that can also be used as control characters in regular expressions (like `-` and `(`) are escaped.
+ * @type {string[]}
+ */
+const punctuationList = exports.punctuationList = punctuationRegexString.split("");
+const punctuationRegexStart = exports.punctuationRegexStart = new RegExp("^[" + punctuationRegexString + "]+");
+const punctuationRegexEnd = exports.punctuationRegexEnd = new RegExp("[" + punctuationRegexString + "]+$");
+
+/*
+ * \u2014 - em-dash
+ * \u00d7 - multiplication sign
+ * \u002b - plus sign
+ * \u0026 - ampersand
+ * \u06d4 - Urdu full stop
+ * \u061f - Arabic question mark
+ * \u060C - Arabic comma
+ * \u061B - Arabic semicolon
+ */
+
+/**
+ * Replaces punctuation characters from the given text string.
+ *
+ * @param {String} text The text to remove the punctuation characters for.
+ *
+ * @returns {String} The sanitized text.
+ */
+function _default(text) {
+  // Unify whitespaces and non-breaking spaces.
+  text = (0, _unifyWhitespace.unifyNonBreakingSpace)(text);
+
+  // Remove &amp from the string. In some editors (a.o. Block and Elementor) the ampersand (&) is transformed into &amp.
+  // If it is not removed, then it is returned as "amp" and counted as a word in assessments downstream.
+  text = text.replace("\u0026amp", "");
+
+  /*
+   * Remove backslash from the beginning and end of a word/text.
+   * When a string such as `This is a \"calico\" cat` enters the Paper,
+   * the Paper adds two extra backslash in front of the original backslash.
+   * After the text is split into words, we also need to remove those backslashes from the word.
+   * Otherwise, it will be problematic when word boundary regex is added to the word.
+   */
+  const backslashRegex = new RegExp("(\\\\)", "g");
+  text = text.replace(backslashRegex, "");
+  text = text.replace(punctuationRegexStart, "");
+  text = text.replace(punctuationRegexEnd, "");
+  return text;
+}
+//# sourceMappingURL=removePunctuation.js.map