axyseo 2.0.0-alpha.0.0.2 → 2.0.0-alpha.0.0.21

package/build/languageProcessing/helpers/word/indices.js

@@ -0,0 +1,141 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.filterIndices = exports.default = void 0;
+exports.getIndicesByWord = getIndicesByWord;
+exports.sortIndices = exports.getIndicesByWordListSorted = exports.getIndicesByWordList = void 0;
+var _lodash = require("lodash");
+var _stripSpaces = _interopRequireDefault(require("../sanitize/stripSpaces.js"));
+var _matchWordInSentence = require("./matchWordInSentence.js");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Returns the indices of a string in a text. If it is found multiple times, it will return multiple indices.
+ *
+ * @param {string} word The word to find in the text.
+ * @param {string} text The text to check for the given word.
+ *
+ * @returns {Array} All indices found.
+ */
+function getIndicesByWord(word, text) {
+  let startIndex = 0;
+  const searchStringLength = word.length;
+  let index;
+  const indices = [];
+  while ((index = text.indexOf(word, startIndex)) > -1) {
+    // Check if the previous and next character are word boundaries to determine if a complete word was detected
+    const isPreviousCharacterWordBoundary = (0, _matchWordInSentence.characterInBoundary)(text[index - 1]) || index === 0;
+    const isNextCharacterWordBoundary = (0, _matchWordInSentence.characterInBoundary)(text[index + searchStringLength]) || text.length === index + searchStringLength;
+    if (isPreviousCharacterWordBoundary && isNextCharacterWordBoundary) {
+      indices.push({
+        index: index,
+        match: word
+      });
+    }
+    startIndex = index + searchStringLength;
+  }
+  return indices;
+}
+
+/**
+ * Matches string with an array, returns the word and the index it was found on.
+ *
+ * @param {Array} words The array with strings to match.
+ * @param {string} text The text to match the strings from the array to.
+ *
+ * @returns {Array} The array with words, containing the index of the match and the matched string.
+ * Returns an empty array if none are found.
+ */
+const getIndicesByWordList = function (words, text) {
+  let matchedWords = [];
+  (0, _lodash.forEach)(words, function (word) {
+    word = (0, _stripSpaces.default)(word);
+    if (!(0, _matchWordInSentence.isWordInSentence)(word, text)) {
+      return;
+    }
+    matchedWords = matchedWords.concat(getIndicesByWord(word, text));
+  });
+  return matchedWords;
+};
+
+/**
+ * Sorts the array on the index property of each entry.
+ *
+ * @param {Array} indices The array with indices.
+ *
+ * @returns {Array} The sorted array with indices.
+ */
+exports.getIndicesByWordList = getIndicesByWordList;
+const sortIndices = function (indices) {
+  return indices.sort(function (a, b) {
+    return a.index - b.index;
+  });
+};
+
+/**
+ * Filters duplicate entries if the indices overlap.
+ *
+ * @param {Array} indices The array with indices to be filtered.
+ *
+ * @returns {Array} The filtered array.
+ */
+exports.sortIndices = sortIndices;
+const filterIndices = function (indices) {
+  indices = sortIndices(indices);
+  const filtered = [];
+  for (let i = 0; i < indices.length; i++) {
+    // If the next index is within the range of the current index and the length of the word, remove it
+    // This makes sure we don't match combinations twice, like "even though" and "though".
+    if (!(0, _lodash.isUndefined)(indices[i + 1]) && indices[i + 1].index < indices[i].index + indices[i].match.length) {
+      filtered.push(indices[i]);
+
+      // Adds 1 to i, so we skip the next index that is overlapping with the current index.
+      i++;
+      continue;
+    }
+    filtered.push(indices[i]);
+  }
+  return filtered;
+};
+
+/**
+ * Matches string with an array, returns the word and the index it was found on, and sorts the match instances based on
+ * the index property of the match.
+ *
+ * @param {Array} words The array with strings to match.
+ * @param {string} text The text to match the strings from the array to.
+ *
+ * @returns {Array} The array with words, containing the index of the match and the matched string.
+ * Returns an empty array if none are found.
+ */
+exports.filterIndices = filterIndices;
+const getIndicesByWordListSorted = function (words, text) {
+  let matchedWords = [];
+  (0, _lodash.forEach)(words, function (word) {
+    word = (0, _stripSpaces.default)(word);
+    if (!(0, _matchWordInSentence.isWordInSentence)(word, text)) {
+      return matchedWords;
+    }
+    matchedWords = matchedWords.concat(getIndicesByWord(word, text));
+  });
+  matchedWords = matchedWords.sort(function (a, b) {
+    if (a.index < b.index) {
+      return -1;
+    }
+    if (a.index > b.index) {
+      return 1;
+    }
+    return 0;
+  });
+  return matchedWords;
+};
+exports.getIndicesByWordListSorted = getIndicesByWordListSorted;
+var _default = exports.default = {
+  getIndicesByWord,
+  getIndicesByWordList,
+  filterIndices,
+  sortIndices,
+  getIndicesByWordListSorted
+};
+//# sourceMappingURL=indices.js.map

package/build/languageProcessing/helpers/word/indices.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indices.js","names":["_lodash","require","_stripSpaces","_interopRequireDefault","_matchWordInSentence","e","__esModule","default","getIndicesByWord","word","text","startIndex","searchStringLength","length","index","indices","indexOf","isPreviousCharacterWordBoundary","characterInBoundary","isNextCharacterWordBoundary","push","match","getIndicesByWordList","words","matchedWords","forEach","stripSpaces","matchWordInSentence","concat","exports","sortIndices","sort","a","b","filterIndices","filtered","i","isUndefined","getIndicesByWordListSorted","_default"],"sources":["../../../../src/languageProcessing/helpers/word/indices.js"],"sourcesContent":["import { isUndefined, forEach } from \"lodash\";\nimport stripSpaces from \"../sanitize/stripSpaces.js\";\nimport { isWordInSentence as matchWordInSentence, characterInBoundary } from \"./matchWordInSentence.js\";\n\n/**\n * Returns the indices of a string in a text. If it is found multiple times, it will return multiple indices.\n *\n * @param {string} word The word to find in the text.\n * @param {string} text The text to check for the given word.\n *\n * @returns {Array} All indices found.\n */\nfunction getIndicesByWord( word, text ) {\n\tlet startIndex = 0;\n\tconst searchStringLength = word.length;\n\tlet index;\n\tconst indices = [];\n\twhile ( ( index = text.indexOf( word, startIndex ) ) > -1 ) {\n\t\t// Check if the previous and next character are word boundaries to determine if a complete word was detected\n\t\tconst isPreviousCharacterWordBoundary = characterInBoundary( text[ index - 1 ] ) || index === 0;\n\n\t\tconst isNextCharacterWordBoundary = characterInBoundary( text[ index + searchStringLength ] ) ||\n\t\t\t( text.length === index + searchStringLength );\n\n\t\tif ( isPreviousCharacterWordBoundary && isNextCharacterWordBoundary ) {\n\t\t\tindices.push(\n\t\t\t\t{\n\t\t\t\t\tindex: index,\n\t\t\t\t\tmatch: word,\n\t\t\t\t}\n\t\t\t);\n\t\t}\n\t\tstartIndex = index + searchStringLength;\n\t}\n\treturn indices;\n}\n\n/**\n * Matches string with an array, returns the word and the index it was found on.\n *\n * @param {Array} words The array with strings to match.\n * @param {string} text The text to match the strings from the array to.\n *\n * @returns {Array} The array with words, containing the index of the match and the matched string.\n * Returns an empty array if none are found.\n */\nconst getIndicesByWordList = function( words, text ) {\n\tlet matchedWords = [];\n\n\tforEach( words, function( word ) {\n\t\tword = stripSpaces( word );\n\t\tif ( ! matchWordInSentence( word, text ) ) {\n\t\t\treturn;\n\t\t}\n\t\tmatchedWords = matchedWords.concat( getIndicesByWord( word, text ) );\n\t} );\n\treturn matchedWords;\n};\n\n/**\n * Sorts the array on the index property of each entry.\n *\n * @param {Array} indices The array with indices.\n *\n * @returns {Array} The sorted array with indices.\n */\nconst sortIndices = function( indices ) {\n\treturn indices.sort( function( a, b ) {\n\t\treturn ( a.index - b.index );\n\t} );\n};\n\n/**\n * Filters duplicate entries if the indices overlap.\n *\n * @param {Array} indices The array with indices to be filtered.\n *\n * @returns {Array} The filtered array.\n */\nconst filterIndices = function( indices ) {\n\tindices = sortIndices( indices );\n\tconst filtered = [];\n\tfor ( let i = 0; i < indices.length; i++ ) {\n\t\t// If the next index is within the range of the current index and the length of the word, remove it\n\t\t// This makes sure we don't match combinations twice, like \"even though\" and \"though\".\n\t\tif ( ! isUndefined( indices[ i + 1 ] ) && indices[ i + 1 ].index < indices[ i ].index + indices[ i ].match.length ) {\n\t\t\tfiltered.push( indices[ i ] );\n\n\t\t\t// Adds 1 to i, so we skip the next index that is overlapping with the current index.\n\t\t\ti++;\n\t\t\tcontinue;\n\t\t}\n\t\tfiltered.push( indices[ i ] );\n\t}\n\treturn filtered;\n};\n\n/**\n * Matches string with an array, returns the word and the index it was found on, and sorts the match instances based on\n * the index property of the match.\n *\n * @param {Array} words The array with strings to match.\n * @param {string} text The text to match the strings from the array to.\n *\n * @returns {Array} The array with words, containing the index of the match and the matched string.\n * Returns an empty array if none are found.\n */\nconst getIndicesByWordListSorted = function( words, text ) {\n\tlet matchedWords = [];\n\n\tforEach( words, function( word ) {\n\t\tword = stripSpaces( word );\n\t\tif ( ! matchWordInSentence( word, text ) ) {\n\t\t\treturn matchedWords;\n\t\t}\n\t\tmatchedWords = matchedWords.concat( getIndicesByWord( word, text ) );\n\t} );\n\n\tmatchedWords = matchedWords.sort( function( a, b ) {\n\t\tif ( a.index < b.index ) {\n\t\t\treturn -1;\n\t\t}\n\t\tif ( a.index > b.index ) {\n\t\t\treturn 1;\n\t\t}\n\t\treturn 0;\n\t} );\n\n\treturn matchedWords;\n};\n\nexport {\n\tgetIndicesByWord,\n\tgetIndicesByWordList,\n\tfilterIndices,\n\tsortIndices,\n\tgetIndicesByWordListSorted,\n};\n\nexport default {\n\tgetIndicesByWord,\n\tgetIndicesByWordList,\n\tfilterIndices,\n\tsortIndices,\n\tgetIndicesByWordListSorted,\n};\n"],"mappings":";;;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,oBAAA,GAAAH,OAAA;AAAwG,SAAAE,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAExG;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASG,gBAAgBA,CAAEC,IAAI,EAAEC,IAAI,EAAG;EACvC,IAAIC,UAAU,GAAG,CAAC;EAClB,MAAMC,kBAAkB,GAAGH,IAAI,CAACI,MAAM;EACtC,IAAIC,KAAK;EACT,MAAMC,OAAO,GAAG,EAAE;EAClB,OAAQ,CAAED,KAAK,GAAGJ,IAAI,CAACM,OAAO,CAAEP,IAAI,EAAEE,UAAW,CAAC,IAAK,CAAC,CAAC,EAAG;IAC3D;IACA,MAAMM,+BAA+B,GAAG,IAAAC,wCAAmB,EAAER,IAAI,CAAEI,KAAK,GAAG,CAAC,CAAG,CAAC,IAAIA,KAAK,KAAK,CAAC;IAE/F,MAAMK,2BAA2B,GAAG,IAAAD,wCAAmB,EAAER,IAAI,CAAEI,KAAK,GAAGF,kBAAkB,CAAG,CAAC,IAC1FF,IAAI,CAACG,MAAM,KAAKC,KAAK,GAAGF,kBAAoB;IAE/C,IAAKK,+BAA+B,IAAIE,2BAA2B,EAAG;MACrEJ,OAAO,CAACK,IAAI,CACX;QACCN,KAAK,EAAEA,KAAK;QACZO,KAAK,EAAEZ;MACR,CACD,CAAC;IACF;IACAE,UAAU,GAAGG,KAAK,GAAGF,kBAAkB;EACxC;EACA,OAAOG,OAAO;AACf;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMO,oBAAoB,GAAG,SAAAA,CAAUC,KAAK,EAAEb,IAAI,EAAG;EACpD,IAAIc,YAAY,GAAG,EAAE;EAErB,IAAAC,eAAO,EAAEF,KAAK,EAAE,UAAUd,IAAI,EAAG;IAChCA,IAAI,GAAG,IAAAiB,oBAAW,EAAEjB,IAAK,CAAC;IAC1B,IAAK,CAAE,IAAAkB,qCAAmB,EAAElB,IAAI,EAAEC,IAAK,CAAC,EAAG;MAC1C;IACD;IACAc,YAAY,GAAGA,YAAY,CAACI,MAAM,CAAEpB,gBAAgB,CAAEC,IAAI,EAAEC,IAAK,CAAE,CAAC;EACrE,CAAE,CAAC;EACH,OAAOc,YAAY;AACpB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AANAK,OAAA,CAAAP,oBAAA,GAAAA,oBAAA;AAOA,MAAMQ,WAAW,GAAG,SAAAA,CAAUf,OAAO,EAAG;EACvC,OAAOA,OAAO,CAACgB,IAAI,CAAE,UAAUC,CAAC,EAAEC,CAAC,EAAG;IACrC,OAASD,CAAC,CAAClB,KAAK,GAAGmB,CAAC,CAACnB,KAAK;EAC3B,CAAE,CAAC;AACJ,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AANAe,OAAA,CAAAC,WAAA,GAAAA,WAAA;AAOA,MAAMI,aAAa,GAAG,SAAAA,CAAUnB,OAAO,EAAG;EACzCA,OAAO,GAAGe,WAAW,CAAEf,OAAQ,CAAC;EAChC,MAAMoB,QAAQ,GAAG,EAAE;EACnB,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGrB,OAAO,CAACF,MAAM,EAAEuB,CAAC,EAAE,EAAG;IAC1C;IACA;IACA,IAAK,CAAE,IAAAC,mBAAW,EAAEtB,OAAO,CAAEqB,CAAC,GAAG,CAAC,CAAG,CAAC,IAAIrB,OAAO,CAAEqB,CAAC,GAAG,CAAC,CAAE,CAACtB,KAAK,GAAGC,OAAO,CAAEqB,CAAC,CAAE,CAACtB,KAAK,GAAGC,OAAO,CAAEqB,CAAC,CAAE,CAACf,KAAK,CAACR,MAAM,EAAG;MACnHsB,QAAQ,CAACf,IAAI,CAAEL,OAAO,CAAEqB,CAAC,CAAG,CAAC;;MAE7B;MACAA,CAAC,EAAE;MACH;IACD;IACAD,QAAQ,CAACf,IAAI,CAAEL,OAAO,CAAEqB,CAAC,CAAG,CAAC;EAC9B;EACA,OAAOD,QAAQ;AAChB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AATAN,OAAA,CAAAK,aAAA,GAAAA,aAAA;AAUA,MAAMI,0BAA0B,GAAG,SAAAA,CAAUf,KAAK,EAAEb,IAAI,EAAG;EAC1D,IAAIc,YAAY,GAAG,EAAE;EAErB,IAAAC,eAAO,EAAEF,KAAK,EAAE,UAAUd,IAAI,EAAG;IAChCA,IAAI,GAAG,IAAAiB,oBAAW,EAAEjB,IAAK,CAAC;IAC1B,IAAK,CAAE,IAAAkB,qCAAmB,EAAElB,IAAI,EAAEC,IAAK,CAAC,EAAG;MAC1C,OAAOc,YAAY;IACpB;IACAA,YAAY,GAAGA,YAAY,CAACI,MAAM,CAAEpB,gBAAgB,CAAEC,IAAI,EAAEC,IAAK,CAAE,CAAC;EACrE,CAAE,CAAC;EAEHc,YAAY,GAAGA,YAAY,CAACO,IAAI,CAAE,UAAUC,CAAC,EAAEC,CAAC,EAAG;IAClD,IAAKD,CAAC,CAAClB,KAAK,GAAGmB,CAAC,CAACnB,KAAK,EAAG;MACxB,OAAO,CAAC,CAAC;IACV;IACA,IAAKkB,CAAC,CAAClB,KAAK,GAAGmB,CAAC,CAACnB,KAAK,EAAG;MACxB,OAAO,CAAC;IACT;IACA,OAAO,CAAC;EACT,CAAE,CAAC;EAEH,OAAOU,YAAY;AACpB,CAAC;AAACK,OAAA,CAAAS,0BAAA,GAAAA,0BAAA;AAAA,IAAAC,QAAA,GAAAV,OAAA,CAAAtB,OAAA,GAUa;EACdC,gBAAgB;EAChBc,oBAAoB;EACpBY,aAAa;EACbJ,WAAW;EACXQ;AACD,CAAC","ignoreList":[]}

package/build/languageProcessing/helpers/word/markWordsInSentences.js

@@ -0,0 +1,189 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.deConstructAnchor = exports.collectMarkingsInSentence = void 0;
+exports.markWordsInASentence = markWordsInASentence;
+exports.markWordsInSentences = markWordsInSentences;
+exports.reConstructAnchor = void 0;
+var _matchTextWithArray = _interopRequireDefault(require("../match/matchTextWithArray"));
+var _createRegexFromArray = _interopRequireDefault(require("../regex/createRegexFromArray"));
+var _addMarkSingleWord = _interopRequireDefault(require("../../../markers/addMarkSingleWord"));
+var _Mark = _interopRequireDefault(require("../../../values/Mark"));
+var _lodash = require("lodash");
+var _getAnchorsFromText = _interopRequireDefault(require("../link/getAnchorsFromText"));
+var _quotes = require("../sanitize/quotes");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+// Regex to deconstruct an anchor into open tag, content and close tag.
+const anchorDeconstructionRegex = /(<a[\s]+[^>]+>)([^]*?)(<\/a>)/;
+
+/**
+ * Deconstructs an anchor to the opening tag and the content. The content is the anchor text.
+ * We don't return the closing tag since the value would always be the same, i.e. </a>.
+ *
+ * @param {string} anchor An anchor of the shape <a ...>...</a>.
+ *
+ * @returns {object} An object containing the opening tag and the content.
+ */
+const deConstructAnchor = function (anchor) {
+  // The const array mirrors the anchorDeconstructionRegex, using a comma to access the first element without a name.
+  const [, openTag, content] = anchor.match(anchorDeconstructionRegex);
+  return {
+    openTag: openTag,
+    content: content
+  };
+};
+
+/**
+ * Reconstructs an anchor from an openTag, the content, and the closing tag.
+ *
+ * @param {string} openTag The opening tag of the anchor. Must be of the shape <a ...>.
+ * @param {string} content The text of the anchor.
+ *
+ * @returns {string} An anchor.
+ */
+exports.deConstructAnchor = deConstructAnchor;
+const reConstructAnchor = function (openTag, content) {
+  return `${openTag}${content}</a>`;
+};
+
+/**
+ * Gets the anchors and marks the anchors' text if the words are found in it.
+ *
+ * @param {string} sentence The sentence to retrieve the anchors from.
+ * @param {RegExp} wordsRegex The regex of the words.
+ *
+ * @returns {Object} The anchors and the marked anchors.
+ */
+exports.reConstructAnchor = reConstructAnchor;
+const getMarkedAnchors = function (sentence, wordsRegex) {
+  // Retrieve the anchors.
+  const anchors = (0, _getAnchorsFromText.default)(sentence);
+  // For every anchor, apply the markings only to the anchor tag.
+  const markedAnchors = anchors.map(anchor => {
+    // Retrieve the open tag and the content/anchor text.
+    const {
+      openTag,
+      content
+    } = deConstructAnchor(anchor);
+
+    // Apply the marking to the anchor text if there is a match.
+    const markedAnchorText = content.replace(wordsRegex, x => (0, _addMarkSingleWord.default)(x));
+
+    // Create a new anchor tag with a (marked) anchor text.
+    return reConstructAnchor(openTag, markedAnchorText);
+  });
+  return {
+    anchors,
+    markedAnchors
+  };
+};
+
+/**
+ * Adds marks to a sentence and merges marks if those are only separated by a space
+ * (e.g., if highlighting words "ballet" and "shoes" in a sentence "I have a lot of ballet shoes and other paraphernalia."
+ * the marks will be put around "ballet shoes" together, not "`ballet` `shoes`".)
+ *
+ * @param {string}    sentence               The sentence to mark words in.
+ * @param {[string]}  wordsFoundInSentence   The words to mark in the sentence.
+ * @param {function}  matchWordCustomHelper  The language-specific helper function to match word in text.
+ *
+ * @returns {string} The sentence with marks.
+ */
+const collectMarkingsInSentence = function (sentence, wordsFoundInSentence, matchWordCustomHelper) {
+  const allWordsFound = [];
+  wordsFoundInSentence.forEach(word => {
+    // Check if the word in `wordsFoundInSentence` contains single quote(s).
+    const matchedSingleQuotes = word.match(_quotes.SINGLE_QUOTES_REGEX);
+    if (matchedSingleQuotes) {
+      /*
+       * If yes, make all different combinations of the word with different types of single quotes in the array.
+       * Later, a regex will be created for all words that were found in the sentence including their variations.
+       *
+       * For example:
+       * `wordsFoundInSentence`: [ "red", "panda’s" ]
+       * For the word "panda’s", we'll create the following variations:
+       * "panda's", "panda‘s", "panda’s", "panda‛s", "panda`s", "panda‹s", "panda›s"
+       * And those variations will be added to `allWordsFound`.
+       */
+      _quotes.SINGLE_QUOTES_ARRAY.forEach(singleQuote => {
+        matchedSingleQuotes.forEach(matchedSingleQuote => {
+          allWordsFound.push((0, _lodash.escapeRegExp)(word.replace(new RegExp(matchedSingleQuote, "g"), singleQuote)));
+        });
+      });
+    } else {
+      allWordsFound.push((0, _lodash.escapeRegExp)(word));
+    }
+  });
+
+  // If a language has a custom helper to match words, we disable the word boundary when creating the regex.
+  const wordsRegex = matchWordCustomHelper ? (0, _createRegexFromArray.default)(allWordsFound, true) : (0, _createRegexFromArray.default)(allWordsFound);
+
+  // Retrieve the anchors and mark the anchors' text if the words are found in the anchors' text.
+  const {
+    anchors,
+    markedAnchors
+  } = getMarkedAnchors(sentence, wordsRegex);
+  let markup = sentence.replace(wordsRegex, function (x) {
+    return (0, _addMarkSingleWord.default)(x);
+  });
+
+  /**
+   * In 'markup', we apply the markings also inside the anchor's attribute if there is a match, on top of
+   * marking the anchor's text.
+   * The step below is to replace the incorrectly marked anchors with the marked anchors that we want:
+   * where the markings are only applied in the anchor's text.
+   */
+  if (anchors.length > 0) {
+    const markupAnchors = (0, _getAnchorsFromText.default)(markup);
+    for (let i = 0; i < markupAnchors.length; i++) {
+      markup = markup.replace(markupAnchors[i], markedAnchors[i]);
+    }
+  }
+
+  /*
+   * If two marks are separated by only a space, remove the closing tag of the first mark and the opening tag of the
+   * second mark so that the two marks can be combined into one.
+   */
+  return markup.replace(new RegExp("</yoastmark> <yoastmark class='yoast-text-mark'>", "ig"), " ");
+};
+
+/**
+ * Adds marks to a sentence.
+ *
+ * @param {string}      sentence                The sentence in which we want to apply highlighting.
+ * @param {Array}       wordsFoundInSentence    The words to highlight in a sentence.
+ * @param {function}    matchWordCustomHelper   The language-specific helper function to match word in text.
+ * @returns {Mark[]}  The array of Mark objects of each sentence.
+ */
+exports.collectMarkingsInSentence = collectMarkingsInSentence;
+function markWordsInASentence(sentence, wordsFoundInSentence, matchWordCustomHelper) {
+  return [new _Mark.default({
+    original: sentence,
+    marked: collectMarkingsInSentence(sentence, wordsFoundInSentence, matchWordCustomHelper)
+  })];
+}
+
+/**
+ * Adds marks to an array of sentences.
+ *
+ * @param {[string]}    wordsToMark The words to mark.
+ * @param {[string]}    sentences   The sentences in which to mark these words.
+ * @param {string}      locale      The locale.
+ * @param {function}    matchWordCustomHelper   The language-specific helper function to match word in text.
+ *
+ * @returns {[string]} The sentences with marks.
+ */
+function markWordsInSentences(wordsToMark, sentences, locale, matchWordCustomHelper) {
+  let wordsFoundInSentence = [];
+  let markings = [];
+  sentences.forEach(function (sentence) {
+    wordsFoundInSentence = (0, _matchTextWithArray.default)(sentence, wordsToMark, locale, matchWordCustomHelper).matches;
+    if (wordsFoundInSentence.length > 0) {
+      markings = markings.concat(markWordsInASentence(sentence, wordsFoundInSentence, matchWordCustomHelper));
+    }
+  });
+  return markings;
+}
+//# sourceMappingURL=markWordsInSentences.js.map

package/build/languageProcessing/helpers/word/markWordsInSentences.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markWordsInSentences.js","names":["_matchTextWithArray","_interopRequireDefault","require","_createRegexFromArray","_addMarkSingleWord","_Mark","_lodash","_getAnchorsFromText","_quotes","e","__esModule","default","anchorDeconstructionRegex","deConstructAnchor","anchor","openTag","content","match","exports","reConstructAnchor","getMarkedAnchors","sentence","wordsRegex","anchors","getAnchorsFromText","markedAnchors","map","markedAnchorText","replace","x","addMark","collectMarkingsInSentence","wordsFoundInSentence","matchWordCustomHelper","allWordsFound","forEach","word","matchedSingleQuotes","SINGLE_QUOTES_REGEX","SINGLE_QUOTES_ARRAY","singleQuote","matchedSingleQuote","push","escapeRegExp","RegExp","arrayToRegex","markup","length","markupAnchors","i","markWordsInASentence","Mark","original","marked","markWordsInSentences","wordsToMark","sentences","locale","markings","matchWords","matches","concat"],"sources":["../../../../src/languageProcessing/helpers/word/markWordsInSentences.js"],"sourcesContent":["import matchWords from \"../match/matchTextWithArray\";\nimport arrayToRegex from \"../regex/createRegexFromArray\";\nimport addMark from \"../../../markers/addMarkSingleWord\";\nimport Mark from \"../../../values/Mark\";\nimport { escapeRegExp } from \"lodash\";\nimport getAnchorsFromText from \"../link/getAnchorsFromText\";\nimport { SINGLE_QUOTES_ARRAY, SINGLE_QUOTES_REGEX } from \"../sanitize/quotes\";\n\n// Regex to deconstruct an anchor into open tag, content and close tag.\nconst anchorDeconstructionRegex = /(<a[\\s]+[^>]+>)([^]*?)(<\\/a>)/;\n\n/**\n * Deconstructs an anchor to the opening tag and the content. The content is the anchor text.\n * We don't return the closing tag since the value would always be the same, i.e. </a>.\n *\n * @param {string} anchor An anchor of the shape <a ...>...</a>.\n *\n * @returns {object} An object containing the opening tag and the content.\n */\nexport const deConstructAnchor = function( anchor ) {\n\t// The const array mirrors the anchorDeconstructionRegex, using a comma to access the first element without a name.\n\tconst [ , openTag, content ] = anchor.match( anchorDeconstructionRegex );\n\treturn {\n\t\topenTag: openTag,\n\t\tcontent: content,\n\t};\n};\n\n/**\n * Reconstructs an anchor from an openTag, the content, and the closing tag.\n *\n * @param {string} openTag The opening tag of the anchor. Must be of the shape <a ...>.\n * @param {string} content The text of the anchor.\n *\n * @returns {string} An anchor.\n */\nexport const reConstructAnchor = function( openTag, content ) {\n\treturn `${openTag}${content}</a>`;\n};\n\n\n/**\n * Gets the anchors and marks the anchors' text if the words are found in it.\n *\n * @param {string} sentence The sentence to retrieve the anchors from.\n * @param {RegExp} wordsRegex The regex of the words.\n *\n * @returns {Object} The anchors and the marked anchors.\n */\nconst getMarkedAnchors = function( sentence, wordsRegex ) {\n\t// Retrieve the anchors.\n\tconst anchors = getAnchorsFromText( sentence );\n\t// For every anchor, apply the markings only to the anchor tag.\n\tconst markedAnchors = anchors.map( anchor => {\n\t\t// Retrieve the open tag and the content/anchor text.\n\t\tconst { openTag, content } = deConstructAnchor( anchor );\n\n\t\t// Apply the marking to the anchor text if there is a match.\n\t\tconst markedAnchorText = content.replace( wordsRegex, ( x ) => addMark( x ) );\n\n\t\t// Create a new anchor tag with a (marked) anchor text.\n\t\treturn reConstructAnchor( openTag, markedAnchorText );\n\t} );\n\n\treturn { anchors, markedAnchors };\n};\n\n/**\n * Adds marks to a sentence and merges marks if those are only separated by a space\n * (e.g., if highlighting words \"ballet\" and \"shoes\" in a sentence \"I have a lot of ballet shoes and other paraphernalia.\"\n * the marks will be put around \"ballet shoes\" together, not \"`ballet` `shoes`\".)\n *\n * @param {string}    sentence               The sentence to mark words in.\n * @param {[string]}  wordsFoundInSentence   The words to mark in the sentence.\n * @param {function}  matchWordCustomHelper  The language-specific helper function to match word in text.\n *\n * @returns {string} The sentence with marks.\n */\nexport const collectMarkingsInSentence = function( sentence, wordsFoundInSentence, matchWordCustomHelper ) {\n\tconst allWordsFound = [];\n\twordsFoundInSentence.forEach( word => {\n\t\t// Check if the word in `wordsFoundInSentence` contains single quote(s).\n\t\tconst matchedSingleQuotes = word.match( SINGLE_QUOTES_REGEX );\n\t\tif ( matchedSingleQuotes ) {\n\t\t\t/*\n\t\t\t * If yes, make all different combinations of the word with different types of single quotes in the array.\n\t\t\t * Later, a regex will be created for all words that were found in the sentence including their variations.\n\t\t\t *\n\t\t\t * For example:\n\t\t\t * `wordsFoundInSentence`: [ \"red\", \"panda’s\" ]\n\t\t\t * For the word \"panda’s\", we'll create the following variations:\n\t\t\t * \"panda's\", \"panda‘s\", \"panda’s\", \"panda‛s\", \"panda`s\", \"panda‹s\", \"panda›s\"\n\t\t\t * And those variations will be added to `allWordsFound`.\n\t\t\t */\n\t\t\tSINGLE_QUOTES_ARRAY.forEach( singleQuote => {\n\t\t\t\tmatchedSingleQuotes.forEach( matchedSingleQuote => {\n\t\t\t\t\tallWordsFound.push( escapeRegExp( word.replace( new RegExp( matchedSingleQuote, \"g\" ), singleQuote ) ) );\n\t\t\t\t} );\n\t\t\t} );\n\t\t} else {\n\t\t\tallWordsFound.push( escapeRegExp( word ) );\n\t\t}\n\t} );\n\n\t// If a language has a custom helper to match words, we disable the word boundary when creating the regex.\n\tconst wordsRegex = matchWordCustomHelper ? arrayToRegex( allWordsFound, true ) : arrayToRegex( allWordsFound );\n\n\t// Retrieve the anchors and mark the anchors' text if the words are found in the anchors' text.\n\tconst { anchors, markedAnchors } = getMarkedAnchors( sentence, wordsRegex );\n\n\tlet markup = sentence.replace( wordsRegex, function( x ) {\n\t\treturn addMark( x );\n\t} );\n\n\t/**\n\t * In 'markup', we apply the markings also inside the anchor's attribute if there is a match, on top of\n\t * marking the anchor's text.\n\t * The step below is to replace the incorrectly marked anchors with the marked anchors that we want:\n\t * where the markings are only applied in the anchor's text.\n\t */\n\tif ( anchors.length > 0 ) {\n\t\tconst markupAnchors = getAnchorsFromText( markup );\n\t\tfor ( let i = 0; i < markupAnchors.length; i++ ) {\n\t\t\tmarkup = markup.replace( markupAnchors[ i ], markedAnchors[ i ] );\n\t\t}\n\t}\n\n\t/*\n\t * If two marks are separated by only a space, remove the closing tag of the first mark and the opening tag of the\n\t * second mark so that the two marks can be combined into one.\n\t */\n\treturn ( markup.replace( new RegExp( \"</yoastmark> <yoastmark class='yoast-text-mark'>\", \"ig\" ), \" \" ) );\n};\n\n/**\n * Adds marks to a sentence.\n *\n * @param {string}      sentence                The sentence in which we want to apply highlighting.\n * @param {Array}       wordsFoundInSentence    The words to highlight in a sentence.\n * @param {function}    matchWordCustomHelper   The language-specific helper function to match word in text.\n * @returns {Mark[]}  The array of Mark objects of each sentence.\n */\nexport function markWordsInASentence( sentence, wordsFoundInSentence, matchWordCustomHelper ) {\n\treturn [ new Mark( {\n\t\toriginal: sentence,\n\t\tmarked: collectMarkingsInSentence( sentence, wordsFoundInSentence, matchWordCustomHelper ),\n\t} ) ];\n}\n\n/**\n * Adds marks to an array of sentences.\n *\n * @param {[string]}    wordsToMark The words to mark.\n * @param {[string]}    sentences   The sentences in which to mark these words.\n * @param {string}      locale      The locale.\n * @param {function}    matchWordCustomHelper   The language-specific helper function to match word in text.\n *\n * @returns {[string]} The sentences with marks.\n */\nexport function markWordsInSentences( wordsToMark, sentences, locale, matchWordCustomHelper ) {\n\tlet wordsFoundInSentence = [];\n\tlet markings = [];\n\n\tsentences.forEach( function( sentence ) {\n\t\twordsFoundInSentence = matchWords( sentence, wordsToMark, locale, matchWordCustomHelper ).matches;\n\n\t\tif ( wordsFoundInSentence.length > 0 ) {\n\t\t\tmarkings = markings.concat( markWordsInASentence( sentence, wordsFoundInSentence, matchWordCustomHelper ) );\n\t\t}\n\t} );\n\n\treturn markings;\n}\n"],"mappings":";;;;;;;;;AAAA,IAAAA,mBAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAF,sBAAA,CAAAC,OAAA;AACA,IAAAE,kBAAA,GAAAH,sBAAA,CAAAC,OAAA;AACA,IAAAG,KAAA,GAAAJ,sBAAA,CAAAC,OAAA;AACA,IAAAI,OAAA,GAAAJ,OAAA;AACA,IAAAK,mBAAA,GAAAN,sBAAA,CAAAC,OAAA;AACA,IAAAM,OAAA,GAAAN,OAAA;AAA8E,SAAAD,uBAAAQ,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE9E;AACA,MAAMG,yBAAyB,GAAG,+BAA+B;;AAEjE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,MAAMC,iBAAiB,GAAG,SAAAA,CAAUC,MAAM,EAAG;EACnD;EACA,MAAM,GAAIC,OAAO,EAAEC,OAAO,CAAE,GAAGF,MAAM,CAACG,KAAK,CAAEL,yBAA0B,CAAC;EACxE,OAAO;IACNG,OAAO,EAAEA,OAAO;IAChBC,OAAO,EAAEA;EACV,CAAC;AACF,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAPAE,OAAA,CAAAL,iBAAA,GAAAA,iBAAA;AAQO,MAAMM,iBAAiB,GAAG,SAAAA,CAAUJ,OAAO,EAAEC,OAAO,EAAG;EAC7D,OAAO,GAAGD,OAAO,GAAGC,OAAO,MAAM;AAClC,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAPAE,OAAA,CAAAC,iBAAA,GAAAA,iBAAA;AAQA,MAAMC,gBAAgB,GAAG,SAAAA,CAAUC,QAAQ,EAAEC,UAAU,EAAG;EACzD;EACA,MAAMC,OAAO,GAAG,IAAAC,2BAAkB,EAAEH,QAAS,CAAC;EAC9C;EACA,MAAMI,aAAa,GAAGF,OAAO,CAACG,GAAG,CAAEZ,MAAM,IAAI;IAC5C;IACA,MAAM;MAAEC,OAAO;MAAEC;IAAQ,CAAC,GAAGH,iBAAiB,CAAEC,MAAO,CAAC;;IAExD;IACA,MAAMa,gBAAgB,GAAGX,OAAO,CAACY,OAAO,CAAEN,UAAU,EAAIO,CAAC,IAAM,IAAAC,0BAAO,EAAED,CAAE,CAAE,CAAC;;IAE7E;IACA,OAAOV,iBAAiB,CAAEJ,OAAO,EAAEY,gBAAiB,CAAC;EACtD,CAAE,CAAC;EAEH,OAAO;IAAEJ,OAAO;IAAEE;EAAc,CAAC;AAClC,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,MAAMM,yBAAyB,GAAG,SAAAA,CAAUV,QAAQ,EAAEW,oBAAoB,EAAEC,qBAAqB,EAAG;EAC1G,MAAMC,aAAa,GAAG,EAAE;EACxBF,oBAAoB,CAACG,OAAO,CAAEC,IAAI,IAAI;IACrC;IACA,MAAMC,mBAAmB,GAAGD,IAAI,CAACnB,KAAK,CAAEqB,2BAAoB,CAAC;IAC7D,IAAKD,mBAAmB,EAAG;MAC1B;AACH;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;MACGE,2BAAmB,CAACJ,OAAO,CAAEK,WAAW,IAAI;QAC3CH,mBAAmB,CAACF,OAAO,CAAEM,kBAAkB,IAAI;UAClDP,aAAa,CAACQ,IAAI,CAAE,IAAAC,oBAAY,EAAEP,IAAI,CAACR,OAAO,CAAE,IAAIgB,MAAM,CAAEH,kBAAkB,EAAE,GAAI,CAAC,EAAED,WAAY,CAAE,CAAE,CAAC;QACzG,CAAE,CAAC;MACJ,CAAE,CAAC;IACJ,CAAC,MAAM;MACNN,aAAa,CAACQ,IAAI,CAAE,IAAAC,oBAAY,EAAEP,IAAK,CAAE,CAAC;IAC3C;EACD,CAAE,CAAC;;EAEH;EACA,MAAMd,UAAU,GAAGW,qBAAqB,GAAG,IAAAY,6BAAY,EAAEX,aAAa,EAAE,IAAK,CAAC,GAAG,IAAAW,6BAAY,EAAEX,aAAc,CAAC;;EAE9G;EACA,MAAM;IAAEX,OAAO;IAAEE;EAAc,CAAC,GAAGL,gBAAgB,CAAEC,QAAQ,EAAEC,UAAW,CAAC;EAE3E,IAAIwB,MAAM,GAAGzB,QAAQ,CAACO,OAAO,CAAEN,UAAU,EAAE,UAAUO,CAAC,EAAG;IACxD,OAAO,IAAAC,0BAAO,EAAED,CAAE,CAAC;EACpB,CAAE,CAAC;;EAEH;AACD;AACA;AACA;AACA;AACA;EACC,IAAKN,OAAO,CAACwB,MAAM,GAAG,CAAC,EAAG;IACzB,MAAMC,aAAa,GAAG,IAAAxB,2BAAkB,EAAEsB,MAAO,CAAC;IAClD,KAAM,IAAIG,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGD,aAAa,CAACD,MAAM,EAAEE,CAAC,EAAE,EAAG;MAChDH,MAAM,GAAGA,MAAM,CAAClB,OAAO,CAAEoB,aAAa,CAAEC,CAAC,CAAE,EAAExB,aAAa,CAAEwB,CAAC,CAAG,CAAC;IAClE;EACD;;EAEA;AACD;AACA;AACA;EACC,OAASH,MAAM,CAAClB,OAAO,CAAE,IAAIgB,MAAM,CAAE,kDAAkD,EAAE,IAAK,CAAC,EAAE,GAAI,CAAC;AACvG,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAPA1B,OAAA,CAAAa,yBAAA,GAAAA,yBAAA;AAQO,SAASmB,oBAAoBA,CAAE7B,QAAQ,EAAEW,oBAAoB,EAAEC,qBAAqB,EAAG;EAC7F,OAAO,CAAE,IAAIkB,aAAI,CAAE;IAClBC,QAAQ,EAAE/B,QAAQ;IAClBgC,MAAM,EAAEtB,yBAAyB,CAAEV,QAAQ,EAAEW,oBAAoB,EAAEC,qBAAsB;EAC1F,CAAE,CAAC,CAAE;AACN;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASqB,oBAAoBA,CAAEC,WAAW,EAAEC,SAAS,EAAEC,MAAM,EAAExB,qBAAqB,EAAG;EAC7F,IAAID,oBAAoB,GAAG,EAAE;EAC7B,IAAI0B,QAAQ,GAAG,EAAE;EAEjBF,SAAS,CAACrB,OAAO,CAAE,UAAUd,QAAQ,EAAG;IACvCW,oBAAoB,GAAG,IAAA2B,2BAAU,EAAEtC,QAAQ,EAAEkC,WAAW,EAAEE,MAAM,EAAExB,qBAAsB,CAAC,CAAC2B,OAAO;IAEjG,IAAK5B,oBAAoB,CAACe,MAAM,GAAG,CAAC,EAAG;MACtCW,QAAQ,GAAGA,QAAQ,CAACG,MAAM,CAAEX,oBAAoB,CAAE7B,QAAQ,EAAEW,oBAAoB,EAAEC,qBAAsB,CAAE,CAAC;IAC5G;EACD,CAAE,CAAC;EAEH,OAAOyB,QAAQ;AAChB","ignoreList":[]}

package/build/languageProcessing/helpers/word/matchWordInSentence.js

@@ -0,0 +1,63 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.isWordInSentence = exports.default = exports.characterInBoundary = void 0;
+var _wordBoundaries = _interopRequireDefault(require("../../../config/wordBoundaries.js"));
+var _lodash = require("lodash");
+var _addWordboundary = _interopRequireDefault(require("./addWordboundary.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const wordBoundaries = (0, _wordBoundaries.default)();
+/**
+ * Checks whether a character is present in the list of word boundaries.
+ *
+ * @param {string} character The character to look for.
+ * @returns {boolean} Whether or not the character is present in the list of word boundaries.
+ */
+const characterInBoundary = function (character) {
+  return (0, _lodash.includes)(wordBoundaries, character);
+};
+
+/**
+ * Checks whether a word is present in a sentence.
+ *
+ * @param {string} word The word to search for in the sentence.
+ * @param {string} sentence The sentence to look through.
+ * @returns {boolean} Whether or not the word is present in the sentence.
+ */
+exports.characterInBoundary = characterInBoundary;
+const isWordInSentence = function (word, sentence) {
+  // To ensure proper matching, make everything lowercase.
+  word = word.toLocaleLowerCase();
+  sentence = sentence.toLocaleLowerCase();
+
+  // Escape regex in word, since we use regex characters like in abbreviations ("e.g.").
+  const wordWithBoundaries = (0, _addWordboundary.default)((0, _lodash.escapeRegExp)(word));
+  let occurrenceStart = sentence.search(new RegExp(wordWithBoundaries, "ig"));
+  // Return false if no match has been found.
+  if (occurrenceStart === -1) {
+    return false;
+  }
+  /*
+  If there is a word boundary before the matched word, the regex includes this word boundary in the match.
+  This means that occurrenceStart is the index of the word boundary before the match. Therefore 1 has to
+  be added to occurrenceStart, except when there is no word boundary before the match (i.e. at the start
+  of a sentence).
+   */
+  if (occurrenceStart > 0) {
+    occurrenceStart += 1;
+  }
+  const occurrenceEnd = occurrenceStart + word.length;
+
+  // Check if the previous and next character are word boundaries to determine if a complete word was detected
+  const previousCharacter = characterInBoundary(sentence[occurrenceStart - 1]) || occurrenceStart === 0;
+  const nextCharacter = characterInBoundary(sentence[occurrenceEnd]) || occurrenceEnd === sentence.length;
+  return previousCharacter && nextCharacter;
+};
+exports.isWordInSentence = isWordInSentence;
+var _default = exports.default = {
+  characterInBoundary,
+  isWordInSentence
+};
+//# sourceMappingURL=matchWordInSentence.js.map

package/build/languageProcessing/helpers/word/matchWordInSentence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchWordInSentence.js","names":["_wordBoundaries","_interopRequireDefault","require","_lodash","_addWordboundary","e","__esModule","default","wordBoundaries","wordBoundariesFactory","characterInBoundary","character","includes","exports","isWordInSentence","word","sentence","toLocaleLowerCase","wordWithBoundaries","addWordBoundary","escapeRegExp","occurrenceStart","search","RegExp","occurrenceEnd","length","previousCharacter","nextCharacter","_default"],"sources":["../../../../src/languageProcessing/helpers/word/matchWordInSentence.js"],"sourcesContent":["import wordBoundariesFactory from \"../../../config/wordBoundaries.js\";\nconst wordBoundaries = wordBoundariesFactory();\nimport { escapeRegExp, includes } from \"lodash\";\nimport addWordBoundary from \"./addWordboundary.js\";\n\n/**\n * Checks whether a character is present in the list of word boundaries.\n *\n * @param {string} character The character to look for.\n * @returns {boolean} Whether or not the character is present in the list of word boundaries.\n */\nconst characterInBoundary = function( character ) {\n\treturn includes( wordBoundaries, character );\n};\n\n/**\n * Checks whether a word is present in a sentence.\n *\n * @param {string} word The word to search for in the sentence.\n * @param {string} sentence The sentence to look through.\n * @returns {boolean} Whether or not the word is present in the sentence.\n */\nconst isWordInSentence = function( word, sentence ) {\n\t// To ensure proper matching, make everything lowercase.\n\tword = word.toLocaleLowerCase();\n\tsentence = sentence.toLocaleLowerCase();\n\n\t// Escape regex in word, since we use regex characters like in abbreviations (\"e.g.\").\n\tconst wordWithBoundaries = addWordBoundary( escapeRegExp( word ) );\n\tlet occurrenceStart = sentence.search( new RegExp( wordWithBoundaries, \"ig\" ) );\n\t// Return false if no match has been found.\n\tif ( occurrenceStart === -1 ) {\n\t\treturn false;\n\t}\n\t/*\n\tIf there is a word boundary before the matched word, the regex includes this word boundary in the match.\n\tThis means that occurrenceStart is the index of the word boundary before the match. Therefore 1 has to\n\tbe added to occurrenceStart, except when there is no word boundary before the match (i.e. at the start\n\tof a sentence).\n\t */\n\tif ( occurrenceStart > 0 ) {\n\t\toccurrenceStart += 1;\n\t}\n\tconst occurrenceEnd = occurrenceStart + word.length;\n\n\t// Check if the previous and next character are word boundaries to determine if a complete word was detected\n\tconst previousCharacter = characterInBoundary( sentence[ occurrenceStart - 1 ] ) || occurrenceStart === 0;\n\tconst nextCharacter = characterInBoundary( sentence[ occurrenceEnd ] ) || occurrenceEnd === sentence.length;\n\n\treturn ( ( previousCharacter ) && ( nextCharacter ) );\n};\n\nexport {\n\tcharacterInBoundary,\n\tisWordInSentence,\n};\n\nexport default {\n\tcharacterInBoundary,\n\tisWordInSentence,\n};\n"],"mappings":";;;;;;AAAA,IAAAA,eAAA,GAAAC,sBAAA,CAAAC,OAAA;AAEA,IAAAC,OAAA,GAAAD,OAAA;AACA,IAAAE,gBAAA,GAAAH,sBAAA,CAAAC,OAAA;AAAmD,SAAAD,uBAAAI,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAFnD,MAAMG,cAAc,GAAG,IAAAC,uBAAqB,EAAC,CAAC;AAI9C;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,mBAAmB,GAAG,SAAAA,CAAUC,SAAS,EAAG;EACjD,OAAO,IAAAC,gBAAQ,EAAEJ,cAAc,EAAEG,SAAU,CAAC;AAC7C,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AANAE,OAAA,CAAAH,mBAAA,GAAAA,mBAAA;AAOA,MAAMI,gBAAgB,GAAG,SAAAA,CAAUC,IAAI,EAAEC,QAAQ,EAAG;EACnD;EACAD,IAAI,GAAGA,IAAI,CAACE,iBAAiB,CAAC,CAAC;EAC/BD,QAAQ,GAAGA,QAAQ,CAACC,iBAAiB,CAAC,CAAC;;EAEvC;EACA,MAAMC,kBAAkB,GAAG,IAAAC,wBAAe,EAAE,IAAAC,oBAAY,EAAEL,IAAK,CAAE,CAAC;EAClE,IAAIM,eAAe,GAAGL,QAAQ,CAACM,MAAM,CAAE,IAAIC,MAAM,CAAEL,kBAAkB,EAAE,IAAK,CAAE,CAAC;EAC/E;EACA,IAAKG,eAAe,KAAK,CAAC,CAAC,EAAG;IAC7B,OAAO,KAAK;EACb;EACA;AACD;AACA;AACA;AACA;AACA;EACC,IAAKA,eAAe,GAAG,CAAC,EAAG;IAC1BA,eAAe,IAAI,CAAC;EACrB;EACA,MAAMG,aAAa,GAAGH,eAAe,GAAGN,IAAI,CAACU,MAAM;;EAEnD;EACA,MAAMC,iBAAiB,GAAGhB,mBAAmB,CAAEM,QAAQ,CAAEK,eAAe,GAAG,CAAC,CAAG,CAAC,IAAIA,eAAe,KAAK,CAAC;EACzG,MAAMM,aAAa,GAAGjB,mBAAmB,CAAEM,QAAQ,CAAEQ,aAAa,CAAG,CAAC,IAAIA,aAAa,KAAKR,QAAQ,CAACS,MAAM;EAE3G,OAAWC,iBAAiB,IAAQC,aAAe;AACpD,CAAC;AAACd,OAAA,CAAAC,gBAAA,GAAAA,gBAAA;AAAA,IAAAc,QAAA,GAAAf,OAAA,CAAAN,OAAA,GAOa;EACdG,mBAAmB;EACnBI;AACD,CAAC","ignoreList":[]}

package/build/languageProcessing/helpers/word/splitIntoTokens.js

@@ -0,0 +1,52 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _createPunctuationTokens = _interopRequireDefault(require("./createPunctuationTokens"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/*
+*  The following regex is used to split a text into tokens.
+ * The regex matches word separators. A word separator is either a whitespace, a slash, a
+ * tab, a non-breaking space, a hyphen, an en-dash or an em-dash.
+ * Opening and closing square brackets are added to deal correctly with shortcodes downstream.
+ * The regex is used to split a text into tokens.
+ * Do not add punctuation marks to this regex, as they are handled separately inside `createPunctuationTokens()`.
+ * The word separator explicitly only contains characters that split two words and not a word and a space.
+ * - A space is a word separator because it separates two words if it occurs between them. For example: "foo bar"
+ * - A tab is a word separator because it separates two words if it occurs between them. For example: "foo	bar"
+ * - A non-breaking space (u00A0) is a word separator because it separates two words if it occurs between them. For example: "foo\u00A0bar".
+ * - An en-dash (u2013), an em-dash (u2014), and a hyphen (u002d) are word separators because they separate two words if they occur between them.
+ *   For example: "foo–bar".
+ * Note that   is added here as #nbsp; -- we transform the & to # to prevent parse5 from converting it to a space and messing up highlighting.
+ */
+const WORD_SEPARATORS_REGEX = /([\s\t\u00A0\u2013\u2014\u002d[\]]|#nbsp;)/;
+
+/**
+ * Tokenizes a text similarly to `getWords`, but in a way that's suitable for the HTML parser.
+ * 1. It does not normalize whitespace.
+ * This operation is too risky for the HTML parser because it may throw away characters and as a result, the token positions are corrupted.
+ * 2. It does not remove punctuation marks but keeps them.
+ *
+ * This algorithm splits the text by word separators: tokens that are the border between two words.
+ * This algorithm separates punctuation marks from words and keeps them as separate tokens.
+ * It only splits them off if they appear at the start or the end of a word.
+ *
+ * @param {string} text The text to tokenize.
+ *
+ * @returns {string[]} 	An array of tokens.
+ */
+const splitIntoTokens = text => {
+  if (!text) {
+    return [];
+  }
+
+  // Split the sentence string into tokens. Those tokens are unrefined as they may contain punctuation.
+  const rawTokens = text.split(WORD_SEPARATORS_REGEX).filter(x => x !== "");
+
+  // Remove punctuation from the beginning and end of word tokens, and make them into separate tokens.
+  return (0, _createPunctuationTokens.default)(rawTokens);
+};
+var _default = exports.default = splitIntoTokens;
+//# sourceMappingURL=splitIntoTokens.js.map

package/build/languageProcessing/helpers/word/splitIntoTokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"splitIntoTokens.js","names":["_createPunctuationTokens","_interopRequireDefault","require","e","__esModule","default","WORD_SEPARATORS_REGEX","splitIntoTokens","text","rawTokens","split","filter","x","createPunctuationTokens","_default","exports"],"sources":["../../../../src/languageProcessing/helpers/word/splitIntoTokens.js"],"sourcesContent":["import createPunctuationTokens from \"./createPunctuationTokens\";\n\n/*\n*  The following regex is used to split a text into tokens.\n * The regex matches word separators. A word separator is either a whitespace, a slash, a\n * tab, a non-breaking space, a hyphen, an en-dash or an em-dash.\n * Opening and closing square brackets are added to deal correctly with shortcodes downstream.\n * The regex is used to split a text into tokens.\n * Do not add punctuation marks to this regex, as they are handled separately inside `createPunctuationTokens()`.\n * The word separator explicitly only contains characters that split two words and not a word and a space.\n * - A space is a word separator because it separates two words if it occurs between them. For example: \"foo bar\"\n * - A tab is a word separator because it separates two words if it occurs between them. For example: \"foo\tbar\"\n * - A non-breaking space (u00A0) is a word separator because it separates two words if it occurs between them. For example: \"foo\\u00A0bar\".\n * - An en-dash (u2013), an em-dash (u2014), and a hyphen (u002d) are word separators because they separate two words if they occur between them.\n *   For example: \"foo–bar\".\n * Note that   is added here as #nbsp; -- we transform the & to # to prevent parse5 from converting it to a space and messing up highlighting.\n */\nconst WORD_SEPARATORS_REGEX = /([\\s\\t\\u00A0\\u2013\\u2014\\u002d[\\]]|#nbsp;)/;\n\n/**\n * Tokenizes a text similarly to `getWords`, but in a way that's suitable for the HTML parser.\n * 1. It does not normalize whitespace.\n * This operation is too risky for the HTML parser because it may throw away characters and as a result, the token positions are corrupted.\n * 2. It does not remove punctuation marks but keeps them.\n *\n * This algorithm splits the text by word separators: tokens that are the border between two words.\n * This algorithm separates punctuation marks from words and keeps them as separate tokens.\n * It only splits them off if they appear at the start or the end of a word.\n *\n * @param {string} text The text to tokenize.\n *\n * @returns {string[]} \tAn array of tokens.\n */\nconst splitIntoTokens = ( text ) => {\n\tif ( ! text ) {\n\t\treturn [];\n\t}\n\n\t// Split the sentence string into tokens. Those tokens are unrefined as they may contain punctuation.\n\tconst rawTokens = text.split( WORD_SEPARATORS_REGEX ).filter( x => x !== \"\" );\n\n\t// Remove punctuation from the beginning and end of word tokens, and make them into separate tokens.\n\treturn createPunctuationTokens( rawTokens );\n};\n\nexport default splitIntoTokens;\n"],"mappings":";;;;;;AAAA,IAAAA,wBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAgE,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEhE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,qBAAqB,GAAG,4CAA4C;;AAE1E;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,eAAe,GAAKC,IAAI,IAAM;EACnC,IAAK,CAAEA,IAAI,EAAG;IACb,OAAO,EAAE;EACV;;EAEA;EACA,MAAMC,SAAS,GAAGD,IAAI,CAACE,KAAK,CAAEJ,qBAAsB,CAAC,CAACK,MAAM,CAAEC,CAAC,IAAIA,CAAC,KAAK,EAAG,CAAC;;EAE7E;EACA,OAAO,IAAAC,gCAAuB,EAAEJ,SAAU,CAAC;AAC5C,CAAC;AAAC,IAAAK,QAAA,GAAAC,OAAA,CAAAV,OAAA,GAEaE,eAAe","ignoreList":[]}