axyseo 2.0.0-alpha.2 → 2.1.0

package/build/languageProcessing/researches/keyphraseDistribution.js

@@ -1,18 +1,12 @@
-"use strict";
+import { flattenDeep, max, uniq as unique, zipWith } from "lodash";
+import { findWordFormsInString } from "../helpers/match/findKeywordFormsInString";
+import { markWordsInSentences } from "../helpers/word/markWordsInSentences";
+import getSentences from "../helpers/sentence/getSentences";
+import parseSynonyms from "../helpers/sanitize/parseSynonyms";
+import { mergeListItems } from "../helpers/sanitize/mergeListItems";
+import removeHtmlBlocks from "../helpers/html/htmlParser";
+import { filterShortcodesFromHTML } from "../helpers/sanitize/filterShortcodesFromTree";
 
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.maximizeSentenceScores = exports.keyphraseDistributionResearcher = exports.getDistraction = exports.default = exports.computeScoresPerSentenceShortTopic = exports.computeScoresPerSentenceLongTopic = void 0;
-var _lodash = require("lodash");
-var _findKeywordFormsInString = require("../helpers/match/findKeywordFormsInString");
-var _markWordsInSentences = require("../helpers/word/markWordsInSentences");
-var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences"));
-var _parseSynonyms = _interopRequireDefault(require("../helpers/sanitize/parseSynonyms"));
-var _mergeListItems = require("../helpers/sanitize/mergeListItems");
-var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
-var _filterShortcodesFromTree = require("../helpers/sanitize/filterShortcodesFromTree");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * Checks whether at least half of the content words from the topic are found within the sentence.
  * Assigns a score to every sentence following the following schema:
@@ -29,7 +23,7 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
 const computeScoresPerSentenceLongTopic = function (topic, sentences, locale, matchWordCustomHelper) {
   const sentenceScores = Array(sentences.length);
   for (let i = 0; i < sentences.length; i++) {
-    const foundInCurrentSentence = (0, _findKeywordFormsInString.findWordFormsInString)(topic, sentences[i], locale, matchWordCustomHelper);
+    const foundInCurrentSentence = findWordFormsInString(topic, sentences[i], locale, matchWordCustomHelper);
     if (foundInCurrentSentence.percentWordMatches >= 50) {
       sentenceScores[i] = 9;
     } else {
@@ -52,12 +46,11 @@ const computeScoresPerSentenceLongTopic = function (topic, sentences, locale, ma
  *
  * @returns {Array} The scores per sentence.
  */
-exports.computeScoresPerSentenceLongTopic = computeScoresPerSentenceLongTopic;
 const computeScoresPerSentenceShortTopic = function (topic, sentences, locale, matchWordCustomHelper) {
   const sentenceScores = Array(sentences.length);
   for (let i = 0; i < sentences.length; i++) {
     const currentSentence = sentences[i];
-    const foundInCurrentSentence = (0, _findKeywordFormsInString.findWordFormsInString)(topic, currentSentence, locale, matchWordCustomHelper);
+    const foundInCurrentSentence = findWordFormsInString(topic, currentSentence, locale, matchWordCustomHelper);
     if (foundInCurrentSentence.percentWordMatches === 100) {
       sentenceScores[i] = 9;
     } else {
@@ -74,7 +67,6 @@ const computeScoresPerSentenceShortTopic = function (topic, sentences, locale, m
  *
  * @returns {Array} Maximal scores of topic relevance per sentence.
  */
-exports.computeScoresPerSentenceShortTopic = computeScoresPerSentenceShortTopic;
 const maximizeSentenceScores = function (sentenceScores) {
   const sentenceScoresTransposed = sentenceScores[0].map(function (col, i) {
     return sentenceScores.map(function (row) {
@@ -82,7 +74,7 @@ const maximizeSentenceScores = function (sentenceScores) {
     });
   });
   return sentenceScoresTransposed.map(function (scoresForOneSentence) {
-    return (0, _lodash.max)(scoresForOneSentence);
+    return max(scoresForOneSentence);
   });
 };
 
@@ -93,7 +85,6 @@ const maximizeSentenceScores = function (sentenceScores) {
  *
  * @returns {number} The maximum number of sentences that do not include the topic.
  */
-exports.maximizeSentenceScores = maximizeSentenceScores;
 const getDistraction = function (sentenceScores) {
   const numberOfSentences = sentenceScores.length;
   const allTopicSentencesIndices = [];
@@ -117,7 +108,7 @@ const getDistraction = function (sentenceScores) {
   for (let i = 1; i < numberOfTopicSentences + 2; i++) {
     distances.push(allTopicSentencesIndices[i] - allTopicSentencesIndices[i - 1] - 1);
   }
-  return (0, _lodash.max)(distances);
+  return max(distances);
 };
 
 /**
@@ -135,7 +126,6 @@ const getDistraction = function (sentenceScores) {
  *
  * @returns {Object} An array with maximized score per sentence and an array with all sentences that do not contain the topic.
  */
-exports.getDistraction = getDistraction;
 const getSentenceScores = function (sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper, topicLengthCriteria = 4, originalTopic, wordsCharacterCount) {
   // Compute per-sentence scores of topic-relatedness.
   const topicNumber = topicFormsInOneArray.length;
@@ -169,7 +159,7 @@ const getSentenceScores = function (sentences, topicFormsInOneArray, locale, fun
   const maximizedSentenceScores = maximizeSentenceScores(sentenceScores);
 
   // Zip an array combining each sentence with the associated maximized score.
-  const sentencesWithMaximizedScores = (0, _lodash.zipWith)(sentences, maximizedSentenceScores, (sentence, score) => {
+  const sentencesWithMaximizedScores = zipWith(sentences, maximizedSentenceScores, (sentence, score) => {
     return {
       sentence,
       score
@@ -202,32 +192,32 @@ const keyphraseDistributionResearcher = function (paper, researcher) {
   // Custom topic length criteria for languages that don't use the default value to determine whether a topic is long or short.
   const topicLengthCriteria = researcher.getConfig("topicLength").lengthCriteria;
   let text = paper.getText();
-  text = (0, _htmlParser.default)(text);
-  text = (0, _filterShortcodesFromTree.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
-  text = (0, _mergeListItems.mergeListItems)(text);
-  const sentences = (0, _getSentences.default)(text, memoizedTokenizer);
+  text = removeHtmlBlocks(text);
+  text = filterShortcodesFromHTML(text, paper._attributes && paper._attributes.shortcodes);
+  text = mergeListItems(text);
+  const sentences = getSentences(text, memoizedTokenizer);
   const topicForms = researcher.getResearch("morphology");
   const originalTopic = [];
   if (getContentWordsHelper) {
     originalTopic.push(getContentWordsHelper(paper.getKeyword()));
-    (0, _parseSynonyms.default)(paper.getSynonyms()).forEach(synonym => originalTopic.push(getContentWordsHelper(synonym)));
+    parseSynonyms(paper.getSynonyms()).forEach(synonym => originalTopic.push(getContentWordsHelper(synonym)));
   }
   const locale = paper.getLocale();
   const topicFormsInOneArray = [topicForms.keyphraseForms];
   topicForms.synonymsForms.forEach(function (synonym) {
     topicFormsInOneArray.push(synonym);
   });
-  const allTopicWords = (0, _lodash.uniq)((0, _lodash.flattenDeep)(topicFormsInOneArray)).sort((a, b) => b.length - a.length);
+  const allTopicWords = unique(flattenDeep(topicFormsInOneArray)).sort((a, b) => b.length - a.length);
 
   // Get per-sentence scores and sentences that have topic.
   const sentenceScores = getSentenceScores(sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper, topicLengthCriteria, originalTopic, wordsCharacterCount);
   const maximizedSentenceScores = sentenceScores.maximizedSentenceScores;
   const maxLengthDistraction = getDistraction(maximizedSentenceScores);
   return {
-    sentencesToHighlight: (0, _markWordsInSentences.markWordsInSentences)(allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper),
+    sentencesToHighlight: markWordsInSentences(allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper),
     keyphraseDistributionScore: maxLengthDistraction / sentences.length * 100
   };
 };
-exports.keyphraseDistributionResearcher = keyphraseDistributionResearcher;
-var _default = exports.default = keyphraseDistributionResearcher;
+export { computeScoresPerSentenceShortTopic, computeScoresPerSentenceLongTopic, maximizeSentenceScores, keyphraseDistributionResearcher, getDistraction };
+export default keyphraseDistributionResearcher;
 //# sourceMappingURL=keyphraseDistribution.js.map

package/build/languageProcessing/researches/keyphraseDistribution.js.map

@@ -1 +1 @@
-{"version":3,"file":"keyphraseDistribution.js","names":["_lodash","require","_findKeywordFormsInString","_markWordsInSentences","_getSentences","_interopRequireDefault","_parseSynonyms","_mergeListItems","_htmlParser","_filterShortcodesFromTree","e","__esModule","default","computeScoresPerSentenceLongTopic","topic","sentences","locale","matchWordCustomHelper","sentenceScores","Array","length","i","foundInCurrentSentence","findWordFormsInString","percentWordMatches","exports","computeScoresPerSentenceShortTopic","currentSentence","maximizeSentenceScores","sentenceScoresTransposed","map","col","row","scoresForOneSentence","max","getDistraction","numberOfSentences","allTopicSentencesIndices","push","numberOfTopicSentences","unshift","distances","getSentenceScores","topicFormsInOneArray","functionWords","topicLengthCriteria","originalTopic","wordsCharacterCount","topicNumber","topicLength","maximizedSentenceScores","sentencesWithMaximizedScores","zipWith","sentence","score","sentencesWithTopic","filter","sentenceObject","keyphraseDistributionResearcher","paper","researcher","getConfig","getHelper","getContentWordsHelper","getResearch","memoizedTokenizer","lengthCriteria","text","getText","removeHtmlBlocks","filterShortcodesFromHTML","_attributes","shortcodes","mergeListItems","getSentences","topicForms","getKeyword","parseSynonyms","getSynonyms","forEach","synonym","getLocale","keyphraseForms","synonymsForms","allTopicWords","unique","flattenDeep","sort","a","b","maxLengthDistraction","sentencesToHighlight","markWordsInSentences","keyphraseDistributionScore","_default"],"sources":["../../../src/languageProcessing/researches/keyphraseDistribution.js"],"sourcesContent":["import { flattenDeep, max, uniq as unique, zipWith } from \"lodash\";\nimport { findWordFormsInString } from \"../helpers/match/findKeywordFormsInString\";\nimport { markWordsInSentences } from \"../helpers/word/markWordsInSentences\";\nimport getSentences from \"../helpers/sentence/getSentences\";\nimport parseSynonyms from \"../helpers/sanitize/parseSynonyms\";\nimport { mergeListItems } from \"../helpers/sanitize/mergeListItems\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers/sanitize/filterShortcodesFromTree\";\n\n/**\n * Checks whether at least half of the content words from the topic are found within the sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if at least half of the content words from the topic are in the sentence,\n * 3 otherwise.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceLongTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, sentences[ i ], locale, matchWordCustomHelper );\n\n\t\tif ( foundInCurrentSentence.percentWordMatches >= 50 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\n\treturn sentenceScores;\n};\n\n\n/**\n * Checks whether all content words from the topic are found within one sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if all content words from the topic are in the sentence,\n * 3 if not all content words from the topic were found in the sentence.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceShortTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst currentSentence = sentences[ i ];\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, currentSentence, locale, matchWordCustomHelper );\n\t\tif ( foundInCurrentSentence.percentWordMatches === 100 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\treturn sentenceScores;\n};\n\n/**\n * Maximizes scores: Give every sentence a maximal score that it got from analysis of all topics\n *\n * @param {Array} sentenceScores The scores for every sentence, as assessed per keyphrase and every synonym.\n *\n * @returns {Array} Maximal scores of topic relevance per sentence.\n */\nconst maximizeSentenceScores = function( sentenceScores ) {\n\tconst sentenceScoresTransposed = sentenceScores[ 0 ].map( function( col, i ) {\n\t\treturn sentenceScores.map( function( row ) {\n\t\t\treturn row[ i ];\n\t\t} );\n\t} );\n\n\treturn sentenceScoresTransposed.map( function( scoresForOneSentence ) {\n\t\treturn max( scoresForOneSentence );\n\t} );\n};\n\n\n/**\n * Computes the maximally long piece of text that does not include the topic.\n *\n * @param {Array} sentenceScores The array of scores per sentence.\n *\n * @returns {number} The maximum number of sentences that do not include the topic.\n */\nconst getDistraction = function( sentenceScores ) {\n\tconst numberOfSentences = sentenceScores.length;\n\tconst allTopicSentencesIndices = [];\n\n\tfor ( let i = 0; i < numberOfSentences; i++ ) {\n\t\tif ( sentenceScores[ i ] > 3 ) {\n\t\t\tallTopicSentencesIndices.push( i );\n\t\t}\n\t}\n\n\tconst numberOfTopicSentences = allTopicSentencesIndices.length;\n\n\tif ( numberOfTopicSentences === 0 ) {\n\t\treturn numberOfSentences;\n\t}\n\n\t/**\n\t * Add fake topic sentences at the very beginning and at the very end\n\t * to account for cases when the text starts or ends with a train of distraction.\n\t */\n\tallTopicSentencesIndices.unshift( -1 );\n\tallTopicSentencesIndices.push( numberOfSentences );\n\n\tconst distances = [];\n\n\tfor ( let i = 1; i < numberOfTopicSentences + 2; i++ ) {\n\t\tdistances.push( allTopicSentencesIndices[ i ] - allTopicSentencesIndices[ i - 1 ] - 1 );\n\t}\n\n\treturn max( distances );\n};\n\n/**\n * Computes the per-sentence scores depending on the length of the topic phrase and maximizes them over all topic phrases.\n *\n * @param {Array}       sentences              The sentences to get scores for.\n * @param {Array}       topicFormsInOneArray   The topic phrases forms to search for in the sentences.\n * @param {string}      locale                 The locale to work in.\n * @param {Array}       functionWords           The function words list.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n * @param {int}         topicLengthCriteria     The topic length criteria. The default value is 4, where a topic is considered short\n *                                              if it's less than 4 word long, and otherwise long.\n * @param {Array}       originalTopic           The array of the original form of the topic with function words filtered out.\n * @param {function}    wordsCharacterCount     The helper to calculate the characters length of all the words in the array.\n *\n * @returns {Object} An array with maximized score per sentence and an array with all sentences that do not contain the topic.\n */\nconst getSentenceScores = function( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\ttopicLengthCriteria = 4, originalTopic, wordsCharacterCount ) {\n\t// Compute per-sentence scores of topic-relatedness.\n\tconst topicNumber = topicFormsInOneArray.length;\n\n\tconst sentenceScores = Array( topicNumber );\n\n\t// For languages with function words apply either full match or partial match depending on topic length\n\tif ( functionWords.length > 0 ) {\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\t/*\n\t\t\t * If the helper to calculate the characters length of all the words in the array is available,\n\t\t\t * we use this helper to calculate the characters length of the original topic form.\n\t\t\t * We then use the result and compare it with the topicLengthCriteria.\n\t\t\t */\n\t\t\tconst topicLength = wordsCharacterCount ? wordsCharacterCount( originalTopic[ i ] ) : topic.length;\n\t\t\tif ( topicLength < topicLengthCriteria ) {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t} else {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceLongTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// For languages without function words apply the full match always\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t}\n\t}\n\n\t// Maximize scores: Give every sentence a maximal score that it got from analysis of all topics\n\tconst maximizedSentenceScores = maximizeSentenceScores( sentenceScores );\n\n\t// Zip an array combining each sentence with the associated maximized score.\n\tconst sentencesWithMaximizedScores =  zipWith( sentences, maximizedSentenceScores, ( sentence, score ) => {\n\t\treturn { sentence, score };\n\t} );\n\n\t// Filter sentences that contain topic words for future highlights.\n\tconst sentencesWithTopic = sentencesWithMaximizedScores.filter( sentenceObject => sentenceObject.score > 3 );\n\n\treturn {\n\t\tmaximizedSentenceScores: maximizedSentenceScores,\n\t\tsentencesWithTopic: sentencesWithTopic.map( sentenceObject => sentenceObject.sentence ),\n\t};\n};\n\n\n/**\n * Determines which portions of the text did not receive a lot of content words from keyphrase and synonyms.\n *\n * @param {Paper}       paper               The paper to check the keyphrase distribution for.\n * @param {Researcher}  researcher          The researcher to use for analysis.\n *\n * @returns {Object} The scores of topic relevance per portion of text and an array of all word forms to highlight.\n */\nconst keyphraseDistributionResearcher = function( paper, researcher ) {\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\tconst matchWordCustomHelper = researcher.getHelper( \"matchWordCustomHelper\" );\n\tconst getContentWordsHelper = researcher.getHelper( \"getContentWords\" );\n\tconst wordsCharacterCount = researcher.getResearch( \"wordsCharacterCount\" );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\n\t// Custom topic length criteria for languages that don't use the default value to determine whether a topic is long or short.\n\tconst topicLengthCriteria = researcher.getConfig( \"topicLength\" ).lengthCriteria;\n\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\ttext = mergeListItems( text );\n\tconst sentences = getSentences( text, memoizedTokenizer );\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\n\tconst originalTopic = [];\n\tif ( getContentWordsHelper ) {\n\t\toriginalTopic.push( getContentWordsHelper( paper.getKeyword() ) );\n\t\tparseSynonyms( paper.getSynonyms() ).forEach( synonym => originalTopic.push( getContentWordsHelper( synonym ) ) );\n\t}\n\tconst locale = paper.getLocale();\n\tconst topicFormsInOneArray = [ topicForms.keyphraseForms ];\n\ttopicForms.synonymsForms.forEach( function( synonym ) {\n\t\ttopicFormsInOneArray.push( synonym );\n\t} );\n\n\tconst allTopicWords = unique( flattenDeep( topicFormsInOneArray ) ).sort( ( a, b ) => b.length - a.length );\n\n\t// Get per-sentence scores and sentences that have topic.\n\tconst sentenceScores = getSentenceScores( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\t\ttopicLengthCriteria, originalTopic, wordsCharacterCount );\n\tconst maximizedSentenceScores = sentenceScores.maximizedSentenceScores;\n\tconst maxLengthDistraction = getDistraction( maximizedSentenceScores );\n\n\treturn {\n\t\tsentencesToHighlight: markWordsInSentences( allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper ),\n\t\tkeyphraseDistributionScore: maxLengthDistraction / sentences.length * 100,\n\t};\n};\n\nexport {\n\tcomputeScoresPerSentenceShortTopic,\n\tcomputeScoresPerSentenceLongTopic,\n\tmaximizeSentenceScores,\n\tkeyphraseDistributionResearcher,\n\tgetDistraction,\n};\n\nexport default keyphraseDistributionResearcher;\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,yBAAA,GAAAD,OAAA;AACA,IAAAE,qBAAA,GAAAF,OAAA;AACA,IAAAG,aAAA,GAAAC,sBAAA,CAAAJ,OAAA;AACA,IAAAK,cAAA,GAAAD,sBAAA,CAAAJ,OAAA;AACA,IAAAM,eAAA,GAAAN,OAAA;AACA,IAAAO,WAAA,GAAAH,sBAAA,CAAAJ,OAAA;AACA,IAAAQ,yBAAA,GAAAR,OAAA;AAAwF,SAAAI,uBAAAK,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAExF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,iCAAiC,GAAG,SAAAA,CAAUC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACrG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMC,sBAAsB,GAAG,IAAAC,+CAAqB,EAAET,KAAK,EAAEC,SAAS,CAAEM,CAAC,CAAE,EAAEL,MAAM,EAAEC,qBAAsB,CAAC;IAE5G,IAAKK,sBAAsB,CAACE,kBAAkB,IAAI,EAAE,EAAG;MACtDN,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EAEA,OAAOH,cAAc;AACtB,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAZAO,OAAA,CAAAZ,iCAAA,GAAAA,iCAAA;AAaA,MAAMa,kCAAkC,GAAG,SAAAA,CAAUZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACtG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMM,eAAe,GAAGZ,SAAS,CAAEM,CAAC,CAAE;IACtC,MAAMC,sBAAsB,GAAG,IAAAC,+CAAqB,EAAET,KAAK,EAAEa,eAAe,EAAEX,MAAM,EAAEC,qBAAsB,CAAC;IAC7G,IAAKK,sBAAsB,CAACE,kBAAkB,KAAK,GAAG,EAAG;MACxDN,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EACA,OAAOH,cAAc;AACtB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AANAO,OAAA,CAAAC,kCAAA,GAAAA,kCAAA;AAOA,MAAME,sBAAsB,GAAG,SAAAA,CAAUV,cAAc,EAAG;EACzD,MAAMW,wBAAwB,GAAGX,cAAc,CAAE,CAAC,CAAE,CAACY,GAAG,CAAE,UAAUC,GAAG,EAAEV,CAAC,EAAG;IAC5E,OAAOH,cAAc,CAACY,GAAG,CAAE,UAAUE,GAAG,EAAG;MAC1C,OAAOA,GAAG,CAAEX,CAAC,CAAE;IAChB,CAAE,CAAC;EACJ,CAAE,CAAC;EAEH,OAAOQ,wBAAwB,CAACC,GAAG,CAAE,UAAUG,oBAAoB,EAAG;IACrE,OAAO,IAAAC,WAAG,EAAED,oBAAqB,CAAC;EACnC,CAAE,CAAC;AACJ,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AANAR,OAAA,CAAAG,sBAAA,GAAAA,sBAAA;AAOA,MAAMO,cAAc,GAAG,SAAAA,CAAUjB,cAAc,EAAG;EACjD,MAAMkB,iBAAiB,GAAGlB,cAAc,CAACE,MAAM;EAC/C,MAAMiB,wBAAwB,GAAG,EAAE;EAEnC,KAAM,IAAIhB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGe,iBAAiB,EAAEf,CAAC,EAAE,EAAG;IAC7C,IAAKH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC,EAAG;MAC9BgB,wBAAwB,CAACC,IAAI,CAAEjB,CAAE,CAAC;IACnC;EACD;EAEA,MAAMkB,sBAAsB,GAAGF,wBAAwB,CAACjB,MAAM;EAE9D,IAAKmB,sBAAsB,KAAK,CAAC,EAAG;IACnC,OAAOH,iBAAiB;EACzB;;EAEA;AACD;AACA;AACA;EACCC,wBAAwB,CAACG,OAAO,CAAE,CAAC,CAAE,CAAC;EACtCH,wBAAwB,CAACC,IAAI,CAAEF,iBAAkB,CAAC;EAElD,MAAMK,SAAS,GAAG,EAAE;EAEpB,KAAM,IAAIpB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGkB,sBAAsB,GAAG,CAAC,EAAElB,CAAC,EAAE,EAAG;IACtDoB,SAAS,CAACH,IAAI,CAAED,wBAAwB,CAAEhB,CAAC,CAAE,GAAGgB,wBAAwB,CAAEhB,CAAC,GAAG,CAAC,CAAE,GAAG,CAAE,CAAC;EACxF;EAEA,OAAO,IAAAa,WAAG,EAAEO,SAAU,CAAC;AACxB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAdAhB,OAAA,CAAAU,cAAA,GAAAA,cAAA;AAeA,MAAMO,iBAAiB,GAAG,SAAAA,CAAU3B,SAAS,EAAE4B,oBAAoB,EAAE3B,MAAM,EAAE4B,aAAa,EAAE3B,qBAAqB,EAChH4B,mBAAmB,GAAG,CAAC,EAAEC,aAAa,EAAEC,mBAAmB,EAAG;EAC9D;EACA,MAAMC,WAAW,GAAGL,oBAAoB,CAACvB,MAAM;EAE/C,MAAMF,cAAc,GAAGC,KAAK,CAAE6B,WAAY,CAAC;;EAE3C;EACA,IAAKJ,aAAa,CAACxB,MAAM,GAAG,CAAC,EAAG;IAC/B,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2B,WAAW,EAAE3B,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG6B,oBAAoB,CAAEtB,CAAC,CAAE;MACvC;AACH;AACA;AACA;AACA;MACG,MAAM4B,WAAW,GAAGF,mBAAmB,GAAGA,mBAAmB,CAAED,aAAa,CAAEzB,CAAC,CAAG,CAAC,GAAGP,KAAK,CAACM,MAAM;MAClG,IAAK6B,WAAW,GAAGJ,mBAAmB,EAAG;QACxC3B,cAAc,CAAEG,CAAC,CAAE,GAAGK,kCAAkC,CAAEZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC5G,CAAC,MAAM;QACNC,cAAc,CAAEG,CAAC,CAAE,GAAGR,iCAAiC,CAAEC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC3G;IACD;EACD,CAAC,MAAM;IACN;IACA,KAAM,IAAII,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2B,WAAW,EAAE3B,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG6B,oBAAoB,CAAEtB,CAAC,CAAE;MACvCH,cAAc,CAAEG,CAAC,CAAE,GAAGK,kCAAkC,CAAEZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;IAC5G;EACD;;EAEA;EACA,MAAMiC,uBAAuB,GAAGtB,sBAAsB,CAAEV,cAAe,CAAC;;EAExE;EACA,MAAMiC,4BAA4B,GAAI,IAAAC,eAAO,EAAErC,SAAS,EAAEmC,uBAAuB,EAAE,CAAEG,QAAQ,EAAEC,KAAK,KAAM;IACzG,OAAO;MAAED,QAAQ;MAAEC;IAAM,CAAC;EAC3B,CAAE,CAAC;;EAEH;EACA,MAAMC,kBAAkB,GAAGJ,4BAA4B,CAACK,MAAM,CAAEC,cAAc,IAAIA,cAAc,CAACH,KAAK,GAAG,CAAE,CAAC;EAE5G,OAAO;IACNJ,uBAAuB,EAAEA,uBAAuB;IAChDK,kBAAkB,EAAEA,kBAAkB,CAACzB,GAAG,CAAE2B,cAAc,IAAIA,cAAc,CAACJ,QAAS;EACvF,CAAC;AACF,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMK,+BAA+B,GAAG,SAAAA,CAAUC,KAAK,EAAEC,UAAU,EAAG;EACrE,MAAMhB,aAAa,GAAGgB,UAAU,CAACC,SAAS,CAAE,eAAgB,CAAC;EAC7D,MAAM5C,qBAAqB,GAAG2C,UAAU,CAACE,SAAS,CAAE,uBAAwB,CAAC;EAC7E,MAAMC,qBAAqB,GAAGH,UAAU,CAACE,SAAS,CAAE,iBAAkB,CAAC;EACvE,MAAMf,mBAAmB,GAAGa,UAAU,CAACI,WAAW,CAAE,qBAAsB,CAAC;EAC3E,MAAMC,iBAAiB,GAAGL,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;;EAErE;EACA,MAAMjB,mBAAmB,GAAGe,UAAU,CAACC,SAAS,CAAE,aAAc,CAAC,CAACK,cAAc;EAEhF,IAAIC,IAAI,GAAGR,KAAK,CAACS,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,kDAAwB,EAAEH,IAAI,EAAER,KAAK,CAACY,WAAW,IAAIZ,KAAK,CAACY,WAAW,CAACC,UAAW,CAAC;EAC1FL,IAAI,GAAG,IAAAM,8BAAc,EAAEN,IAAK,CAAC;EAC7B,MAAMpD,SAAS,GAAG,IAAA2D,qBAAY,EAAEP,IAAI,EAAEF,iBAAkB,CAAC;EACzD,MAAMU,UAAU,GAAGf,UAAU,CAACI,WAAW,CAAE,YAAa,CAAC;EAEzD,MAAMlB,aAAa,GAAG,EAAE;EACxB,IAAKiB,qBAAqB,EAAG;IAC5BjB,aAAa,CAACR,IAAI,CAAEyB,qBAAqB,CAAEJ,KAAK,CAACiB,UAAU,CAAC,CAAE,CAAE,CAAC;IACjE,IAAAC,sBAAa,EAAElB,KAAK,CAACmB,WAAW,CAAC,CAAE,CAAC,CAACC,OAAO,CAAEC,OAAO,IAAIlC,aAAa,CAACR,IAAI,CAAEyB,qBAAqB,CAAEiB,OAAQ,CAAE,CAAE,CAAC;EAClH;EACA,MAAMhE,MAAM,GAAG2C,KAAK,CAACsB,SAAS,CAAC,CAAC;EAChC,MAAMtC,oBAAoB,GAAG,CAAEgC,UAAU,CAACO,cAAc,CAAE;EAC1DP,UAAU,CAACQ,aAAa,CAACJ,OAAO,CAAE,UAAUC,OAAO,EAAG;IACrDrC,oBAAoB,CAACL,IAAI,CAAE0C,OAAQ,CAAC;EACrC,CAAE,CAAC;EAEH,MAAMI,aAAa,GAAG,IAAAC,YAAM,EAAE,IAAAC,mBAAW,EAAE3C,oBAAqB,CAAE,CAAC,CAAC4C,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMA,CAAC,CAACrE,MAAM,GAAGoE,CAAC,CAACpE,MAAO,CAAC;;EAE3G;EACA,MAAMF,cAAc,GAAGwB,iBAAiB,CAAE3B,SAAS,EAAE4B,oBAAoB,EAAE3B,MAAM,EAAE4B,aAAa,EAAE3B,qBAAqB,EACtH4B,mBAAmB,EAAEC,aAAa,EAAEC,mBAAoB,CAAC;EAC1D,MAAMG,uBAAuB,GAAGhC,cAAc,CAACgC,uBAAuB;EACtE,MAAMwC,oBAAoB,GAAGvD,cAAc,CAAEe,uBAAwB,CAAC;EAEtE,OAAO;IACNyC,oBAAoB,EAAE,IAAAC,0CAAoB,EAAER,aAAa,EAAElE,cAAc,CAACqC,kBAAkB,EAAEvC,MAAM,EAAEC,qBAAsB,CAAC;IAC7H4E,0BAA0B,EAAEH,oBAAoB,GAAG3E,SAAS,CAACK,MAAM,GAAG;EACvE,CAAC;AACF,CAAC;AAACK,OAAA,CAAAiC,+BAAA,GAAAA,+BAAA;AAAA,IAAAoC,QAAA,GAAArE,OAAA,CAAAb,OAAA,GAUa8C,+BAA+B","ignoreList":[]}
+{"version":3,"file":"keyphraseDistribution.js","names":["flattenDeep","max","uniq","unique","zipWith","findWordFormsInString","markWordsInSentences","getSentences","parseSynonyms","mergeListItems","removeHtmlBlocks","filterShortcodesFromHTML","computeScoresPerSentenceLongTopic","topic","sentences","locale","matchWordCustomHelper","sentenceScores","Array","length","i","foundInCurrentSentence","percentWordMatches","computeScoresPerSentenceShortTopic","currentSentence","maximizeSentenceScores","sentenceScoresTransposed","map","col","row","scoresForOneSentence","getDistraction","numberOfSentences","allTopicSentencesIndices","push","numberOfTopicSentences","unshift","distances","getSentenceScores","topicFormsInOneArray","functionWords","topicLengthCriteria","originalTopic","wordsCharacterCount","topicNumber","topicLength","maximizedSentenceScores","sentencesWithMaximizedScores","sentence","score","sentencesWithTopic","filter","sentenceObject","keyphraseDistributionResearcher","paper","researcher","getConfig","getHelper","getContentWordsHelper","getResearch","memoizedTokenizer","lengthCriteria","text","getText","_attributes","shortcodes","topicForms","getKeyword","getSynonyms","forEach","synonym","getLocale","keyphraseForms","synonymsForms","allTopicWords","sort","a","b","maxLengthDistraction","sentencesToHighlight","keyphraseDistributionScore"],"sources":["../../../src/languageProcessing/researches/keyphraseDistribution.js"],"sourcesContent":["import { flattenDeep, max, uniq as unique, zipWith } from \"lodash\";\nimport { findWordFormsInString } from \"../helpers/match/findKeywordFormsInString\";\nimport { markWordsInSentences } from \"../helpers/word/markWordsInSentences\";\nimport getSentences from \"../helpers/sentence/getSentences\";\nimport parseSynonyms from \"../helpers/sanitize/parseSynonyms\";\nimport { mergeListItems } from \"../helpers/sanitize/mergeListItems\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers/sanitize/filterShortcodesFromTree\";\n\n/**\n * Checks whether at least half of the content words from the topic are found within the sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if at least half of the content words from the topic are in the sentence,\n * 3 otherwise.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceLongTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, sentences[ i ], locale, matchWordCustomHelper );\n\n\t\tif ( foundInCurrentSentence.percentWordMatches >= 50 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\n\treturn sentenceScores;\n};\n\n\n/**\n * Checks whether all content words from the topic are found within one sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if all content words from the topic are in the sentence,\n * 3 if not all content words from the topic were found in the sentence.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceShortTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst currentSentence = sentences[ i ];\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, currentSentence, locale, matchWordCustomHelper );\n\t\tif ( foundInCurrentSentence.percentWordMatches === 100 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\treturn sentenceScores;\n};\n\n/**\n * Maximizes scores: Give every sentence a maximal score that it got from analysis of all topics\n *\n * @param {Array} sentenceScores The scores for every sentence, as assessed per keyphrase and every synonym.\n *\n * @returns {Array} Maximal scores of topic relevance per sentence.\n */\nconst maximizeSentenceScores = function( sentenceScores ) {\n\tconst sentenceScoresTransposed = sentenceScores[ 0 ].map( function( col, i ) {\n\t\treturn sentenceScores.map( function( row ) {\n\t\t\treturn row[ i ];\n\t\t} );\n\t} );\n\n\treturn sentenceScoresTransposed.map( function( scoresForOneSentence ) {\n\t\treturn max( scoresForOneSentence );\n\t} );\n};\n\n\n/**\n * Computes the maximally long piece of text that does not include the topic.\n *\n * @param {Array} sentenceScores The array of scores per sentence.\n *\n * @returns {number} The maximum number of sentences that do not include the topic.\n */\nconst getDistraction = function( sentenceScores ) {\n\tconst numberOfSentences = sentenceScores.length;\n\tconst allTopicSentencesIndices = [];\n\n\tfor ( let i = 0; i < numberOfSentences; i++ ) {\n\t\tif ( sentenceScores[ i ] > 3 ) {\n\t\t\tallTopicSentencesIndices.push( i );\n\t\t}\n\t}\n\n\tconst numberOfTopicSentences = allTopicSentencesIndices.length;\n\n\tif ( numberOfTopicSentences === 0 ) {\n\t\treturn numberOfSentences;\n\t}\n\n\t/**\n\t * Add fake topic sentences at the very beginning and at the very end\n\t * to account for cases when the text starts or ends with a train of distraction.\n\t */\n\tallTopicSentencesIndices.unshift( -1 );\n\tallTopicSentencesIndices.push( numberOfSentences );\n\n\tconst distances = [];\n\n\tfor ( let i = 1; i < numberOfTopicSentences + 2; i++ ) {\n\t\tdistances.push( allTopicSentencesIndices[ i ] - allTopicSentencesIndices[ i - 1 ] - 1 );\n\t}\n\n\treturn max( distances );\n};\n\n/**\n * Computes the per-sentence scores depending on the length of the topic phrase and maximizes them over all topic phrases.\n *\n * @param {Array}       sentences              The sentences to get scores for.\n * @param {Array}       topicFormsInOneArray   The topic phrases forms to search for in the sentences.\n * @param {string}      locale                 The locale to work in.\n * @param {Array}       functionWords           The function words list.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n * @param {int}         topicLengthCriteria     The topic length criteria. The default value is 4, where a topic is considered short\n *                                              if it's less than 4 word long, and otherwise long.\n * @param {Array}       originalTopic           The array of the original form of the topic with function words filtered out.\n * @param {function}    wordsCharacterCount     The helper to calculate the characters length of all the words in the array.\n *\n * @returns {Object} An array with maximized score per sentence and an array with all sentences that do not contain the topic.\n */\nconst getSentenceScores = function( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\ttopicLengthCriteria = 4, originalTopic, wordsCharacterCount ) {\n\t// Compute per-sentence scores of topic-relatedness.\n\tconst topicNumber = topicFormsInOneArray.length;\n\n\tconst sentenceScores = Array( topicNumber );\n\n\t// For languages with function words apply either full match or partial match depending on topic length\n\tif ( functionWords.length > 0 ) {\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\t/*\n\t\t\t * If the helper to calculate the characters length of all the words in the array is available,\n\t\t\t * we use this helper to calculate the characters length of the original topic form.\n\t\t\t * We then use the result and compare it with the topicLengthCriteria.\n\t\t\t */\n\t\t\tconst topicLength = wordsCharacterCount ? wordsCharacterCount( originalTopic[ i ] ) : topic.length;\n\t\t\tif ( topicLength < topicLengthCriteria ) {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t} else {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceLongTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// For languages without function words apply the full match always\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t}\n\t}\n\n\t// Maximize scores: Give every sentence a maximal score that it got from analysis of all topics\n\tconst maximizedSentenceScores = maximizeSentenceScores( sentenceScores );\n\n\t// Zip an array combining each sentence with the associated maximized score.\n\tconst sentencesWithMaximizedScores =  zipWith( sentences, maximizedSentenceScores, ( sentence, score ) => {\n\t\treturn { sentence, score };\n\t} );\n\n\t// Filter sentences that contain topic words for future highlights.\n\tconst sentencesWithTopic = sentencesWithMaximizedScores.filter( sentenceObject => sentenceObject.score > 3 );\n\n\treturn {\n\t\tmaximizedSentenceScores: maximizedSentenceScores,\n\t\tsentencesWithTopic: sentencesWithTopic.map( sentenceObject => sentenceObject.sentence ),\n\t};\n};\n\n\n/**\n * Determines which portions of the text did not receive a lot of content words from keyphrase and synonyms.\n *\n * @param {Paper}       paper               The paper to check the keyphrase distribution for.\n * @param {Researcher}  researcher          The researcher to use for analysis.\n *\n * @returns {Object} The scores of topic relevance per portion of text and an array of all word forms to highlight.\n */\nconst keyphraseDistributionResearcher = function( paper, researcher ) {\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\tconst matchWordCustomHelper = researcher.getHelper( \"matchWordCustomHelper\" );\n\tconst getContentWordsHelper = researcher.getHelper( \"getContentWords\" );\n\tconst wordsCharacterCount = researcher.getResearch( \"wordsCharacterCount\" );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\n\t// Custom topic length criteria for languages that don't use the default value to determine whether a topic is long or short.\n\tconst topicLengthCriteria = researcher.getConfig( \"topicLength\" ).lengthCriteria;\n\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\ttext = mergeListItems( text );\n\tconst sentences = getSentences( text, memoizedTokenizer );\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\n\tconst originalTopic = [];\n\tif ( getContentWordsHelper ) {\n\t\toriginalTopic.push( getContentWordsHelper( paper.getKeyword() ) );\n\t\tparseSynonyms( paper.getSynonyms() ).forEach( synonym => originalTopic.push( getContentWordsHelper( synonym ) ) );\n\t}\n\tconst locale = paper.getLocale();\n\tconst topicFormsInOneArray = [ topicForms.keyphraseForms ];\n\ttopicForms.synonymsForms.forEach( function( synonym ) {\n\t\ttopicFormsInOneArray.push( synonym );\n\t} );\n\n\tconst allTopicWords = unique( flattenDeep( topicFormsInOneArray ) ).sort( ( a, b ) => b.length - a.length );\n\n\t// Get per-sentence scores and sentences that have topic.\n\tconst sentenceScores = getSentenceScores( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\t\ttopicLengthCriteria, originalTopic, wordsCharacterCount );\n\tconst maximizedSentenceScores = sentenceScores.maximizedSentenceScores;\n\tconst maxLengthDistraction = getDistraction( maximizedSentenceScores );\n\n\treturn {\n\t\tsentencesToHighlight: markWordsInSentences( allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper ),\n\t\tkeyphraseDistributionScore: maxLengthDistraction / sentences.length * 100,\n\t};\n};\n\nexport {\n\tcomputeScoresPerSentenceShortTopic,\n\tcomputeScoresPerSentenceLongTopic,\n\tmaximizeSentenceScores,\n\tkeyphraseDistributionResearcher,\n\tgetDistraction,\n};\n\nexport default keyphraseDistributionResearcher;\n"],"mappings":"AAAA,SAASA,WAAW,EAAEC,GAAG,EAAEC,IAAI,IAAIC,MAAM,EAAEC,OAAO,QAAQ,QAAQ;AAClE,SAASC,qBAAqB;AAC9B,SAASC,oBAAoB;AAC7B,OAAOC,YAAY;AACnB,OAAOC,aAAa;AACpB,SAASC,cAAc;AACvB,OAAOC,gBAAgB;AACvB,SAASC,wBAAwB;;AAEjC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,iCAAiC,GAAG,SAAAA,CAAUC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACrG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMC,sBAAsB,GAAGhB,qBAAqB,CAAEQ,KAAK,EAAEC,SAAS,CAAEM,CAAC,CAAE,EAAEL,MAAM,EAAEC,qBAAsB,CAAC;IAE5G,IAAKK,sBAAsB,CAACC,kBAAkB,IAAI,EAAE,EAAG;MACtDL,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EAEA,OAAOH,cAAc;AACtB,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMM,kCAAkC,GAAG,SAAAA,CAAUV,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACtG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMI,eAAe,GAAGV,SAAS,CAAEM,CAAC,CAAE;IACtC,MAAMC,sBAAsB,GAAGhB,qBAAqB,CAAEQ,KAAK,EAAEW,eAAe,EAAET,MAAM,EAAEC,qBAAsB,CAAC;IAC7G,IAAKK,sBAAsB,CAACC,kBAAkB,KAAK,GAAG,EAAG;MACxDL,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EACA,OAAOH,cAAc;AACtB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMQ,sBAAsB,GAAG,SAAAA,CAAUR,cAAc,EAAG;EACzD,MAAMS,wBAAwB,GAAGT,cAAc,CAAE,CAAC,CAAE,CAACU,GAAG,CAAE,UAAUC,GAAG,EAAER,CAAC,EAAG;IAC5E,OAAOH,cAAc,CAACU,GAAG,CAAE,UAAUE,GAAG,EAAG;MAC1C,OAAOA,GAAG,CAAET,CAAC,CAAE;IAChB,CAAE,CAAC;EACJ,CAAE,CAAC;EAEH,OAAOM,wBAAwB,CAACC,GAAG,CAAE,UAAUG,oBAAoB,EAAG;IACrE,OAAO7B,GAAG,CAAE6B,oBAAqB,CAAC;EACnC,CAAE,CAAC;AACJ,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,cAAc,GAAG,SAAAA,CAAUd,cAAc,EAAG;EACjD,MAAMe,iBAAiB,GAAGf,cAAc,CAACE,MAAM;EAC/C,MAAMc,wBAAwB,GAAG,EAAE;EAEnC,KAAM,IAAIb,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGY,iBAAiB,EAAEZ,CAAC,EAAE,EAAG;IAC7C,IAAKH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC,EAAG;MAC9Ba,wBAAwB,CAACC,IAAI,CAAEd,CAAE,CAAC;IACnC;EACD;EAEA,MAAMe,sBAAsB,GAAGF,wBAAwB,CAACd,MAAM;EAE9D,IAAKgB,sBAAsB,KAAK,CAAC,EAAG;IACnC,OAAOH,iBAAiB;EACzB;;EAEA;AACD;AACA;AACA;EACCC,wBAAwB,CAACG,OAAO,CAAE,CAAC,CAAE,CAAC;EACtCH,wBAAwB,CAACC,IAAI,CAAEF,iBAAkB,CAAC;EAElD,MAAMK,SAAS,GAAG,EAAE;EAEpB,KAAM,IAAIjB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGe,sBAAsB,GAAG,CAAC,EAAEf,CAAC,EAAE,EAAG;IACtDiB,SAAS,CAACH,IAAI,CAAED,wBAAwB,CAAEb,CAAC,CAAE,GAAGa,wBAAwB,CAAEb,CAAC,GAAG,CAAC,CAAE,GAAG,CAAE,CAAC;EACxF;EAEA,OAAOnB,GAAG,CAAEoC,SAAU,CAAC;AACxB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,iBAAiB,GAAG,SAAAA,CAAUxB,SAAS,EAAEyB,oBAAoB,EAAExB,MAAM,EAAEyB,aAAa,EAAExB,qBAAqB,EAChHyB,mBAAmB,GAAG,CAAC,EAAEC,aAAa,EAAEC,mBAAmB,EAAG;EAC9D;EACA,MAAMC,WAAW,GAAGL,oBAAoB,CAACpB,MAAM;EAE/C,MAAMF,cAAc,GAAGC,KAAK,CAAE0B,WAAY,CAAC;;EAE3C;EACA,IAAKJ,aAAa,CAACrB,MAAM,GAAG,CAAC,EAAG;IAC/B,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGwB,WAAW,EAAExB,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG0B,oBAAoB,CAAEnB,CAAC,CAAE;MACvC;AACH;AACA;AACA;AACA;MACG,MAAMyB,WAAW,GAAGF,mBAAmB,GAAGA,mBAAmB,CAAED,aAAa,CAAEtB,CAAC,CAAG,CAAC,GAAGP,KAAK,CAACM,MAAM;MAClG,IAAK0B,WAAW,GAAGJ,mBAAmB,EAAG;QACxCxB,cAAc,CAAEG,CAAC,CAAE,GAAGG,kCAAkC,CAAEV,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC5G,CAAC,MAAM;QACNC,cAAc,CAAEG,CAAC,CAAE,GAAGR,iCAAiC,CAAEC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC3G;IACD;EACD,CAAC,MAAM;IACN;IACA,KAAM,IAAII,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGwB,WAAW,EAAExB,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG0B,oBAAoB,CAAEnB,CAAC,CAAE;MACvCH,cAAc,CAAEG,CAAC,CAAE,GAAGG,kCAAkC,CAAEV,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;IAC5G;EACD;;EAEA;EACA,MAAM8B,uBAAuB,GAAGrB,sBAAsB,CAAER,cAAe,CAAC;;EAExE;EACA,MAAM8B,4BAA4B,GAAI3C,OAAO,CAAEU,SAAS,EAAEgC,uBAAuB,EAAE,CAAEE,QAAQ,EAAEC,KAAK,KAAM;IACzG,OAAO;MAAED,QAAQ;MAAEC;IAAM,CAAC;EAC3B,CAAE,CAAC;;EAEH;EACA,MAAMC,kBAAkB,GAAGH,4BAA4B,CAACI,MAAM,CAAEC,cAAc,IAAIA,cAAc,CAACH,KAAK,GAAG,CAAE,CAAC;EAE5G,OAAO;IACNH,uBAAuB,EAAEA,uBAAuB;IAChDI,kBAAkB,EAAEA,kBAAkB,CAACvB,GAAG,CAAEyB,cAAc,IAAIA,cAAc,CAACJ,QAAS;EACvF,CAAC;AACF,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMK,+BAA+B,GAAG,SAAAA,CAAUC,KAAK,EAAEC,UAAU,EAAG;EACrE,MAAMf,aAAa,GAAGe,UAAU,CAACC,SAAS,CAAE,eAAgB,CAAC;EAC7D,MAAMxC,qBAAqB,GAAGuC,UAAU,CAACE,SAAS,CAAE,uBAAwB,CAAC;EAC7E,MAAMC,qBAAqB,GAAGH,UAAU,CAACE,SAAS,CAAE,iBAAkB,CAAC;EACvE,MAAMd,mBAAmB,GAAGY,UAAU,CAACI,WAAW,CAAE,qBAAsB,CAAC;EAC3E,MAAMC,iBAAiB,GAAGL,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;;EAErE;EACA,MAAMhB,mBAAmB,GAAGc,UAAU,CAACC,SAAS,CAAE,aAAc,CAAC,CAACK,cAAc;EAEhF,IAAIC,IAAI,GAAGR,KAAK,CAACS,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAGpD,gBAAgB,CAAEoD,IAAK,CAAC;EAC/BA,IAAI,GAAGnD,wBAAwB,CAAEmD,IAAI,EAAER,KAAK,CAACU,WAAW,IAAIV,KAAK,CAACU,WAAW,CAACC,UAAW,CAAC;EAC1FH,IAAI,GAAGrD,cAAc,CAAEqD,IAAK,CAAC;EAC7B,MAAMhD,SAAS,GAAGP,YAAY,CAAEuD,IAAI,EAAEF,iBAAkB,CAAC;EACzD,MAAMM,UAAU,GAAGX,UAAU,CAACI,WAAW,CAAE,YAAa,CAAC;EAEzD,MAAMjB,aAAa,GAAG,EAAE;EACxB,IAAKgB,qBAAqB,EAAG;IAC5BhB,aAAa,CAACR,IAAI,CAAEwB,qBAAqB,CAAEJ,KAAK,CAACa,UAAU,CAAC,CAAE,CAAE,CAAC;IACjE3D,aAAa,CAAE8C,KAAK,CAACc,WAAW,CAAC,CAAE,CAAC,CAACC,OAAO,CAAEC,OAAO,IAAI5B,aAAa,CAACR,IAAI,CAAEwB,qBAAqB,CAAEY,OAAQ,CAAE,CAAE,CAAC;EAClH;EACA,MAAMvD,MAAM,GAAGuC,KAAK,CAACiB,SAAS,CAAC,CAAC;EAChC,MAAMhC,oBAAoB,GAAG,CAAE2B,UAAU,CAACM,cAAc,CAAE;EAC1DN,UAAU,CAACO,aAAa,CAACJ,OAAO,CAAE,UAAUC,OAAO,EAAG;IACrD/B,oBAAoB,CAACL,IAAI,CAAEoC,OAAQ,CAAC;EACrC,CAAE,CAAC;EAEH,MAAMI,aAAa,GAAGvE,MAAM,CAAEH,WAAW,CAAEuC,oBAAqB,CAAE,CAAC,CAACoC,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMA,CAAC,CAAC1D,MAAM,GAAGyD,CAAC,CAACzD,MAAO,CAAC;;EAE3G;EACA,MAAMF,cAAc,GAAGqB,iBAAiB,CAAExB,SAAS,EAAEyB,oBAAoB,EAAExB,MAAM,EAAEyB,aAAa,EAAExB,qBAAqB,EACtHyB,mBAAmB,EAAEC,aAAa,EAAEC,mBAAoB,CAAC;EAC1D,MAAMG,uBAAuB,GAAG7B,cAAc,CAAC6B,uBAAuB;EACtE,MAAMgC,oBAAoB,GAAG/C,cAAc,CAAEe,uBAAwB,CAAC;EAEtE,OAAO;IACNiC,oBAAoB,EAAEzE,oBAAoB,CAAEoE,aAAa,EAAEzD,cAAc,CAACiC,kBAAkB,EAAEnC,MAAM,EAAEC,qBAAsB,CAAC;IAC7HgE,0BAA0B,EAAEF,oBAAoB,GAAGhE,SAAS,CAACK,MAAM,GAAG;EACvE,CAAC;AACF,CAAC;AAED,SACCI,kCAAkC,EAClCX,iCAAiC,EACjCa,sBAAsB,EACtB4B,+BAA+B,EAC/BtB,cAAc;AAGf,eAAesB,+BAA+B","ignoreList":[]}

package/build/languageProcessing/researches/keyphraseLength.js

@@ -1,9 +1,3 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = _default;
 /**
  * Determines the length in words of the keyphrase.
  *
@@ -12,7 +6,7 @@ exports.default = _default;
  *
  * @returns {Object} The length of the keyphrase and the function words list.
  */
-function _default(paper, researcher) {
+export default function (paper, researcher) {
   const topicForms = researcher.getResearch("morphology");
   const functionWords = researcher.getConfig("functionWords");
   return {

package/build/languageProcessing/researches/keyphraseLength.js.map

@@ -1 +1 @@
-{"version":3,"file":"keyphraseLength.js","names":["_default","paper","researcher","topicForms","getResearch","functionWords","getConfig","keyphraseLength","keyphraseForms","length"],"sources":["../../../src/languageProcessing/researches/keyphraseLength.js"],"sourcesContent":["/**\n * Determines the length in words of the keyphrase.\n *\n * @param {Paper} paper \t\t\tThe paper to research\n * @param {Researcher} researcher \tThe researcher to use for analysis\n *\n * @returns {Object} The length of the keyphrase and the function words list.\n */\nexport default function( paper, researcher ) {\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\n\treturn {\n\t\tkeyphraseLength: topicForms.keyphraseForms.length,\n\t\tfunctionWords: functionWords,\n\t};\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAA,SAAUC,KAAK,EAAEC,UAAU,EAAG;EAC5C,MAAMC,UAAU,GAAGD,UAAU,CAACE,WAAW,CAAE,YAAa,CAAC;EACzD,MAAMC,aAAa,GAAGH,UAAU,CAACI,SAAS,CAAE,eAAgB,CAAC;EAE7D,OAAO;IACNC,eAAe,EAAEJ,UAAU,CAACK,cAAc,CAACC,MAAM;IACjDJ,aAAa,EAAEA;EAChB,CAAC;AACF","ignoreList":[]}
+{"version":3,"file":"keyphraseLength.js","names":["paper","researcher","topicForms","getResearch","functionWords","getConfig","keyphraseLength","keyphraseForms","length"],"sources":["../../../src/languageProcessing/researches/keyphraseLength.js"],"sourcesContent":["/**\n * Determines the length in words of the keyphrase.\n *\n * @param {Paper} paper \t\t\tThe paper to research\n * @param {Researcher} researcher \tThe researcher to use for analysis\n *\n * @returns {Object} The length of the keyphrase and the function words list.\n */\nexport default function( paper, researcher ) {\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\n\treturn {\n\t\tkeyphraseLength: topicForms.keyphraseForms.length,\n\t\tfunctionWords: functionWords,\n\t};\n}\n"],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,eAAe,UAAUA,KAAK,EAAEC,UAAU,EAAG;EAC5C,MAAMC,UAAU,GAAGD,UAAU,CAACE,WAAW,CAAE,YAAa,CAAC;EACzD,MAAMC,aAAa,GAAGH,UAAU,CAACI,SAAS,CAAE,eAAgB,CAAC;EAE7D,OAAO;IACNC,eAAe,EAAEJ,UAAU,CAACK,cAAc,CAACC,MAAM;IACjDJ,aAAa,EAAEA;EAChB,CAAC;AACF","ignoreList":[]}

package/build/languageProcessing/researches/keywordCount.js

@@ -1,21 +1,13 @@
-"use strict";
+import { flatten, flattenDeep } from 'lodash';
+import getSentencesFromTree from "../helpers/sentence/getSentencesFromTree";
+import { normalizeSingle } from "../helpers/sanitize/quotes";
+import getMarkingsInSentence from "../helpers/highlighting/getMarkingsInSentence";
+import matchWordFormsWithSentence from "../helpers/match/matchWordFormsWithSentence";
+import isDoubleQuoted from "../helpers/match/isDoubleQuoted";
+import { markWordsInASentence } from "../helpers/word/markWordsInSentences";
+import getSentences from "../helpers/sentence/getSentences";
+import { filterShortcodesFromHTML } from "../helpers";
 
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.countKeyphraseInText = countKeyphraseInText;
-exports.default = getKeyphraseCount;
-exports.keywordCount = keywordCount;
-var _lodash = require("lodash");
-var _getSentencesFromTree = _interopRequireDefault(require("../helpers/sentence/getSentencesFromTree"));
-var _quotes = require("../helpers/sanitize/quotes");
-var _getMarkingsInSentence = _interopRequireDefault(require("../helpers/highlighting/getMarkingsInSentence"));
-var _matchWordFormsWithSentence = _interopRequireDefault(require("../helpers/match/matchWordFormsWithSentence"));
-var _isDoubleQuoted = _interopRequireDefault(require("../helpers/match/isDoubleQuoted"));
-var _markWordsInSentences = require("../helpers/word/markWordsInSentences");
-var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences"));
-var _helpers = require("../helpers");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * Counts the occurrences of the keyphrase in the text and creates the Mark objects for the matches.
  *
@@ -28,14 +20,13 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
  *
  * @returns {{markings: Mark[], count: number}} The number of keyphrase occurrences in the text and the Mark objects of the matches.
  */
-function countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper) {
+export function countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper) {
   const result = {
     count: 0,
     markings: []
   };
   sentences.forEach(sentence => {
-    const matchesInSentence = keyphraseForms.map(wordForms => (0, _matchWordFormsWithSentence.default)(sentence, wordForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper));
-
+    const matchesInSentence = keyphraseForms.map(wordForms => matchWordFormsWithSentence(sentence, wordForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper));
     // A sentence has at least one full-match of the keyphrase if each word occurs at least once.
     const isEachWordFound = matchesInSentence.every(wordForms => wordForms.count > 0);
     if (isEachWordFound) {
@@ -54,13 +45,13 @@ function countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustom
        * to be counted as a full-match.
        */
       const totalMatchCount = Math.min(...counts);
-      const foundWords = (0, _lodash.flattenDeep)(matchesInSentence.map(match => match.matches));
+      const foundWords = flattenDeep(matchesInSentence.map(match => match.matches));
       let markings = [];
       if (matchWordCustomHelper) {
         // Currently, this check is only applicable for Japanese.
-        markings = (0, _markWordsInSentences.markWordsInASentence)(sentence, foundWords, matchWordCustomHelper);
+        markings = markWordsInASentence(sentence, foundWords, matchWordCustomHelper);
       } else {
-        markings = (0, _getMarkingsInSentence.default)(sentence, foundWords);
+        markings = getMarkingsInSentence(sentence, foundWords);
       }
       result.count += totalMatchCount;
       result.markings.push(markings);
@@ -77,13 +68,13 @@ function countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustom
  *
  * @returns {{count: number, markings: Mark[], keyphraseLength: number}} An object containing the keyphrase count, markings and the kephrase length.
  */
-function getKeyphraseCount(paper, researcher) {
+export default function getKeyphraseCount(paper, researcher) {
   const result = {
     count: 0,
     markings: [],
     keyphraseLength: 0
   };
-  const topicForms = researcher.getResearch("morphology");
+  const topicForms = researcher.getResearch('morphology');
   let keyphraseForms = topicForms.keyphraseForms;
   const keyphraseLength = keyphraseForms.length;
 
@@ -91,29 +82,29 @@ function getKeyphraseCount(paper, researcher) {
    * Normalize single quotes so that word form with different type of single quotes can still be matched.
    * For example, "key‛word" should match "key'word".
    */
-  keyphraseForms = keyphraseForms.map(word => word.map(form => (0, _quotes.normalizeSingle)(form)));
+  keyphraseForms = keyphraseForms.map(word => word.map(form => normalizeSingle(form)));
   if (keyphraseLength === 0) {
     return result;
   }
-  const matchWordCustomHelper = researcher.getHelper("matchWordCustomHelper");
-  const customSentenceTokenizer = researcher.getHelper("memoizedTokenizer");
-  const customSplitIntoTokensHelper = researcher.getHelper("splitIntoTokensCustom");
+  const matchWordCustomHelper = researcher.getHelper('matchWordCustomHelper');
+  const customSentenceTokenizer = researcher.getHelper('memoizedTokenizer');
+  const customSplitIntoTokensHelper = researcher.getHelper('splitIntoTokensCustom');
   const locale = paper.getLocale();
-  const text = matchWordCustomHelper ? (0, _helpers.filterShortcodesFromHTML)(paper.getText(), paper._attributes && paper._attributes.shortcodes) : paper.getText();
+  const text = matchWordCustomHelper ? filterShortcodesFromHTML(paper.getText(), paper._attributes && paper._attributes.shortcodes) : paper.getText();
 
   // When the custom helper is available, we're using the sentences retrieved from the text for the analysis.
-  const sentences = matchWordCustomHelper ? (0, _getSentences.default)(text, customSentenceTokenizer) : (0, _getSentencesFromTree.default)(paper);
+  const sentences = matchWordCustomHelper ? getSentences(text, customSentenceTokenizer) : getSentencesFromTree(paper);
   // Exact matching is requested when the keyphrase is enclosed in double quotes.
-  const isExactMatchRequested = (0, _isDoubleQuoted.default)(paper.getKeyword());
+  const isExactMatchRequested = isDoubleQuoted(paper.getKeyword());
 
   /*
-  * Count the amount of keyphrase occurrences in the sentences.
-  * An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.
-  * (e.g. "The apple potato is an apple and a potato." has two occurrences of the keyphrase "apple potato").
-  */
+   * Count the amount of keyphrase occurrences in the sentences.
+   * An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.
+   * (e.g. "The apple potato is an apple and a potato." has two occurrences of the keyphrase "apple potato").
+   */
   const keyphraseFound = countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper);
   result.count = keyphraseFound.count;
-  result.markings = (0, _lodash.flatten)(keyphraseFound.markings);
+  result.markings = flatten(keyphraseFound.markings);
   result.keyphraseLength = keyphraseLength;
   return result;
 }
@@ -128,8 +119,8 @@ function getKeyphraseCount(paper, researcher) {
  *
  * @returns {Object} An array of all the matches, markings and the keyphrase count.
  */
-function keywordCount(paper, researcher) {
-  console.warn("This function is deprecated, use getKeyphraseCount instead.");
+export function keywordCount(paper, researcher) {
+  console.warn('This function is deprecated, use getKeyphraseCount instead.');
   return getKeyphraseCount(paper, researcher);
 }
 //# sourceMappingURL=keywordCount.js.map

package/build/languageProcessing/researches/keywordCount.js.map

@@ -1 +1 @@
-{"version":3,"file":"keywordCount.js","names":["_lodash","require","_getSentencesFromTree","_interopRequireDefault","_quotes","_getMarkingsInSentence","_matchWordFormsWithSentence","_isDoubleQuoted","_markWordsInSentences","_getSentences","_helpers","e","__esModule","default","countKeyphraseInText","sentences","keyphraseForms","locale","matchWordCustomHelper","isExactMatchRequested","customSplitIntoTokensHelper","result","count","markings","forEach","sentence","matchesInSentence","map","wordForms","matchWordFormsWithSentence","isEachWordFound","every","counts","match","totalMatchCount","Math","min","foundWords","flattenDeep","matches","markWordsInASentence","getMarkingsInSentence","push","getKeyphraseCount","paper","researcher","keyphraseLength","topicForms","getResearch","length","word","form","normalizeSingle","getHelper","customSentenceTokenizer","getLocale","text","filterShortcodesFromHTML","getText","_attributes","shortcodes","getSentences","getSentencesFromTree","isDoubleQuoted","getKeyword","keyphraseFound","flatten","keywordCount","console","warn"],"sources":["../../../src/languageProcessing/researches/keywordCount.js"],"sourcesContent":["import { flatten, flattenDeep } from \"lodash\";\nimport getSentencesFromTree from \"../helpers/sentence/getSentencesFromTree\";\nimport { normalizeSingle } from \"../helpers/sanitize/quotes\";\nimport getMarkingsInSentence from \"../helpers/highlighting/getMarkingsInSentence\";\nimport matchWordFormsWithSentence from \"../helpers/match/matchWordFormsWithSentence\";\nimport isDoubleQuoted from \"../helpers/match/isDoubleQuoted\";\nimport { markWordsInASentence } from \"../helpers/word/markWordsInSentences\";\nimport getSentences from \"../helpers/sentence/getSentences\";\nimport { filterShortcodesFromHTML } from \"../helpers\";\n\n/**\n * Counts the occurrences of the keyphrase in the text and creates the Mark objects for the matches.\n *\n * @param {(Sentence|string)[]}\tsentences\t\t\tThe sentences to check.\n * @param {Array}\t\tkeyphraseForms\t\t\t\tThe keyphrase forms.\n * @param {string}\t\tlocale\t\t\t\t\t\tThe locale used in the analysis.\n * @param {function}\tmatchWordCustomHelper\t\tA custom helper to match words with a text.\n * @param {boolean}\t\tisExactMatchRequested\t\tWhether the exact matching is requested.\n * @param {function}\tcustomSplitIntoTokensHelper\tA custom helper to split sentences into tokens.\n *\n * @returns {{markings: Mark[], count: number}} The number of keyphrase occurrences in the text and the Mark objects of the matches.\n */\nexport function countKeyphraseInText( sentences, keyphraseForms, locale, matchWordCustomHelper,\n\t\t\t\t\t\t\t\t\t  isExactMatchRequested, customSplitIntoTokensHelper ) {\n\tconst result = { count: 0, markings: [] };\n\n\tsentences.forEach( sentence => {\n\t\tconst matchesInSentence = keyphraseForms.map( wordForms => matchWordFormsWithSentence( sentence,\n\t\t\twordForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper ) );\n\n\t\t// A sentence has at least one full-match of the keyphrase if each word occurs at least once.\n\t\tconst isEachWordFound = matchesInSentence.every( wordForms => wordForms.count > 0 );\n\n\t\tif ( isEachWordFound ) {\n\t\t\t/*\n\t\t\t * Retrieve all the occurrences' count of each word of the keyphrase and save it in an array.\n\t\t\t * matches: [ [ { matches: [\"red\"], count: 1 } ], [ { matches: [\"pandas\"], count: 2 } ] ]\n\t\t\t * counts: [ 1, 2 ]\n\t\t\t */\n\t\t\tconst counts = matchesInSentence.map( match => match.count );\n\t\t\t/*\n\t\t\t * The number of the full-match count is the lowest count of the occurrences.\n\t\t\t * counts: [ 1, 2 ]\n\t\t\t * totalMatchCount: 1\n\t\t\t *\n\t\t\t * From the example above, the full-match is 1, because one of the \"pandas\" occurrences is not accompanied by \"red\"\n\t\t\t * to be counted as a full-match.\n\t\t\t */\n\t\t\tconst totalMatchCount = Math.min( ...counts );\n\t\t\tconst foundWords = flattenDeep( matchesInSentence.map( match => match.matches ) );\n\n\t\t\tlet markings = [];\n\n\t\t\tif ( matchWordCustomHelper ) {\n\t\t\t\t// Currently, this check is only applicable for Japanese.\n\t\t\t\tmarkings = markWordsInASentence( sentence, foundWords, matchWordCustomHelper );\n\t\t\t} else {\n\t\t\t\tmarkings = getMarkingsInSentence( sentence, foundWords );\n\t\t\t}\n\n\t\t\tresult.count += totalMatchCount;\n\t\t\tresult.markings.push( markings );\n\t\t}\n\t} );\n\n\treturn result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {{count: number, markings: Mark[], keyphraseLength: number}} An object containing the keyphrase count, markings and the kephrase length.\n */\nexport default function getKeyphraseCount( paper, researcher ) {\n\tconst result = { count: 0, markings: [], keyphraseLength: 0 };\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\tlet keyphraseForms = topicForms.keyphraseForms;\n\tconst keyphraseLength = keyphraseForms.length;\n\n\t/*\n\t * Normalize single quotes so that word form with different type of single quotes can still be matched.\n\t * For example, \"key‛word\" should match \"key'word\".\n\t */\n\tkeyphraseForms = keyphraseForms.map( word => word.map( form => normalizeSingle( form ) ) );\n\n\tif ( keyphraseLength === 0 ) {\n\t\treturn result;\n\t}\n\n\tconst matchWordCustomHelper = researcher.getHelper( \"matchWordCustomHelper\" );\n\tconst customSentenceTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\tconst customSplitIntoTokensHelper = researcher.getHelper( \"splitIntoTokensCustom\" );\n\tconst locale = paper.getLocale();\n\tconst text = matchWordCustomHelper\n\t\t? filterShortcodesFromHTML( paper.getText(), paper._attributes && paper._attributes.shortcodes )\n\t\t: paper.getText();\n\n\t// When the custom helper is available, we're using the sentences retrieved from the text for the analysis.\n\tconst sentences = matchWordCustomHelper ? getSentences( text, customSentenceTokenizer ) : getSentencesFromTree( paper );\n\t// Exact matching is requested when the keyphrase is enclosed in double quotes.\n\tconst isExactMatchRequested = isDoubleQuoted( paper.getKeyword() );\n\n\t/*\n\t* Count the amount of keyphrase occurrences in the sentences.\n\t* An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.\n\t* (e.g. \"The apple potato is an apple and a potato.\" has two occurrences of the keyphrase \"apple potato\").\n\t*/\n\tconst keyphraseFound = countKeyphraseInText( sentences, keyphraseForms, locale, matchWordCustomHelper,\n\t\tisExactMatchRequested, customSplitIntoTokensHelper );\n\n\tresult.count = keyphraseFound.count;\n\tresult.markings = flatten( keyphraseFound.markings );\n\tresult.keyphraseLength = keyphraseLength;\n\n\treturn result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @deprecated Use getKeyphraseCount instead.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {Object} An array of all the matches, markings and the keyphrase count.\n */\nexport function keywordCount( paper, researcher ) {\n\tconsole.warn( \"This function is deprecated, use getKeyphraseCount instead.\" );\n\treturn getKeyphraseCount( paper, researcher );\n}\n"],"mappings":";;;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AACA,IAAAI,sBAAA,GAAAF,sBAAA,CAAAF,OAAA;AACA,IAAAK,2BAAA,GAAAH,sBAAA,CAAAF,OAAA;AACA,IAAAM,eAAA,GAAAJ,sBAAA,CAAAF,OAAA;AACA,IAAAO,qBAAA,GAAAP,OAAA;AACA,IAAAQ,aAAA,GAAAN,sBAAA,CAAAF,OAAA;AACA,IAAAS,QAAA,GAAAT,OAAA;AAAsD,SAAAE,uBAAAQ,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEtD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASG,oBAAoBA,CAAEC,SAAS,EAAEC,cAAc,EAAEC,MAAM,EAAEC,qBAAqB,EACnFC,qBAAqB,EAAEC,2BAA2B,EAAG;EAC/D,MAAMC,MAAM,GAAG;IAAEC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE;EAAG,CAAC;EAEzCR,SAAS,CAACS,OAAO,CAAEC,QAAQ,IAAI;IAC9B,MAAMC,iBAAiB,GAAGV,cAAc,CAACW,GAAG,CAAEC,SAAS,IAAI,IAAAC,mCAA0B,EAAEJ,QAAQ,EAC9FG,SAAS,EAAEX,MAAM,EAAEC,qBAAqB,EAAEC,qBAAqB,EAAEC,2BAA4B,CAAE,CAAC;;IAEjG;IACA,MAAMU,eAAe,GAAGJ,iBAAiB,CAACK,KAAK,CAAEH,SAAS,IAAIA,SAAS,CAACN,KAAK,GAAG,CAAE,CAAC;IAEnF,IAAKQ,eAAe,EAAG;MACtB;AACH;AACA;AACA;AACA;MACG,MAAME,MAAM,GAAGN,iBAAiB,CAACC,GAAG,CAAEM,KAAK,IAAIA,KAAK,CAACX,KAAM,CAAC;MAC5D;AACH;AACA;AACA;AACA;AACA;AACA;AACA;MACG,MAAMY,eAAe,GAAGC,IAAI,CAACC,GAAG,CAAE,GAAGJ,MAAO,CAAC;MAC7C,MAAMK,UAAU,GAAG,IAAAC,mBAAW,EAAEZ,iBAAiB,CAACC,GAAG,CAAEM,KAAK,IAAIA,KAAK,CAACM,OAAQ,CAAE,CAAC;MAEjF,IAAIhB,QAAQ,GAAG,EAAE;MAEjB,IAAKL,qBAAqB,EAAG;QAC5B;QACAK,QAAQ,GAAG,IAAAiB,0CAAoB,EAAEf,QAAQ,EAAEY,UAAU,EAAEnB,qBAAsB,CAAC;MAC/E,CAAC,MAAM;QACNK,QAAQ,GAAG,IAAAkB,8BAAqB,EAAEhB,QAAQ,EAAEY,UAAW,CAAC;MACzD;MAEAhB,MAAM,CAACC,KAAK,IAAIY,eAAe;MAC/Bb,MAAM,CAACE,QAAQ,CAACmB,IAAI,CAAEnB,QAAS,CAAC;IACjC;EACD,CAAE,CAAC;EAEH,OAAOF,MAAM;AACd;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASsB,iBAAiBA,CAAEC,KAAK,EAAEC,UAAU,EAAG;EAC9D,MAAMxB,MAAM,GAAG;IAAEC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE,EAAE;IAAEuB,eAAe,EAAE;EAAE,CAAC;EAC7D,MAAMC,UAAU,GAAGF,UAAU,CAACG,WAAW,CAAE,YAAa,CAAC;EACzD,IAAIhC,cAAc,GAAG+B,UAAU,CAAC/B,cAAc;EAC9C,MAAM8B,eAAe,GAAG9B,cAAc,CAACiC,MAAM;;EAE7C;AACD;AACA;AACA;EACCjC,cAAc,GAAGA,cAAc,CAACW,GAAG,CAAEuB,IAAI,IAAIA,IAAI,CAACvB,GAAG,CAAEwB,IAAI,IAAI,IAAAC,uBAAe,EAAED,IAAK,CAAE,CAAE,CAAC;EAE1F,IAAKL,eAAe,KAAK,CAAC,EAAG;IAC5B,OAAOzB,MAAM;EACd;EAEA,MAAMH,qBAAqB,GAAG2B,UAAU,CAACQ,SAAS,CAAE,uBAAwB,CAAC;EAC7E,MAAMC,uBAAuB,GAAGT,UAAU,CAACQ,SAAS,CAAE,mBAAoB,CAAC;EAC3E,MAAMjC,2BAA2B,GAAGyB,UAAU,CAACQ,SAAS,CAAE,uBAAwB,CAAC;EACnF,MAAMpC,MAAM,GAAG2B,KAAK,CAACW,SAAS,CAAC,CAAC;EAChC,MAAMC,IAAI,GAAGtC,qBAAqB,GAC/B,IAAAuC,iCAAwB,EAAEb,KAAK,CAACc,OAAO,CAAC,CAAC,EAAEd,KAAK,CAACe,WAAW,IAAIf,KAAK,CAACe,WAAW,CAACC,UAAW,CAAC,GAC9FhB,KAAK,CAACc,OAAO,CAAC,CAAC;;EAElB;EACA,MAAM3C,SAAS,GAAGG,qBAAqB,GAAG,IAAA2C,qBAAY,EAAEL,IAAI,EAAEF,uBAAwB,CAAC,GAAG,IAAAQ,6BAAoB,EAAElB,KAAM,CAAC;EACvH;EACA,MAAMzB,qBAAqB,GAAG,IAAA4C,uBAAc,EAAEnB,KAAK,CAACoB,UAAU,CAAC,CAAE,CAAC;;EAElE;AACD;AACA;AACA;AACA;EACC,MAAMC,cAAc,GAAGnD,oBAAoB,CAAEC,SAAS,EAAEC,cAAc,EAAEC,MAAM,EAAEC,qBAAqB,EACpGC,qBAAqB,EAAEC,2BAA4B,CAAC;EAErDC,MAAM,CAACC,KAAK,GAAG2C,cAAc,CAAC3C,KAAK;EACnCD,MAAM,CAACE,QAAQ,GAAG,IAAA2C,eAAO,EAAED,cAAc,CAAC1C,QAAS,CAAC;EACpDF,MAAM,CAACyB,eAAe,GAAGA,eAAe;EAExC,OAAOzB,MAAM;AACd;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAAS8C,YAAYA,CAAEvB,KAAK,EAAEC,UAAU,EAAG;EACjDuB,OAAO,CAACC,IAAI,CAAE,6DAA8D,CAAC;EAC7E,OAAO1B,iBAAiB,CAAEC,KAAK,EAAEC,UAAW,CAAC;AAC9C","ignoreList":[]}
+{"version":3,"file":"keywordCount.js","names":["flatten","flattenDeep","getSentencesFromTree","normalizeSingle","getMarkingsInSentence","matchWordFormsWithSentence","isDoubleQuoted","markWordsInASentence","getSentences","filterShortcodesFromHTML","countKeyphraseInText","sentences","keyphraseForms","locale","matchWordCustomHelper","isExactMatchRequested","customSplitIntoTokensHelper","result","count","markings","forEach","sentence","matchesInSentence","map","wordForms","isEachWordFound","every","counts","match","totalMatchCount","Math","min","foundWords","matches","push","getKeyphraseCount","paper","researcher","keyphraseLength","topicForms","getResearch","length","word","form","getHelper","customSentenceTokenizer","getLocale","text","getText","_attributes","shortcodes","getKeyword","keyphraseFound","keywordCount","console","warn"],"sources":["../../../src/languageProcessing/researches/keywordCount.js"],"sourcesContent":["import {flatten, flattenDeep} from 'lodash';\nimport getSentencesFromTree from '../helpers/sentence/getSentencesFromTree';\nimport {normalizeSingle} from '../helpers/sanitize/quotes';\nimport getMarkingsInSentence from '../helpers/highlighting/getMarkingsInSentence';\nimport matchWordFormsWithSentence from '../helpers/match/matchWordFormsWithSentence';\nimport isDoubleQuoted from '../helpers/match/isDoubleQuoted';\nimport {markWordsInASentence} from '../helpers/word/markWordsInSentences';\nimport getSentences from '../helpers/sentence/getSentences';\nimport {filterShortcodesFromHTML} from '../helpers';\n\n/**\n * Counts the occurrences of the keyphrase in the text and creates the Mark objects for the matches.\n *\n * @param {(Sentence|string)[]}\tsentences\t\t\tThe sentences to check.\n * @param {Array}\t\tkeyphraseForms\t\t\t\tThe keyphrase forms.\n * @param {string}\t\tlocale\t\t\t\t\t\tThe locale used in the analysis.\n * @param {function}\tmatchWordCustomHelper\t\tA custom helper to match words with a text.\n * @param {boolean}\t\tisExactMatchRequested\t\tWhether the exact matching is requested.\n * @param {function}\tcustomSplitIntoTokensHelper\tA custom helper to split sentences into tokens.\n *\n * @returns {{markings: Mark[], count: number}} The number of keyphrase occurrences in the text and the Mark objects of the matches.\n */\nexport function countKeyphraseInText(\n  sentences,\n  keyphraseForms,\n  locale,\n  matchWordCustomHelper,\n  isExactMatchRequested,\n  customSplitIntoTokensHelper\n) {\n  const result = {count: 0, markings: []};\n\n  sentences.forEach(sentence => {\n    const matchesInSentence = keyphraseForms.map(wordForms =>\n      matchWordFormsWithSentence(\n        sentence,\n        wordForms,\n        locale,\n        matchWordCustomHelper,\n        isExactMatchRequested,\n        customSplitIntoTokensHelper\n      )\n    );\n    // A sentence has at least one full-match of the keyphrase if each word occurs at least once.\n    const isEachWordFound = matchesInSentence.every(wordForms => wordForms.count > 0);\n\n    if (isEachWordFound) {\n      /*\n       * Retrieve all the occurrences' count of each word of the keyphrase and save it in an array.\n       * matches: [ [ { matches: [\"red\"], count: 1 } ], [ { matches: [\"pandas\"], count: 2 } ] ]\n       * counts: [ 1, 2 ]\n       */\n      const counts = matchesInSentence.map(match => match.count);\n      /*\n       * The number of the full-match count is the lowest count of the occurrences.\n       * counts: [ 1, 2 ]\n       * totalMatchCount: 1\n       *\n       * From the example above, the full-match is 1, because one of the \"pandas\" occurrences is not accompanied by \"red\"\n       * to be counted as a full-match.\n       */\n      const totalMatchCount = Math.min(...counts);\n      const foundWords = flattenDeep(matchesInSentence.map(match => match.matches));\n\n      let markings = [];\n\n      if (matchWordCustomHelper) {\n        // Currently, this check is only applicable for Japanese.\n        markings = markWordsInASentence(sentence, foundWords, matchWordCustomHelper);\n      } else {\n        markings = getMarkingsInSentence(sentence, foundWords);\n      }\n\n      result.count += totalMatchCount;\n      result.markings.push(markings);\n    }\n  });\n\n  return result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {{count: number, markings: Mark[], keyphraseLength: number}} An object containing the keyphrase count, markings and the kephrase length.\n */\nexport default function getKeyphraseCount(paper, researcher) {\n  const result = {count: 0, markings: [], keyphraseLength: 0};\n  const topicForms = researcher.getResearch('morphology');\n  let keyphraseForms = topicForms.keyphraseForms;\n  const keyphraseLength = keyphraseForms.length;\n\n  /*\n   * Normalize single quotes so that word form with different type of single quotes can still be matched.\n   * For example, \"key‛word\" should match \"key'word\".\n   */\n  keyphraseForms = keyphraseForms.map(word => word.map(form => normalizeSingle(form)));\n\n  if (keyphraseLength === 0) {\n    return result;\n  }\n\n  const matchWordCustomHelper = researcher.getHelper('matchWordCustomHelper');\n  const customSentenceTokenizer = researcher.getHelper('memoizedTokenizer');\n  const customSplitIntoTokensHelper = researcher.getHelper('splitIntoTokensCustom');\n  const locale = paper.getLocale();\n  const text = matchWordCustomHelper\n    ? filterShortcodesFromHTML(paper.getText(), paper._attributes && paper._attributes.shortcodes)\n    : paper.getText();\n\n  // When the custom helper is available, we're using the sentences retrieved from the text for the analysis.\n  const sentences = matchWordCustomHelper\n    ? getSentences(text, customSentenceTokenizer)\n    : getSentencesFromTree(paper);\n  // Exact matching is requested when the keyphrase is enclosed in double quotes.\n  const isExactMatchRequested = isDoubleQuoted(paper.getKeyword());\n\n  /*\n   * Count the amount of keyphrase occurrences in the sentences.\n   * An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.\n   * (e.g. \"The apple potato is an apple and a potato.\" has two occurrences of the keyphrase \"apple potato\").\n   */\n  const keyphraseFound = countKeyphraseInText(\n    sentences,\n    keyphraseForms,\n    locale,\n    matchWordCustomHelper,\n    isExactMatchRequested,\n    customSplitIntoTokensHelper\n  );\n\n  result.count = keyphraseFound.count;\n  result.markings = flatten(keyphraseFound.markings);\n  result.keyphraseLength = keyphraseLength;\n\n  return result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @deprecated Use getKeyphraseCount instead.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {Object} An array of all the matches, markings and the keyphrase count.\n */\nexport function keywordCount(paper, researcher) {\n  console.warn('This function is deprecated, use getKeyphraseCount instead.');\n  return getKeyphraseCount(paper, researcher);\n}\n"],"mappings":"AAAA,SAAQA,OAAO,EAAEC,WAAW,QAAO,QAAQ;AAC3C,OAAOC,oBAAoB;AAC3B,SAAQC,eAAe;AACvB,OAAOC,qBAAqB;AAC5B,OAAOC,0BAA0B;AACjC,OAAOC,cAAc;AACrB,SAAQC,oBAAoB;AAC5B,OAAOC,YAAY;AACnB,SAAQC,wBAAwB;;AAEhC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASC,oBAAoBA,CAClCC,SAAS,EACTC,cAAc,EACdC,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BAA2B,EAC3B;EACA,MAAMC,MAAM,GAAG;IAACC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE;EAAE,CAAC;EAEvCR,SAAS,CAACS,OAAO,CAACC,QAAQ,IAAI;IAC5B,MAAMC,iBAAiB,GAAGV,cAAc,CAACW,GAAG,CAACC,SAAS,IACpDnB,0BAA0B,CACxBgB,QAAQ,EACRG,SAAS,EACTX,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BACF,CACF,CAAC;IACD;IACA,MAAMS,eAAe,GAAGH,iBAAiB,CAACI,KAAK,CAACF,SAAS,IAAIA,SAAS,CAACN,KAAK,GAAG,CAAC,CAAC;IAEjF,IAAIO,eAAe,EAAE;MACnB;AACN;AACA;AACA;AACA;MACM,MAAME,MAAM,GAAGL,iBAAiB,CAACC,GAAG,CAACK,KAAK,IAAIA,KAAK,CAACV,KAAK,CAAC;MAC1D;AACN;AACA;AACA;AACA;AACA;AACA;AACA;MACM,MAAMW,eAAe,GAAGC,IAAI,CAACC,GAAG,CAAC,GAAGJ,MAAM,CAAC;MAC3C,MAAMK,UAAU,GAAG/B,WAAW,CAACqB,iBAAiB,CAACC,GAAG,CAACK,KAAK,IAAIA,KAAK,CAACK,OAAO,CAAC,CAAC;MAE7E,IAAId,QAAQ,GAAG,EAAE;MAEjB,IAAIL,qBAAqB,EAAE;QACzB;QACAK,QAAQ,GAAGZ,oBAAoB,CAACc,QAAQ,EAAEW,UAAU,EAAElB,qBAAqB,CAAC;MAC9E,CAAC,MAAM;QACLK,QAAQ,GAAGf,qBAAqB,CAACiB,QAAQ,EAAEW,UAAU,CAAC;MACxD;MAEAf,MAAM,CAACC,KAAK,IAAIW,eAAe;MAC/BZ,MAAM,CAACE,QAAQ,CAACe,IAAI,CAACf,QAAQ,CAAC;IAChC;EACF,CAAC,CAAC;EAEF,OAAOF,MAAM;AACf;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,eAAe,SAASkB,iBAAiBA,CAACC,KAAK,EAAEC,UAAU,EAAE;EAC3D,MAAMpB,MAAM,GAAG;IAACC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE,EAAE;IAAEmB,eAAe,EAAE;EAAC,CAAC;EAC3D,MAAMC,UAAU,GAAGF,UAAU,CAACG,WAAW,CAAC,YAAY,CAAC;EACvD,IAAI5B,cAAc,GAAG2B,UAAU,CAAC3B,cAAc;EAC9C,MAAM0B,eAAe,GAAG1B,cAAc,CAAC6B,MAAM;;EAE7C;AACF;AACA;AACA;EACE7B,cAAc,GAAGA,cAAc,CAACW,GAAG,CAACmB,IAAI,IAAIA,IAAI,CAACnB,GAAG,CAACoB,IAAI,IAAIxC,eAAe,CAACwC,IAAI,CAAC,CAAC,CAAC;EAEpF,IAAIL,eAAe,KAAK,CAAC,EAAE;IACzB,OAAOrB,MAAM;EACf;EAEA,MAAMH,qBAAqB,GAAGuB,UAAU,CAACO,SAAS,CAAC,uBAAuB,CAAC;EAC3E,MAAMC,uBAAuB,GAAGR,UAAU,CAACO,SAAS,CAAC,mBAAmB,CAAC;EACzE,MAAM5B,2BAA2B,GAAGqB,UAAU,CAACO,SAAS,CAAC,uBAAuB,CAAC;EACjF,MAAM/B,MAAM,GAAGuB,KAAK,CAACU,SAAS,CAAC,CAAC;EAChC,MAAMC,IAAI,GAAGjC,qBAAqB,GAC9BL,wBAAwB,CAAC2B,KAAK,CAACY,OAAO,CAAC,CAAC,EAAEZ,KAAK,CAACa,WAAW,IAAIb,KAAK,CAACa,WAAW,CAACC,UAAU,CAAC,GAC5Fd,KAAK,CAACY,OAAO,CAAC,CAAC;;EAEnB;EACA,MAAMrC,SAAS,GAAGG,qBAAqB,GACnCN,YAAY,CAACuC,IAAI,EAAEF,uBAAuB,CAAC,GAC3C3C,oBAAoB,CAACkC,KAAK,CAAC;EAC/B;EACA,MAAMrB,qBAAqB,GAAGT,cAAc,CAAC8B,KAAK,CAACe,UAAU,CAAC,CAAC,CAAC;;EAEhE;AACF;AACA;AACA;AACA;EACE,MAAMC,cAAc,GAAG1C,oBAAoB,CACzCC,SAAS,EACTC,cAAc,EACdC,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BACF,CAAC;EAEDC,MAAM,CAACC,KAAK,GAAGkC,cAAc,CAAClC,KAAK;EACnCD,MAAM,CAACE,QAAQ,GAAGnB,OAAO,CAACoD,cAAc,CAACjC,QAAQ,CAAC;EAClDF,MAAM,CAACqB,eAAe,GAAGA,eAAe;EAExC,OAAOrB,MAAM;AACf;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASoC,YAAYA,CAACjB,KAAK,EAAEC,UAAU,EAAE;EAC9CiB,OAAO,CAACC,IAAI,CAAC,6DAA6D,CAAC;EAC3E,OAAOpB,iBAAiB,CAACC,KAAK,EAAEC,UAAU,CAAC;AAC7C","ignoreList":[]}

package/build/languageProcessing/researches/keywordCountInUrl.js

@@ -1,16 +1,7 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-exports.keywordCountInSlug = keywordCountInSlug;
-exports.keywordCountInUrl = keywordCountInUrl;
-var _parseSlug = _interopRequireDefault(require("../helpers/url/parseSlug"));
-var _findKeywordFormsInString = require("../helpers/match/findKeywordFormsInString.js");
-var _lodash = require("lodash");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /** @module researches/keywordCountInSlug */
+import parseSlug from "../helpers/url/parseSlug";
+import { findTopicFormsInString } from "../helpers/match/findKeywordFormsInString.js";
+import { cloneDeep } from "lodash";
 
 /**
  * Matches the keyword in the slug. Replaces all dashes and underscores with whitespaces and uses whitespace as word boundary.
@@ -26,12 +17,12 @@ function keywordCountInSlug(paper, researcher) {
    * languages. This is because slugs usually use hyphens (or underscores) as word boundaries. So if only the slug is
    * split on hyphens and not the keyphrase forms, some keyphrase forms won't be found in the slug.
    */
-  const languageResearcher = (0, _lodash.cloneDeep)(researcher);
+  const languageResearcher = cloneDeep(researcher);
   languageResearcher.addConfig("areHyphensWordBoundaries", true);
   const topicForms = languageResearcher.getResearch("morphology");
-  const parsedSlug = (0, _parseSlug.default)(paper.getSlug());
+  const parsedSlug = parseSlug(paper.getSlug());
   const locale = paper.getLocale();
-  const keyphraseInSlug = (0, _findKeywordFormsInString.findTopicFormsInString)(topicForms, parsedSlug, false, locale, false);
+  const keyphraseInSlug = findTopicFormsInString(topicForms, parsedSlug, false, locale, false);
   return {
     keyphraseLength: topicForms.keyphraseForms.length,
     percentWordMatches: keyphraseInSlug.percentWordMatches
@@ -54,5 +45,6 @@ function keywordCountInUrl(paper, researcher) {
   console.warn("This function is deprecated, use keywordCountInSlug instead.");
   return keywordCountInSlug(paper, researcher);
 }
-var _default = exports.default = keywordCountInSlug;
+export { keywordCountInSlug, keywordCountInUrl };
+export default keywordCountInSlug;
 //# sourceMappingURL=keywordCountInUrl.js.map

package/build/languageProcessing/researches/keywordCountInUrl.js.map

@@ -1 +1 @@
-{"version":3,"file":"keywordCountInUrl.js","names":["_parseSlug","_interopRequireDefault","require","_findKeywordFormsInString","_lodash","e","__esModule","default","keywordCountInSlug","paper","researcher","languageResearcher","cloneDeep","addConfig","topicForms","getResearch","parsedSlug","parseSlug","getSlug","locale","getLocale","keyphraseInSlug","findTopicFormsInString","keyphraseLength","keyphraseForms","length","percentWordMatches","keywordCountInUrl","console","warn","_default","exports"],"sources":["../../../src/languageProcessing/researches/keywordCountInUrl.js"],"sourcesContent":["/** @module researches/keywordCountInSlug */\nimport parseSlug from \"../helpers/url/parseSlug\";\nimport { findTopicFormsInString } from \"../helpers/match/findKeywordFormsInString.js\";\nimport { cloneDeep } from \"lodash\";\n\n/**\n * Matches the keyword in the slug. Replaces all dashes and underscores with whitespaces and uses whitespace as word boundary.\n *\n * @param {Paper} paper the Paper object to use in this count.\n * @param {Researcher} researcher The Researcher object containing all available researches.\n *\n * @returns {{keyphraseLength: int, percentWordMatches: int}} The length of the keyphrase and the percentage of keyphrase forms that has been found.\n */\nfunction keywordCountInSlug( paper, researcher ) {\n\t/*\n\t * We want to override the `areHyphensWordBoundaries` config value from the researcher so that it is `true` for all\n\t * languages. This is because slugs usually use hyphens (or underscores) as word boundaries. So if only the slug is\n\t * split on hyphens and not the keyphrase forms, some keyphrase forms won't be found in the slug.\n\t */\n\tconst languageResearcher = cloneDeep( researcher );\n\tlanguageResearcher.addConfig( \"areHyphensWordBoundaries\", true );\n\tconst topicForms = languageResearcher.getResearch( \"morphology\" );\n\n\tconst parsedSlug = parseSlug( paper.getSlug() );\n\tconst locale = paper.getLocale();\n\n\tconst keyphraseInSlug = findTopicFormsInString( topicForms, parsedSlug, false, locale, false );\n\n\treturn {\n\t\tkeyphraseLength: topicForms.keyphraseForms.length,\n\t\tpercentWordMatches: keyphraseInSlug.percentWordMatches,\n\t};\n}\n\n/**\n * Matches the keyword in the slug.\n * keywordCountInUrl was the previous name for keywordCountInSlug (hence the name of this file).\n * We keep (and expose) this research for backwards compatibility.\n *\n * @deprecated Since version 18.7. Use keywordCountInSlug instead.\n *\n * @param {Paper} paper the Paper object to use in this count.\n * @param {Researcher} researcher The Researcher object containing all available researches.\n *\n * @returns {{keyphraseLength: int, percentWordMatches: int}} The length of the keyphrase and the percentage of keyphrase forms that has been found.\n */\nfunction keywordCountInUrl( paper, researcher ) {\n\tconsole.warn( \"This function is deprecated, use keywordCountInSlug instead.\" );\n\treturn keywordCountInSlug( paper, researcher );\n}\n\nexport {\n\tkeywordCountInSlug,\n\tkeywordCountInUrl,\n};\n\nexport default keywordCountInSlug;\n"],"mappings":";;;;;;;;AACA,IAAAA,UAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,yBAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AAAmC,SAAAD,uBAAAI,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAHnC;;AAKA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASG,kBAAkBA,CAAEC,KAAK,EAAEC,UAAU,EAAG;EAChD;AACD;AACA;AACA;AACA;EACC,MAAMC,kBAAkB,GAAG,IAAAC,iBAAS,EAAEF,UAAW,CAAC;EAClDC,kBAAkB,CAACE,SAAS,CAAE,0BAA0B,EAAE,IAAK,CAAC;EAChE,MAAMC,UAAU,GAAGH,kBAAkB,CAACI,WAAW,CAAE,YAAa,CAAC;EAEjE,MAAMC,UAAU,GAAG,IAAAC,kBAAS,EAAER,KAAK,CAACS,OAAO,CAAC,CAAE,CAAC;EAC/C,MAAMC,MAAM,GAAGV,KAAK,CAACW,SAAS,CAAC,CAAC;EAEhC,MAAMC,eAAe,GAAG,IAAAC,gDAAsB,EAAER,UAAU,EAAEE,UAAU,EAAE,KAAK,EAAEG,MAAM,EAAE,KAAM,CAAC;EAE9F,OAAO;IACNI,eAAe,EAAET,UAAU,CAACU,cAAc,CAACC,MAAM;IACjDC,kBAAkB,EAAEL,eAAe,CAACK;EACrC,CAAC;AACF;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAAElB,KAAK,EAAEC,UAAU,EAAG;EAC/CkB,OAAO,CAACC,IAAI,CAAE,8DAA+D,CAAC;EAC9E,OAAOrB,kBAAkB,CAAEC,KAAK,EAAEC,UAAW,CAAC;AAC/C;AAAC,IAAAoB,QAAA,GAAAC,OAAA,CAAAxB,OAAA,GAOcC,kBAAkB","ignoreList":[]}
+{"version":3,"file":"keywordCountInUrl.js","names":["parseSlug","findTopicFormsInString","cloneDeep","keywordCountInSlug","paper","researcher","languageResearcher","addConfig","topicForms","getResearch","parsedSlug","getSlug","locale","getLocale","keyphraseInSlug","keyphraseLength","keyphraseForms","length","percentWordMatches","keywordCountInUrl","console","warn"],"sources":["../../../src/languageProcessing/researches/keywordCountInUrl.js"],"sourcesContent":["/** @module researches/keywordCountInSlug */\nimport parseSlug from \"../helpers/url/parseSlug\";\nimport { findTopicFormsInString } from \"../helpers/match/findKeywordFormsInString.js\";\nimport { cloneDeep } from \"lodash\";\n\n/**\n * Matches the keyword in the slug. Replaces all dashes and underscores with whitespaces and uses whitespace as word boundary.\n *\n * @param {Paper} paper the Paper object to use in this count.\n * @param {Researcher} researcher The Researcher object containing all available researches.\n *\n * @returns {{keyphraseLength: int, percentWordMatches: int}} The length of the keyphrase and the percentage of keyphrase forms that has been found.\n */\nfunction keywordCountInSlug( paper, researcher ) {\n\t/*\n\t * We want to override the `areHyphensWordBoundaries` config value from the researcher so that it is `true` for all\n\t * languages. This is because slugs usually use hyphens (or underscores) as word boundaries. So if only the slug is\n\t * split on hyphens and not the keyphrase forms, some keyphrase forms won't be found in the slug.\n\t */\n\tconst languageResearcher = cloneDeep( researcher );\n\tlanguageResearcher.addConfig( \"areHyphensWordBoundaries\", true );\n\tconst topicForms = languageResearcher.getResearch( \"morphology\" );\n\n\tconst parsedSlug = parseSlug( paper.getSlug() );\n\tconst locale = paper.getLocale();\n\n\tconst keyphraseInSlug = findTopicFormsInString( topicForms, parsedSlug, false, locale, false );\n\n\treturn {\n\t\tkeyphraseLength: topicForms.keyphraseForms.length,\n\t\tpercentWordMatches: keyphraseInSlug.percentWordMatches,\n\t};\n}\n\n/**\n * Matches the keyword in the slug.\n * keywordCountInUrl was the previous name for keywordCountInSlug (hence the name of this file).\n * We keep (and expose) this research for backwards compatibility.\n *\n * @deprecated Since version 18.7. Use keywordCountInSlug instead.\n *\n * @param {Paper} paper the Paper object to use in this count.\n * @param {Researcher} researcher The Researcher object containing all available researches.\n *\n * @returns {{keyphraseLength: int, percentWordMatches: int}} The length of the keyphrase and the percentage of keyphrase forms that has been found.\n */\nfunction keywordCountInUrl( paper, researcher ) {\n\tconsole.warn( \"This function is deprecated, use keywordCountInSlug instead.\" );\n\treturn keywordCountInSlug( paper, researcher );\n}\n\nexport {\n\tkeywordCountInSlug,\n\tkeywordCountInUrl,\n};\n\nexport default keywordCountInSlug;\n"],"mappings":"AAAA;AACA,OAAOA,SAAS;AAChB,SAASC,sBAAsB;AAC/B,SAASC,SAAS,QAAQ,QAAQ;;AAElC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,kBAAkBA,CAAEC,KAAK,EAAEC,UAAU,EAAG;EAChD;AACD;AACA;AACA;AACA;EACC,MAAMC,kBAAkB,GAAGJ,SAAS,CAAEG,UAAW,CAAC;EAClDC,kBAAkB,CAACC,SAAS,CAAE,0BAA0B,EAAE,IAAK,CAAC;EAChE,MAAMC,UAAU,GAAGF,kBAAkB,CAACG,WAAW,CAAE,YAAa,CAAC;EAEjE,MAAMC,UAAU,GAAGV,SAAS,CAAEI,KAAK,CAACO,OAAO,CAAC,CAAE,CAAC;EAC/C,MAAMC,MAAM,GAAGR,KAAK,CAACS,SAAS,CAAC,CAAC;EAEhC,MAAMC,eAAe,GAAGb,sBAAsB,CAAEO,UAAU,EAAEE,UAAU,EAAE,KAAK,EAAEE,MAAM,EAAE,KAAM,CAAC;EAE9F,OAAO;IACNG,eAAe,EAAEP,UAAU,CAACQ,cAAc,CAACC,MAAM;IACjDC,kBAAkB,EAAEJ,eAAe,CAACI;EACrC,CAAC;AACF;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAAEf,KAAK,EAAEC,UAAU,EAAG;EAC/Ce,OAAO,CAACC,IAAI,CAAE,8DAA+D,CAAC;EAC9E,OAAOlB,kBAAkB,CAAEC,KAAK,EAAEC,UAAW,CAAC;AAC/C;AAEA,SACCF,kBAAkB,EAClBgB,iBAAiB;AAGlB,eAAehB,kBAAkB","ignoreList":[]}

package/build/languageProcessing/researches/matchKeywordInSubheadings.js

@@ -1,15 +1,9 @@
-"use strict";
+import { getSubheadingContentsTopLevel } from "../helpers/html/getSubheadings";
+import stripSomeTags from "../helpers/sanitize/stripNonTextTags";
+import { findTopicFormsInString } from "../helpers/match/findKeywordFormsInString";
+import removeHtmlBlocks from "../helpers/html/htmlParser";
+import { filterShortcodesFromHTML } from "../helpers";
 
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = matchKeywordInSubheadings;
-var _getSubheadings = require("../helpers/html/getSubheadings");
-var _stripNonTextTags = _interopRequireDefault(require("../helpers/sanitize/stripNonTextTags"));
-var _findKeywordFormsInString = require("../helpers/match/findKeywordFormsInString");
-var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
-var _helpers = require("../helpers");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * Computes the amount of subheadings reflecting the topic.
  *
@@ -24,7 +18,7 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
  */
 const numberOfSubheadingsReflectingTopic = function (topicForms, subheadings, useSynonyms, locale, functionWords, matchWordCustomHelper) {
   return subheadings.filter(subheading => {
-    const matchedTopicForms = (0, _findKeywordFormsInString.findTopicFormsInString)(topicForms, subheading, useSynonyms, locale, matchWordCustomHelper);
+    const matchedTopicForms = findTopicFormsInString(topicForms, subheading, useSynonyms, locale, matchWordCustomHelper);
     if (functionWords.length === 0) {
       return matchedTopicForms.percentWordMatches === 100;
     }
@@ -40,16 +34,17 @@ const numberOfSubheadingsReflectingTopic = function (topicForms, subheadings, us
  *
  * @returns {Object} The result object.
  */
-function matchKeywordInSubheadings(paper, researcher) {
-  const functionWords = researcher.getConfig("functionWords");
+export default function matchKeywordInSubheadings(paper, researcher) {
+  const functionWords = researcher.getConfig('functionWords');
 
   // A custom helper to match word in text.
-  const matchWordCustomHelper = researcher.getHelper("matchWordCustomHelper");
+  const matchWordCustomHelper = researcher.getHelper('matchWordCustomHelper');
   let text = paper.getText();
-  text = (0, _htmlParser.default)(text);
-  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
-  text = (0, _stripNonTextTags.default)(text);
-  const topicForms = researcher.getResearch("morphology");
+  text = removeHtmlBlocks(text);
+  text = filterShortcodesFromHTML(text, paper._attributes && paper._attributes.shortcodes);
+  text = stripSomeTags(text);
+  text = text.replace(/\s+/g, ' ').trim();
+  const topicForms = researcher.getResearch('morphology');
   const locale = paper.getLocale();
   const result = {
     count: 0,
@@ -57,7 +52,7 @@ function matchKeywordInSubheadings(paper, researcher) {
     percentReflectingTopic: 0
   };
   const useSynonyms = true;
-  const subheadings = (0, _getSubheadings.getSubheadingContentsTopLevel)(text);
+  const subheadings = getSubheadingContentsTopLevel(text);
   if (subheadings.length !== 0) {
     result.count = subheadings.length;
     result.matches = numberOfSubheadingsReflectingTopic(topicForms, subheadings, useSynonyms, locale, functionWords, matchWordCustomHelper);