axyseo 2.1.12 → 2.1.13

package/build/cjs/languageProcessing/researches/index.js

@@ -0,0 +1,28 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "getLongCenterAlignedTexts", {
+  enumerable: true,
+  get: function () {
+    return _getLongCenterAlignedTexts.default;
+  }
+});
+Object.defineProperty(exports, "keyphraseDistribution", {
+  enumerable: true,
+  get: function () {
+    return _keyphraseDistribution.default;
+  }
+});
+Object.defineProperty(exports, "wordComplexity", {
+  enumerable: true,
+  get: function () {
+    return _wordComplexity.default;
+  }
+});
+var _getLongCenterAlignedTexts = _interopRequireDefault(require("./getLongCenterAlignedTexts"));
+var _wordComplexity = _interopRequireDefault(require("./wordComplexity"));
+var _keyphraseDistribution = _interopRequireDefault(require("./keyphraseDistribution"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+//# sourceMappingURL=index.js.map

package/build/cjs/languageProcessing/researches/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["_getLongCenterAlignedTexts","_interopRequireDefault","require","_wordComplexity","_keyphraseDistribution","e","__esModule","default"],"sources":["../../../../src/languageProcessing/researches/index.js"],"sourcesContent":["import getLongCenterAlignedTexts from './getLongCenterAlignedTexts';\nimport wordComplexity from './wordComplexity';\nimport keyphraseDistribution from './keyphraseDistribution';\n\nexport {getLongCenterAlignedTexts, keyphraseDistribution, wordComplexity};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAAA,IAAAA,0BAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,eAAA,GAAAF,sBAAA,CAAAC,OAAA;AACA,IAAAE,sBAAA,GAAAH,sBAAA,CAAAC,OAAA;AAA4D,SAAAD,uBAAAI,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA","ignoreList":[]}

package/build/cjs/languageProcessing/researches/keyphraseDistribution.js

@@ -0,0 +1,233 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.maximizeSentenceScores = exports.keyphraseDistributionResearcher = exports.getDistraction = exports.default = exports.computeScoresPerSentenceShortTopic = exports.computeScoresPerSentenceLongTopic = void 0;
+var _lodash = require("lodash");
+var _findKeywordFormsInString = require("../helpers/match/findKeywordFormsInString");
+var _markWordsInSentences = require("../helpers/word/markWordsInSentences");
+var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences"));
+var _parseSynonyms = _interopRequireDefault(require("../helpers/sanitize/parseSynonyms"));
+var _mergeListItems = require("../helpers/sanitize/mergeListItems");
+var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
+var _filterShortcodesFromTree = require("../helpers/sanitize/filterShortcodesFromTree");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Checks whether at least half of the content words from the topic are found within the sentence.
+ * Assigns a score to every sentence following the following schema:
+ * 9 if at least half of the content words from the topic are in the sentence,
+ * 3 otherwise.
+ *
+ * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.
+ * @param {Array}  sentences An array of all sentences in the text.
+ * @param {string} locale    The locale of the paper to analyse.
+ * @param {function}    matchWordCustomHelper 	The language-specific helper function to match word in text.
+ *
+ * @returns {Array} The scores per sentence.
+ */
+const computeScoresPerSentenceLongTopic = function (topic, sentences, locale, matchWordCustomHelper) {
+  const sentenceScores = Array(sentences.length);
+  for (let i = 0; i < sentences.length; i++) {
+    const foundInCurrentSentence = (0, _findKeywordFormsInString.findWordFormsInString)(topic, sentences[i], locale, matchWordCustomHelper);
+    if (foundInCurrentSentence.percentWordMatches >= 50) {
+      sentenceScores[i] = 9;
+    } else {
+      sentenceScores[i] = 3;
+    }
+  }
+  return sentenceScores;
+};
+
+/**
+ * Checks whether all content words from the topic are found within one sentence.
+ * Assigns a score to every sentence following the following schema:
+ * 9 if all content words from the topic are in the sentence,
+ * 3 if not all content words from the topic were found in the sentence.
+ *
+ * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.
+ * @param {Array}  sentences An array of all sentences in the text.
+ * @param {string} locale    The locale of the paper to analyse.
+ * @param {function}    matchWordCustomHelper 	The language-specific helper function to match word in text.
+ *
+ * @returns {Array} The scores per sentence.
+ */
+exports.computeScoresPerSentenceLongTopic = computeScoresPerSentenceLongTopic;
+const computeScoresPerSentenceShortTopic = function (topic, sentences, locale, matchWordCustomHelper) {
+  const sentenceScores = Array(sentences.length);
+  for (let i = 0; i < sentences.length; i++) {
+    const currentSentence = sentences[i];
+    const foundInCurrentSentence = (0, _findKeywordFormsInString.findWordFormsInString)(topic, currentSentence, locale, matchWordCustomHelper);
+    if (foundInCurrentSentence.percentWordMatches === 100) {
+      sentenceScores[i] = 9;
+    } else {
+      sentenceScores[i] = 3;
+    }
+  }
+  return sentenceScores;
+};
+
+/**
+ * Maximizes scores: Give every sentence a maximal score that it got from analysis of all topics
+ *
+ * @param {Array} sentenceScores The scores for every sentence, as assessed per keyphrase and every synonym.
+ *
+ * @returns {Array} Maximal scores of topic relevance per sentence.
+ */
+exports.computeScoresPerSentenceShortTopic = computeScoresPerSentenceShortTopic;
+const maximizeSentenceScores = function (sentenceScores) {
+  const sentenceScoresTransposed = sentenceScores[0].map(function (col, i) {
+    return sentenceScores.map(function (row) {
+      return row[i];
+    });
+  });
+  return sentenceScoresTransposed.map(function (scoresForOneSentence) {
+    return (0, _lodash.max)(scoresForOneSentence);
+  });
+};
+
+/**
+ * Computes the maximally long piece of text that does not include the topic.
+ *
+ * @param {Array} sentenceScores The array of scores per sentence.
+ *
+ * @returns {number} The maximum number of sentences that do not include the topic.
+ */
+exports.maximizeSentenceScores = maximizeSentenceScores;
+const getDistraction = function (sentenceScores) {
+  const numberOfSentences = sentenceScores.length;
+  const allTopicSentencesIndices = [];
+  for (let i = 0; i < numberOfSentences; i++) {
+    if (sentenceScores[i] > 3) {
+      allTopicSentencesIndices.push(i);
+    }
+  }
+  const numberOfTopicSentences = allTopicSentencesIndices.length;
+  if (numberOfTopicSentences === 0) {
+    return numberOfSentences;
+  }
+
+  /**
+   * Add fake topic sentences at the very beginning and at the very end
+   * to account for cases when the text starts or ends with a train of distraction.
+   */
+  allTopicSentencesIndices.unshift(-1);
+  allTopicSentencesIndices.push(numberOfSentences);
+  const distances = [];
+  for (let i = 1; i < numberOfTopicSentences + 2; i++) {
+    distances.push(allTopicSentencesIndices[i] - allTopicSentencesIndices[i - 1] - 1);
+  }
+  return (0, _lodash.max)(distances);
+};
+
+/**
+ * Computes the per-sentence scores depending on the length of the topic phrase and maximizes them over all topic phrases.
+ *
+ * @param {Array}       sentences              The sentences to get scores for.
+ * @param {Array}       topicFormsInOneArray   The topic phrases forms to search for in the sentences.
+ * @param {string}      locale                 The locale to work in.
+ * @param {Array}       functionWords           The function words list.
+ * @param {function}    matchWordCustomHelper 	The language-specific helper function to match word in text.
+ * @param {int}         topicLengthCriteria     The topic length criteria. The default value is 4, where a topic is considered short
+ *                                              if it's less than 4 word long, and otherwise long.
+ * @param {Array}       originalTopic           The array of the original form of the topic with function words filtered out.
+ * @param {function}    wordsCharacterCount     The helper to calculate the characters length of all the words in the array.
+ *
+ * @returns {Object} An array with maximized score per sentence and an array with all sentences that do not contain the topic.
+ */
+exports.getDistraction = getDistraction;
+const getSentenceScores = function (sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper, topicLengthCriteria = 4, originalTopic, wordsCharacterCount) {
+  // Compute per-sentence scores of topic-relatedness.
+  const topicNumber = topicFormsInOneArray.length;
+  const sentenceScores = Array(topicNumber);
+
+  // For languages with function words apply either full match or partial match depending on topic length
+  if (functionWords.length > 0) {
+    for (let i = 0; i < topicNumber; i++) {
+      const topic = topicFormsInOneArray[i];
+      /*
+       * If the helper to calculate the characters length of all the words in the array is available,
+       * we use this helper to calculate the characters length of the original topic form.
+       * We then use the result and compare it with the topicLengthCriteria.
+       */
+      const topicLength = wordsCharacterCount ? wordsCharacterCount(originalTopic[i]) : topic.length;
+      if (topicLength < topicLengthCriteria) {
+        sentenceScores[i] = computeScoresPerSentenceShortTopic(topic, sentences, locale, matchWordCustomHelper);
+      } else {
+        sentenceScores[i] = computeScoresPerSentenceLongTopic(topic, sentences, locale, matchWordCustomHelper);
+      }
+    }
+  } else {
+    // For languages without function words apply the full match always
+    for (let i = 0; i < topicNumber; i++) {
+      const topic = topicFormsInOneArray[i];
+      sentenceScores[i] = computeScoresPerSentenceShortTopic(topic, sentences, locale, matchWordCustomHelper);
+    }
+  }
+
+  // Maximize scores: Give every sentence a maximal score that it got from analysis of all topics
+  const maximizedSentenceScores = maximizeSentenceScores(sentenceScores);
+
+  // Zip an array combining each sentence with the associated maximized score.
+  const sentencesWithMaximizedScores = (0, _lodash.zipWith)(sentences, maximizedSentenceScores, (sentence, score) => {
+    return {
+      sentence,
+      score
+    };
+  });
+
+  // Filter sentences that contain topic words for future highlights.
+  const sentencesWithTopic = sentencesWithMaximizedScores.filter(sentenceObject => sentenceObject.score > 3);
+  return {
+    maximizedSentenceScores: maximizedSentenceScores,
+    sentencesWithTopic: sentencesWithTopic.map(sentenceObject => sentenceObject.sentence)
+  };
+};
+
+/**
+ * Determines which portions of the text did not receive a lot of content words from keyphrase and synonyms.
+ *
+ * @param {Paper}       paper               The paper to check the keyphrase distribution for.
+ * @param {Researcher}  researcher          The researcher to use for analysis.
+ *
+ * @returns {Object} The scores of topic relevance per portion of text and an array of all word forms to highlight.
+ */
+const keyphraseDistributionResearcher = function (paper, researcher) {
+  const functionWords = researcher.getConfig("functionWords");
+  const matchWordCustomHelper = researcher.getHelper("matchWordCustomHelper");
+  const getContentWordsHelper = researcher.getHelper("getContentWords");
+  const wordsCharacterCount = researcher.getResearch("wordsCharacterCount");
+  const memoizedTokenizer = researcher.getHelper("memoizedTokenizer");
+
+  // Custom topic length criteria for languages that don't use the default value to determine whether a topic is long or short.
+  const topicLengthCriteria = researcher.getConfig("topicLength").lengthCriteria;
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _filterShortcodesFromTree.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+  text = (0, _mergeListItems.mergeListItems)(text);
+  const sentences = (0, _getSentences.default)(text, memoizedTokenizer);
+  const topicForms = researcher.getResearch("morphology");
+  const originalTopic = [];
+  if (getContentWordsHelper) {
+    originalTopic.push(getContentWordsHelper(paper.getKeyword()));
+    (0, _parseSynonyms.default)(paper.getSynonyms()).forEach(synonym => originalTopic.push(getContentWordsHelper(synonym)));
+  }
+  const locale = paper.getLocale();
+  const topicFormsInOneArray = [topicForms.keyphraseForms];
+  topicForms.synonymsForms.forEach(function (synonym) {
+    topicFormsInOneArray.push(synonym);
+  });
+  const allTopicWords = (0, _lodash.uniq)((0, _lodash.flattenDeep)(topicFormsInOneArray)).sort((a, b) => b.length - a.length);
+
+  // Get per-sentence scores and sentences that have topic.
+  const sentenceScores = getSentenceScores(sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper, topicLengthCriteria, originalTopic, wordsCharacterCount);
+  const maximizedSentenceScores = sentenceScores.maximizedSentenceScores;
+  const maxLengthDistraction = getDistraction(maximizedSentenceScores);
+  return {
+    sentencesToHighlight: (0, _markWordsInSentences.markWordsInSentences)(allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper),
+    keyphraseDistributionScore: maxLengthDistraction / sentences.length * 100
+  };
+};
+exports.keyphraseDistributionResearcher = keyphraseDistributionResearcher;
+var _default = exports.default = keyphraseDistributionResearcher;
+//# sourceMappingURL=keyphraseDistribution.js.map

package/build/cjs/languageProcessing/researches/keyphraseDistribution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyphraseDistribution.js","names":["_lodash","require","_findKeywordFormsInString","_markWordsInSentences","_getSentences","_interopRequireDefault","_parseSynonyms","_mergeListItems","_htmlParser","_filterShortcodesFromTree","e","__esModule","default","computeScoresPerSentenceLongTopic","topic","sentences","locale","matchWordCustomHelper","sentenceScores","Array","length","i","foundInCurrentSentence","findWordFormsInString","percentWordMatches","exports","computeScoresPerSentenceShortTopic","currentSentence","maximizeSentenceScores","sentenceScoresTransposed","map","col","row","scoresForOneSentence","max","getDistraction","numberOfSentences","allTopicSentencesIndices","push","numberOfTopicSentences","unshift","distances","getSentenceScores","topicFormsInOneArray","functionWords","topicLengthCriteria","originalTopic","wordsCharacterCount","topicNumber","topicLength","maximizedSentenceScores","sentencesWithMaximizedScores","zipWith","sentence","score","sentencesWithTopic","filter","sentenceObject","keyphraseDistributionResearcher","paper","researcher","getConfig","getHelper","getContentWordsHelper","getResearch","memoizedTokenizer","lengthCriteria","text","getText","removeHtmlBlocks","filterShortcodesFromHTML","_attributes","shortcodes","mergeListItems","getSentences","topicForms","getKeyword","parseSynonyms","getSynonyms","forEach","synonym","getLocale","keyphraseForms","synonymsForms","allTopicWords","unique","flattenDeep","sort","a","b","maxLengthDistraction","sentencesToHighlight","markWordsInSentences","keyphraseDistributionScore","_default"],"sources":["../../../../src/languageProcessing/researches/keyphraseDistribution.js"],"sourcesContent":["import { flattenDeep, max, uniq as unique, zipWith } from \"lodash\";\nimport { findWordFormsInString } from \"../helpers/match/findKeywordFormsInString\";\nimport { markWordsInSentences } from \"../helpers/word/markWordsInSentences\";\nimport getSentences from \"../helpers/sentence/getSentences\";\nimport parseSynonyms from \"../helpers/sanitize/parseSynonyms\";\nimport { mergeListItems } from \"../helpers/sanitize/mergeListItems\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers/sanitize/filterShortcodesFromTree\";\n\n/**\n * Checks whether at least half of the content words from the topic are found within the sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if at least half of the content words from the topic are in the sentence,\n * 3 otherwise.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceLongTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, sentences[ i ], locale, matchWordCustomHelper );\n\n\t\tif ( foundInCurrentSentence.percentWordMatches >= 50 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\n\treturn sentenceScores;\n};\n\n\n/**\n * Checks whether all content words from the topic are found within one sentence.\n * Assigns a score to every sentence following the following schema:\n * 9 if all content words from the topic are in the sentence,\n * 3 if not all content words from the topic were found in the sentence.\n *\n * @param {Array}  topic     The word forms of all content words in a keyphrase or a synonym.\n * @param {Array}  sentences An array of all sentences in the text.\n * @param {string} locale    The locale of the paper to analyse.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n *\n * @returns {Array} The scores per sentence.\n */\nconst computeScoresPerSentenceShortTopic = function( topic, sentences, locale, matchWordCustomHelper ) {\n\tconst sentenceScores = Array( sentences.length );\n\n\tfor ( let i = 0; i < sentences.length; i++ ) {\n\t\tconst currentSentence = sentences[ i ];\n\t\tconst foundInCurrentSentence = findWordFormsInString( topic, currentSentence, locale, matchWordCustomHelper );\n\t\tif ( foundInCurrentSentence.percentWordMatches === 100 ) {\n\t\t\tsentenceScores[ i ] = 9;\n\t\t} else {\n\t\t\tsentenceScores[ i ] = 3;\n\t\t}\n\t}\n\treturn sentenceScores;\n};\n\n/**\n * Maximizes scores: Give every sentence a maximal score that it got from analysis of all topics\n *\n * @param {Array} sentenceScores The scores for every sentence, as assessed per keyphrase and every synonym.\n *\n * @returns {Array} Maximal scores of topic relevance per sentence.\n */\nconst maximizeSentenceScores = function( sentenceScores ) {\n\tconst sentenceScoresTransposed = sentenceScores[ 0 ].map( function( col, i ) {\n\t\treturn sentenceScores.map( function( row ) {\n\t\t\treturn row[ i ];\n\t\t} );\n\t} );\n\n\treturn sentenceScoresTransposed.map( function( scoresForOneSentence ) {\n\t\treturn max( scoresForOneSentence );\n\t} );\n};\n\n\n/**\n * Computes the maximally long piece of text that does not include the topic.\n *\n * @param {Array} sentenceScores The array of scores per sentence.\n *\n * @returns {number} The maximum number of sentences that do not include the topic.\n */\nconst getDistraction = function( sentenceScores ) {\n\tconst numberOfSentences = sentenceScores.length;\n\tconst allTopicSentencesIndices = [];\n\n\tfor ( let i = 0; i < numberOfSentences; i++ ) {\n\t\tif ( sentenceScores[ i ] > 3 ) {\n\t\t\tallTopicSentencesIndices.push( i );\n\t\t}\n\t}\n\n\tconst numberOfTopicSentences = allTopicSentencesIndices.length;\n\n\tif ( numberOfTopicSentences === 0 ) {\n\t\treturn numberOfSentences;\n\t}\n\n\t/**\n\t * Add fake topic sentences at the very beginning and at the very end\n\t * to account for cases when the text starts or ends with a train of distraction.\n\t */\n\tallTopicSentencesIndices.unshift( -1 );\n\tallTopicSentencesIndices.push( numberOfSentences );\n\n\tconst distances = [];\n\n\tfor ( let i = 1; i < numberOfTopicSentences + 2; i++ ) {\n\t\tdistances.push( allTopicSentencesIndices[ i ] - allTopicSentencesIndices[ i - 1 ] - 1 );\n\t}\n\n\treturn max( distances );\n};\n\n/**\n * Computes the per-sentence scores depending on the length of the topic phrase and maximizes them over all topic phrases.\n *\n * @param {Array}       sentences              The sentences to get scores for.\n * @param {Array}       topicFormsInOneArray   The topic phrases forms to search for in the sentences.\n * @param {string}      locale                 The locale to work in.\n * @param {Array}       functionWords           The function words list.\n * @param {function}    matchWordCustomHelper \tThe language-specific helper function to match word in text.\n * @param {int}         topicLengthCriteria     The topic length criteria. The default value is 4, where a topic is considered short\n *                                              if it's less than 4 word long, and otherwise long.\n * @param {Array}       originalTopic           The array of the original form of the topic with function words filtered out.\n * @param {function}    wordsCharacterCount     The helper to calculate the characters length of all the words in the array.\n *\n * @returns {Object} An array with maximized score per sentence and an array with all sentences that do not contain the topic.\n */\nconst getSentenceScores = function( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\ttopicLengthCriteria = 4, originalTopic, wordsCharacterCount ) {\n\t// Compute per-sentence scores of topic-relatedness.\n\tconst topicNumber = topicFormsInOneArray.length;\n\n\tconst sentenceScores = Array( topicNumber );\n\n\t// For languages with function words apply either full match or partial match depending on topic length\n\tif ( functionWords.length > 0 ) {\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\t/*\n\t\t\t * If the helper to calculate the characters length of all the words in the array is available,\n\t\t\t * we use this helper to calculate the characters length of the original topic form.\n\t\t\t * We then use the result and compare it with the topicLengthCriteria.\n\t\t\t */\n\t\t\tconst topicLength = wordsCharacterCount ? wordsCharacterCount( originalTopic[ i ] ) : topic.length;\n\t\t\tif ( topicLength < topicLengthCriteria ) {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t} else {\n\t\t\t\tsentenceScores[ i ] = computeScoresPerSentenceLongTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// For languages without function words apply the full match always\n\t\tfor ( let i = 0; i < topicNumber; i++ ) {\n\t\t\tconst topic = topicFormsInOneArray[ i ];\n\t\t\tsentenceScores[ i ] = computeScoresPerSentenceShortTopic( topic, sentences, locale, matchWordCustomHelper );\n\t\t}\n\t}\n\n\t// Maximize scores: Give every sentence a maximal score that it got from analysis of all topics\n\tconst maximizedSentenceScores = maximizeSentenceScores( sentenceScores );\n\n\t// Zip an array combining each sentence with the associated maximized score.\n\tconst sentencesWithMaximizedScores =  zipWith( sentences, maximizedSentenceScores, ( sentence, score ) => {\n\t\treturn { sentence, score };\n\t} );\n\n\t// Filter sentences that contain topic words for future highlights.\n\tconst sentencesWithTopic = sentencesWithMaximizedScores.filter( sentenceObject => sentenceObject.score > 3 );\n\n\treturn {\n\t\tmaximizedSentenceScores: maximizedSentenceScores,\n\t\tsentencesWithTopic: sentencesWithTopic.map( sentenceObject => sentenceObject.sentence ),\n\t};\n};\n\n\n/**\n * Determines which portions of the text did not receive a lot of content words from keyphrase and synonyms.\n *\n * @param {Paper}       paper               The paper to check the keyphrase distribution for.\n * @param {Researcher}  researcher          The researcher to use for analysis.\n *\n * @returns {Object} The scores of topic relevance per portion of text and an array of all word forms to highlight.\n */\nconst keyphraseDistributionResearcher = function( paper, researcher ) {\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\tconst matchWordCustomHelper = researcher.getHelper( \"matchWordCustomHelper\" );\n\tconst getContentWordsHelper = researcher.getHelper( \"getContentWords\" );\n\tconst wordsCharacterCount = researcher.getResearch( \"wordsCharacterCount\" );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\n\t// Custom topic length criteria for languages that don't use the default value to determine whether a topic is long or short.\n\tconst topicLengthCriteria = researcher.getConfig( \"topicLength\" ).lengthCriteria;\n\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\ttext = mergeListItems( text );\n\tconst sentences = getSentences( text, memoizedTokenizer );\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\n\tconst originalTopic = [];\n\tif ( getContentWordsHelper ) {\n\t\toriginalTopic.push( getContentWordsHelper( paper.getKeyword() ) );\n\t\tparseSynonyms( paper.getSynonyms() ).forEach( synonym => originalTopic.push( getContentWordsHelper( synonym ) ) );\n\t}\n\tconst locale = paper.getLocale();\n\tconst topicFormsInOneArray = [ topicForms.keyphraseForms ];\n\ttopicForms.synonymsForms.forEach( function( synonym ) {\n\t\ttopicFormsInOneArray.push( synonym );\n\t} );\n\n\tconst allTopicWords = unique( flattenDeep( topicFormsInOneArray ) ).sort( ( a, b ) => b.length - a.length );\n\n\t// Get per-sentence scores and sentences that have topic.\n\tconst sentenceScores = getSentenceScores( sentences, topicFormsInOneArray, locale, functionWords, matchWordCustomHelper,\n\t\ttopicLengthCriteria, originalTopic, wordsCharacterCount );\n\tconst maximizedSentenceScores = sentenceScores.maximizedSentenceScores;\n\tconst maxLengthDistraction = getDistraction( maximizedSentenceScores );\n\n\treturn {\n\t\tsentencesToHighlight: markWordsInSentences( allTopicWords, sentenceScores.sentencesWithTopic, locale, matchWordCustomHelper ),\n\t\tkeyphraseDistributionScore: maxLengthDistraction / sentences.length * 100,\n\t};\n};\n\nexport {\n\tcomputeScoresPerSentenceShortTopic,\n\tcomputeScoresPerSentenceLongTopic,\n\tmaximizeSentenceScores,\n\tkeyphraseDistributionResearcher,\n\tgetDistraction,\n};\n\nexport default keyphraseDistributionResearcher;\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,yBAAA,GAAAD,OAAA;AACA,IAAAE,qBAAA,GAAAF,OAAA;AACA,IAAAG,aAAA,GAAAC,sBAAA,CAAAJ,OAAA;AACA,IAAAK,cAAA,GAAAD,sBAAA,CAAAJ,OAAA;AACA,IAAAM,eAAA,GAAAN,OAAA;AACA,IAAAO,WAAA,GAAAH,sBAAA,CAAAJ,OAAA;AACA,IAAAQ,yBAAA,GAAAR,OAAA;AAAwF,SAAAI,uBAAAK,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAExF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,iCAAiC,GAAG,SAAAA,CAAUC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACrG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMC,sBAAsB,GAAG,IAAAC,+CAAqB,EAAET,KAAK,EAAEC,SAAS,CAAEM,CAAC,CAAE,EAAEL,MAAM,EAAEC,qBAAsB,CAAC;IAE5G,IAAKK,sBAAsB,CAACE,kBAAkB,IAAI,EAAE,EAAG;MACtDN,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EAEA,OAAOH,cAAc;AACtB,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAZAO,OAAA,CAAAZ,iCAAA,GAAAA,iCAAA;AAaA,MAAMa,kCAAkC,GAAG,SAAAA,CAAUZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAqB,EAAG;EACtG,MAAMC,cAAc,GAAGC,KAAK,CAAEJ,SAAS,CAACK,MAAO,CAAC;EAEhD,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,SAAS,CAACK,MAAM,EAAEC,CAAC,EAAE,EAAG;IAC5C,MAAMM,eAAe,GAAGZ,SAAS,CAAEM,CAAC,CAAE;IACtC,MAAMC,sBAAsB,GAAG,IAAAC,+CAAqB,EAAET,KAAK,EAAEa,eAAe,EAAEX,MAAM,EAAEC,qBAAsB,CAAC;IAC7G,IAAKK,sBAAsB,CAACE,kBAAkB,KAAK,GAAG,EAAG;MACxDN,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB,CAAC,MAAM;MACNH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC;IACxB;EACD;EACA,OAAOH,cAAc;AACtB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AANAO,OAAA,CAAAC,kCAAA,GAAAA,kCAAA;AAOA,MAAME,sBAAsB,GAAG,SAAAA,CAAUV,cAAc,EAAG;EACzD,MAAMW,wBAAwB,GAAGX,cAAc,CAAE,CAAC,CAAE,CAACY,GAAG,CAAE,UAAUC,GAAG,EAAEV,CAAC,EAAG;IAC5E,OAAOH,cAAc,CAACY,GAAG,CAAE,UAAUE,GAAG,EAAG;MAC1C,OAAOA,GAAG,CAAEX,CAAC,CAAE;IAChB,CAAE,CAAC;EACJ,CAAE,CAAC;EAEH,OAAOQ,wBAAwB,CAACC,GAAG,CAAE,UAAUG,oBAAoB,EAAG;IACrE,OAAO,IAAAC,WAAG,EAAED,oBAAqB,CAAC;EACnC,CAAE,CAAC;AACJ,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AANAR,OAAA,CAAAG,sBAAA,GAAAA,sBAAA;AAOA,MAAMO,cAAc,GAAG,SAAAA,CAAUjB,cAAc,EAAG;EACjD,MAAMkB,iBAAiB,GAAGlB,cAAc,CAACE,MAAM;EAC/C,MAAMiB,wBAAwB,GAAG,EAAE;EAEnC,KAAM,IAAIhB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGe,iBAAiB,EAAEf,CAAC,EAAE,EAAG;IAC7C,IAAKH,cAAc,CAAEG,CAAC,CAAE,GAAG,CAAC,EAAG;MAC9BgB,wBAAwB,CAACC,IAAI,CAAEjB,CAAE,CAAC;IACnC;EACD;EAEA,MAAMkB,sBAAsB,GAAGF,wBAAwB,CAACjB,MAAM;EAE9D,IAAKmB,sBAAsB,KAAK,CAAC,EAAG;IACnC,OAAOH,iBAAiB;EACzB;;EAEA;AACD;AACA;AACA;EACCC,wBAAwB,CAACG,OAAO,CAAE,CAAC,CAAE,CAAC;EACtCH,wBAAwB,CAACC,IAAI,CAAEF,iBAAkB,CAAC;EAElD,MAAMK,SAAS,GAAG,EAAE;EAEpB,KAAM,IAAIpB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGkB,sBAAsB,GAAG,CAAC,EAAElB,CAAC,EAAE,EAAG;IACtDoB,SAAS,CAACH,IAAI,CAAED,wBAAwB,CAAEhB,CAAC,CAAE,GAAGgB,wBAAwB,CAAEhB,CAAC,GAAG,CAAC,CAAE,GAAG,CAAE,CAAC;EACxF;EAEA,OAAO,IAAAa,WAAG,EAAEO,SAAU,CAAC;AACxB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAdAhB,OAAA,CAAAU,cAAA,GAAAA,cAAA;AAeA,MAAMO,iBAAiB,GAAG,SAAAA,CAAU3B,SAAS,EAAE4B,oBAAoB,EAAE3B,MAAM,EAAE4B,aAAa,EAAE3B,qBAAqB,EAChH4B,mBAAmB,GAAG,CAAC,EAAEC,aAAa,EAAEC,mBAAmB,EAAG;EAC9D;EACA,MAAMC,WAAW,GAAGL,oBAAoB,CAACvB,MAAM;EAE/C,MAAMF,cAAc,GAAGC,KAAK,CAAE6B,WAAY,CAAC;;EAE3C;EACA,IAAKJ,aAAa,CAACxB,MAAM,GAAG,CAAC,EAAG;IAC/B,KAAM,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2B,WAAW,EAAE3B,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG6B,oBAAoB,CAAEtB,CAAC,CAAE;MACvC;AACH;AACA;AACA;AACA;MACG,MAAM4B,WAAW,GAAGF,mBAAmB,GAAGA,mBAAmB,CAAED,aAAa,CAAEzB,CAAC,CAAG,CAAC,GAAGP,KAAK,CAACM,MAAM;MAClG,IAAK6B,WAAW,GAAGJ,mBAAmB,EAAG;QACxC3B,cAAc,CAAEG,CAAC,CAAE,GAAGK,kCAAkC,CAAEZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC5G,CAAC,MAAM;QACNC,cAAc,CAAEG,CAAC,CAAE,GAAGR,iCAAiC,CAAEC,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;MAC3G;IACD;EACD,CAAC,MAAM;IACN;IACA,KAAM,IAAII,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2B,WAAW,EAAE3B,CAAC,EAAE,EAAG;MACvC,MAAMP,KAAK,GAAG6B,oBAAoB,CAAEtB,CAAC,CAAE;MACvCH,cAAc,CAAEG,CAAC,CAAE,GAAGK,kCAAkC,CAAEZ,KAAK,EAAEC,SAAS,EAAEC,MAAM,EAAEC,qBAAsB,CAAC;IAC5G;EACD;;EAEA;EACA,MAAMiC,uBAAuB,GAAGtB,sBAAsB,CAAEV,cAAe,CAAC;;EAExE;EACA,MAAMiC,4BAA4B,GAAI,IAAAC,eAAO,EAAErC,SAAS,EAAEmC,uBAAuB,EAAE,CAAEG,QAAQ,EAAEC,KAAK,KAAM;IACzG,OAAO;MAAED,QAAQ;MAAEC;IAAM,CAAC;EAC3B,CAAE,CAAC;;EAEH;EACA,MAAMC,kBAAkB,GAAGJ,4BAA4B,CAACK,MAAM,CAAEC,cAAc,IAAIA,cAAc,CAACH,KAAK,GAAG,CAAE,CAAC;EAE5G,OAAO;IACNJ,uBAAuB,EAAEA,uBAAuB;IAChDK,kBAAkB,EAAEA,kBAAkB,CAACzB,GAAG,CAAE2B,cAAc,IAAIA,cAAc,CAACJ,QAAS;EACvF,CAAC;AACF,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMK,+BAA+B,GAAG,SAAAA,CAAUC,KAAK,EAAEC,UAAU,EAAG;EACrE,MAAMhB,aAAa,GAAGgB,UAAU,CAACC,SAAS,CAAE,eAAgB,CAAC;EAC7D,MAAM5C,qBAAqB,GAAG2C,UAAU,CAACE,SAAS,CAAE,uBAAwB,CAAC;EAC7E,MAAMC,qBAAqB,GAAGH,UAAU,CAACE,SAAS,CAAE,iBAAkB,CAAC;EACvE,MAAMf,mBAAmB,GAAGa,UAAU,CAACI,WAAW,CAAE,qBAAsB,CAAC;EAC3E,MAAMC,iBAAiB,GAAGL,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;;EAErE;EACA,MAAMjB,mBAAmB,GAAGe,UAAU,CAACC,SAAS,CAAE,aAAc,CAAC,CAACK,cAAc;EAEhF,IAAIC,IAAI,GAAGR,KAAK,CAACS,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,kDAAwB,EAAEH,IAAI,EAAER,KAAK,CAACY,WAAW,IAAIZ,KAAK,CAACY,WAAW,CAACC,UAAW,CAAC;EAC1FL,IAAI,GAAG,IAAAM,8BAAc,EAAEN,IAAK,CAAC;EAC7B,MAAMpD,SAAS,GAAG,IAAA2D,qBAAY,EAAEP,IAAI,EAAEF,iBAAkB,CAAC;EACzD,MAAMU,UAAU,GAAGf,UAAU,CAACI,WAAW,CAAE,YAAa,CAAC;EAEzD,MAAMlB,aAAa,GAAG,EAAE;EACxB,IAAKiB,qBAAqB,EAAG;IAC5BjB,aAAa,CAACR,IAAI,CAAEyB,qBAAqB,CAAEJ,KAAK,CAACiB,UAAU,CAAC,CAAE,CAAE,CAAC;IACjE,IAAAC,sBAAa,EAAElB,KAAK,CAACmB,WAAW,CAAC,CAAE,CAAC,CAACC,OAAO,CAAEC,OAAO,IAAIlC,aAAa,CAACR,IAAI,CAAEyB,qBAAqB,CAAEiB,OAAQ,CAAE,CAAE,CAAC;EAClH;EACA,MAAMhE,MAAM,GAAG2C,KAAK,CAACsB,SAAS,CAAC,CAAC;EAChC,MAAMtC,oBAAoB,GAAG,CAAEgC,UAAU,CAACO,cAAc,CAAE;EAC1DP,UAAU,CAACQ,aAAa,CAACJ,OAAO,CAAE,UAAUC,OAAO,EAAG;IACrDrC,oBAAoB,CAACL,IAAI,CAAE0C,OAAQ,CAAC;EACrC,CAAE,CAAC;EAEH,MAAMI,aAAa,GAAG,IAAAC,YAAM,EAAE,IAAAC,mBAAW,EAAE3C,oBAAqB,CAAE,CAAC,CAAC4C,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMA,CAAC,CAACrE,MAAM,GAAGoE,CAAC,CAACpE,MAAO,CAAC;;EAE3G;EACA,MAAMF,cAAc,GAAGwB,iBAAiB,CAAE3B,SAAS,EAAE4B,oBAAoB,EAAE3B,MAAM,EAAE4B,aAAa,EAAE3B,qBAAqB,EACtH4B,mBAAmB,EAAEC,aAAa,EAAEC,mBAAoB,CAAC;EAC1D,MAAMG,uBAAuB,GAAGhC,cAAc,CAACgC,uBAAuB;EACtE,MAAMwC,oBAAoB,GAAGvD,cAAc,CAAEe,uBAAwB,CAAC;EAEtE,OAAO;IACNyC,oBAAoB,EAAE,IAAAC,0CAAoB,EAAER,aAAa,EAAElE,cAAc,CAACqC,kBAAkB,EAAEvC,MAAM,EAAEC,qBAAsB,CAAC;IAC7H4E,0BAA0B,EAAEH,oBAAoB,GAAG3E,SAAS,CAACK,MAAM,GAAG;EACvE,CAAC;AACF,CAAC;AAACK,OAAA,CAAAiC,+BAAA,GAAAA,+BAAA;AAAA,IAAAoC,QAAA,GAAArE,OAAA,CAAAb,OAAA,GAUa8C,+BAA+B","ignoreList":[]}

package/build/cjs/languageProcessing/researches/keyphraseLength.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+/**
+ * Determines the length in words of the keyphrase.
+ *
+ * @param {Paper} paper 			The paper to research
+ * @param {Researcher} researcher 	The researcher to use for analysis
+ *
+ * @returns {Object} The length of the keyphrase and the function words list.
+ */
+function _default(paper, researcher) {
+  const topicForms = researcher.getResearch("morphology");
+  const functionWords = researcher.getConfig("functionWords");
+  return {
+    keyphraseLength: topicForms.keyphraseForms.length,
+    functionWords: functionWords
+  };
+}
+//# sourceMappingURL=keyphraseLength.js.map

package/build/cjs/languageProcessing/researches/keyphraseLength.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyphraseLength.js","names":["_default","paper","researcher","topicForms","getResearch","functionWords","getConfig","keyphraseLength","keyphraseForms","length"],"sources":["../../../../src/languageProcessing/researches/keyphraseLength.js"],"sourcesContent":["/**\n * Determines the length in words of the keyphrase.\n *\n * @param {Paper} paper \t\t\tThe paper to research\n * @param {Researcher} researcher \tThe researcher to use for analysis\n *\n * @returns {Object} The length of the keyphrase and the function words list.\n */\nexport default function( paper, researcher ) {\n\tconst topicForms = researcher.getResearch( \"morphology\" );\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\n\treturn {\n\t\tkeyphraseLength: topicForms.keyphraseForms.length,\n\t\tfunctionWords: functionWords,\n\t};\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAA,SAAUC,KAAK,EAAEC,UAAU,EAAG;EAC5C,MAAMC,UAAU,GAAGD,UAAU,CAACE,WAAW,CAAE,YAAa,CAAC;EACzD,MAAMC,aAAa,GAAGH,UAAU,CAACI,SAAS,CAAE,eAAgB,CAAC;EAE7D,OAAO;IACNC,eAAe,EAAEJ,UAAU,CAACK,cAAc,CAACC,MAAM;IACjDJ,aAAa,EAAEA;EAChB,CAAC;AACF","ignoreList":[]}

package/build/cjs/languageProcessing/researches/keywordCount.js

@@ -0,0 +1,162 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.countKeyphraseInText = countKeyphraseInText;
+exports.default = getKeyphraseCount;
+exports.keywordCount = keywordCount;
+var _lodash = require("lodash");
+var _getSentencesFromTree = _interopRequireDefault(require("../helpers/sentence/getSentencesFromTree"));
+var _quotes = require("../helpers/sanitize/quotes");
+var _getMarkingsInSentence = _interopRequireDefault(require("../helpers/highlighting/getMarkingsInSentence"));
+var _matchWordFormsWithSentence = _interopRequireDefault(require("../helpers/match/matchWordFormsWithSentence"));
+var _isDoubleQuoted = _interopRequireDefault(require("../helpers/match/isDoubleQuoted"));
+var _markWordsInSentences = require("../helpers/word/markWordsInSentences");
+var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences"));
+var _helpers = require("../helpers");
+var _isChineseText = _interopRequireDefault(require("../helpers/language/isChineseText"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Counts the occurrences of the keyphrase in the text and creates the Mark objects for the matches.
+ *
+ * @param {(Sentence|string)[]}	sentences			The sentences to check.
+ * @param {Array}		keyphraseForms				The keyphrase forms.
+ * @param {string}		locale						The locale used in the analysis.
+ * @param {function}	matchWordCustomHelper		A custom helper to match words with a text.
+ * @param {boolean}		isExactMatchRequested		Whether the exact matching is requested.
+ * @param {function}	customSplitIntoTokensHelper	A custom helper to split sentences into tokens.
+ *
+ * @returns {{markings: Mark[], count: number}} The number of keyphrase occurrences in the text and the Mark objects of the matches.
+ */
+function countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper) {
+  const result = {
+    count: 0,
+    markings: []
+  };
+  sentences.forEach(sentence => {
+    const matchesInSentence = keyphraseForms.map(wordForms => (0, _matchWordFormsWithSentence.default)(sentence, wordForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper));
+    // A sentence has at least one full-match of the keyphrase if each word occurs at least once.
+    const isEachWordFound = matchesInSentence.every(wordForms => wordForms.count > 0);
+    if (isEachWordFound) {
+      /*
+       * Retrieve all the occurrences' count of each word of the keyphrase and save it in an array.
+       * matches: [ [ { matches: ["red"], count: 1 } ], [ { matches: ["pandas"], count: 2 } ] ]
+       * counts: [ 1, 2 ]
+       */
+      const counts = matchesInSentence.map(match => match.count);
+      /*
+       * The number of the full-match count is the lowest count of the occurrences.
+       * counts: [ 1, 2 ]
+       * totalMatchCount: 1
+       *
+       * From the example above, the full-match is 1, because one of the "pandas" occurrences is not accompanied by "red"
+       * to be counted as a full-match.
+       */
+      const totalMatchCount = Math.min(...counts);
+      const foundWords = (0, _lodash.flattenDeep)(matchesInSentence.map(match => match.matches));
+      let markings = [];
+      if (matchWordCustomHelper) {
+        // Currently, this check is only applicable for Japanese.
+        markings = (0, _markWordsInSentences.markWordsInASentence)(sentence, foundWords, matchWordCustomHelper);
+      } else {
+        markings = (0, _getMarkingsInSentence.default)(sentence, foundWords);
+      }
+      result.count += totalMatchCount;
+      result.markings.push(markings);
+    }
+  });
+  return result;
+}
+
+/**
+ * Calculates the keyphrase count, takes morphology into account.
+ *
+ * @param {Paper}       paper       The paper containing keyphrase and text.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {{count: number, markings: Mark[], keyphraseLength: number}} An object containing the keyphrase count, markings and the kephrase length.
+ */
+function getKeyphraseCount(paper, researcher) {
+  const result = {
+    count: 0,
+    markings: [],
+    keyphraseLength: 0
+  };
+  const topicForms = researcher.getResearch('morphology');
+  let keyphraseForms = topicForms.keyphraseForms;
+  const keyphraseLength = keyphraseForms.length;
+
+  /*
+   * Normalize single quotes so that word form with different type of single quotes can still be matched.
+   * For example, "key‛word" should match "key'word".
+   */
+  keyphraseForms = keyphraseForms.map(word => word.map(form => (0, _quotes.normalizeSingle)(form)));
+  if (keyphraseLength === 0) {
+    return result;
+  }
+  let matchWordCustomHelper = researcher.getHelper('matchWordCustomHelper');
+  const customSentenceTokenizer = researcher.getHelper('memoizedTokenizer');
+  const customSplitIntoTokensHelper = researcher.getHelper('splitIntoTokensCustom');
+  const locale = paper.getLocale();
+  const text = matchWordCustomHelper ? (0, _helpers.filterShortcodesFromHTML)(paper.getText(), paper._attributes && paper._attributes.shortcodes) : paper.getText();
+
+  // Auto-detect Chinese and use Chinese helper if not already available
+  if (!matchWordCustomHelper && (0, _isChineseText.default)(paper.getKeyword() + ' ' + text)) {
+    // Import Chinese helper for Chinese text
+    try {
+      // Use simple Chinese word matching for Chinese text
+      matchWordCustomHelper = function (sentence, word) {
+        const text = typeof sentence === 'string' ? sentence : sentence.text || sentence;
+        const matches = [];
+        if (!text || !word) {
+          return matches;
+        }
+        const lowerText = text.toLowerCase();
+        const lowerWord = word.toLowerCase();
+        let startIndex = 0;
+        let index;
+        while ((index = lowerText.indexOf(lowerWord, startIndex)) !== -1) {
+          matches.push(word);
+          startIndex = index + lowerWord.length;
+        }
+        return matches;
+      };
+    } catch (error) {
+      // If there's any issue, continue with default behavior
+      console.warn('Error setting up Chinese matching, using default:', error.message);
+    }
+  }
+
+  // When the custom helper is available, we're using the sentences retrieved from the text for the analysis.
+  const sentences = matchWordCustomHelper ? (0, _getSentences.default)(text, customSentenceTokenizer) : (0, _getSentencesFromTree.default)(paper);
+  // Exact matching is requested when the keyphrase is enclosed in double quotes.
+  const isExactMatchRequested = (0, _isDoubleQuoted.default)(paper.getKeyword());
+
+  /*
+   * Count the amount of keyphrase occurrences in the sentences.
+   * An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.
+   * (e.g. "The apple potato is an apple and a potato." has two occurrences of the keyphrase "apple potato").
+   */
+  const keyphraseFound = countKeyphraseInText(sentences, keyphraseForms, locale, matchWordCustomHelper, isExactMatchRequested, customSplitIntoTokensHelper);
+  result.count = keyphraseFound.count;
+  result.markings = (0, _lodash.flatten)(keyphraseFound.markings);
+  result.keyphraseLength = keyphraseLength;
+  return result;
+}
+
+/**
+ * Calculates the keyphrase count, takes morphology into account.
+ *
+ * @deprecated Use getKeyphraseCount instead.
+ *
+ * @param {Paper}       paper       The paper containing keyphrase and text.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {Object} An array of all the matches, markings and the keyphrase count.
+ */
+function keywordCount(paper, researcher) {
+  console.warn('This function is deprecated, use getKeyphraseCount instead.');
+  return getKeyphraseCount(paper, researcher);
+}
+//# sourceMappingURL=keywordCount.js.map

package/build/cjs/languageProcessing/researches/keywordCount.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keywordCount.js","names":["_lodash","require","_getSentencesFromTree","_interopRequireDefault","_quotes","_getMarkingsInSentence","_matchWordFormsWithSentence","_isDoubleQuoted","_markWordsInSentences","_getSentences","_helpers","_isChineseText","e","__esModule","default","countKeyphraseInText","sentences","keyphraseForms","locale","matchWordCustomHelper","isExactMatchRequested","customSplitIntoTokensHelper","result","count","markings","forEach","sentence","matchesInSentence","map","wordForms","matchWordFormsWithSentence","isEachWordFound","every","counts","match","totalMatchCount","Math","min","foundWords","flattenDeep","matches","markWordsInASentence","getMarkingsInSentence","push","getKeyphraseCount","paper","researcher","keyphraseLength","topicForms","getResearch","length","word","form","normalizeSingle","getHelper","customSentenceTokenizer","getLocale","text","filterShortcodesFromHTML","getText","_attributes","shortcodes","isChineseText","getKeyword","lowerText","toLowerCase","lowerWord","startIndex","index","indexOf","error","console","warn","message","getSentences","getSentencesFromTree","isDoubleQuoted","keyphraseFound","flatten","keywordCount"],"sources":["../../../../src/languageProcessing/researches/keywordCount.js"],"sourcesContent":["import {flatten, flattenDeep} from 'lodash';\nimport getSentencesFromTree from '../helpers/sentence/getSentencesFromTree';\nimport {normalizeSingle} from '../helpers/sanitize/quotes';\nimport getMarkingsInSentence from '../helpers/highlighting/getMarkingsInSentence';\nimport matchWordFormsWithSentence from '../helpers/match/matchWordFormsWithSentence';\nimport isDoubleQuoted from '../helpers/match/isDoubleQuoted';\nimport {markWordsInASentence} from '../helpers/word/markWordsInSentences';\nimport getSentences from '../helpers/sentence/getSentences';\nimport {filterShortcodesFromHTML} from '../helpers';\nimport isChineseText from '../helpers/language/isChineseText';\n\n/**\n * Counts the occurrences of the keyphrase in the text and creates the Mark objects for the matches.\n *\n * @param {(Sentence|string)[]}\tsentences\t\t\tThe sentences to check.\n * @param {Array}\t\tkeyphraseForms\t\t\t\tThe keyphrase forms.\n * @param {string}\t\tlocale\t\t\t\t\t\tThe locale used in the analysis.\n * @param {function}\tmatchWordCustomHelper\t\tA custom helper to match words with a text.\n * @param {boolean}\t\tisExactMatchRequested\t\tWhether the exact matching is requested.\n * @param {function}\tcustomSplitIntoTokensHelper\tA custom helper to split sentences into tokens.\n *\n * @returns {{markings: Mark[], count: number}} The number of keyphrase occurrences in the text and the Mark objects of the matches.\n */\nexport function countKeyphraseInText(\n  sentences,\n  keyphraseForms,\n  locale,\n  matchWordCustomHelper,\n  isExactMatchRequested,\n  customSplitIntoTokensHelper\n) {\n  const result = {count: 0, markings: []};\n\n  sentences.forEach(sentence => {\n    const matchesInSentence = keyphraseForms.map(wordForms =>\n      matchWordFormsWithSentence(\n        sentence,\n        wordForms,\n        locale,\n        matchWordCustomHelper,\n        isExactMatchRequested,\n        customSplitIntoTokensHelper\n      )\n    );\n    // A sentence has at least one full-match of the keyphrase if each word occurs at least once.\n    const isEachWordFound = matchesInSentence.every(wordForms => wordForms.count > 0);\n\n    if (isEachWordFound) {\n      /*\n       * Retrieve all the occurrences' count of each word of the keyphrase and save it in an array.\n       * matches: [ [ { matches: [\"red\"], count: 1 } ], [ { matches: [\"pandas\"], count: 2 } ] ]\n       * counts: [ 1, 2 ]\n       */\n      const counts = matchesInSentence.map(match => match.count);\n      /*\n       * The number of the full-match count is the lowest count of the occurrences.\n       * counts: [ 1, 2 ]\n       * totalMatchCount: 1\n       *\n       * From the example above, the full-match is 1, because one of the \"pandas\" occurrences is not accompanied by \"red\"\n       * to be counted as a full-match.\n       */\n      const totalMatchCount = Math.min(...counts);\n      const foundWords = flattenDeep(matchesInSentence.map(match => match.matches));\n\n      let markings = [];\n\n      if (matchWordCustomHelper) {\n        // Currently, this check is only applicable for Japanese.\n        markings = markWordsInASentence(sentence, foundWords, matchWordCustomHelper);\n      } else {\n        markings = getMarkingsInSentence(sentence, foundWords);\n      }\n\n      result.count += totalMatchCount;\n      result.markings.push(markings);\n    }\n  });\n\n  return result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {{count: number, markings: Mark[], keyphraseLength: number}} An object containing the keyphrase count, markings and the kephrase length.\n */\nexport default function getKeyphraseCount(paper, researcher) {\n  const result = {count: 0, markings: [], keyphraseLength: 0};\n  const topicForms = researcher.getResearch('morphology');\n  let keyphraseForms = topicForms.keyphraseForms;\n  const keyphraseLength = keyphraseForms.length;\n\n  /*\n   * Normalize single quotes so that word form with different type of single quotes can still be matched.\n   * For example, \"key‛word\" should match \"key'word\".\n   */\n  keyphraseForms = keyphraseForms.map(word => word.map(form => normalizeSingle(form)));\n\n  if (keyphraseLength === 0) {\n    return result;\n  }\n\n  let matchWordCustomHelper = researcher.getHelper('matchWordCustomHelper');\n  const customSentenceTokenizer = researcher.getHelper('memoizedTokenizer');\n  const customSplitIntoTokensHelper = researcher.getHelper('splitIntoTokensCustom');\n  const locale = paper.getLocale();\n  const text = matchWordCustomHelper\n    ? filterShortcodesFromHTML(paper.getText(), paper._attributes && paper._attributes.shortcodes)\n    : paper.getText();\n\n  // Auto-detect Chinese and use Chinese helper if not already available\n  if (!matchWordCustomHelper && isChineseText(paper.getKeyword() + ' ' + text)) {\n    // Import Chinese helper for Chinese text\n    try {\n      // Use simple Chinese word matching for Chinese text\n      matchWordCustomHelper = function(sentence, word) {\n        const text = typeof sentence === 'string' ? sentence : sentence.text || sentence;\n        const matches = [];\n\n        if (!text || !word) {\n          return matches;\n        }\n\n        const lowerText = text.toLowerCase();\n        const lowerWord = word.toLowerCase();\n\n        let startIndex = 0;\n        let index;\n\n        while ((index = lowerText.indexOf(lowerWord, startIndex)) !== -1) {\n          matches.push(word);\n          startIndex = index + lowerWord.length;\n        }\n\n        return matches;\n      };\n    } catch (error) {\n      // If there's any issue, continue with default behavior\n      console.warn('Error setting up Chinese matching, using default:', error.message);\n    }\n  }\n\n  // When the custom helper is available, we're using the sentences retrieved from the text for the analysis.\n  const sentences = matchWordCustomHelper\n    ? getSentences(text, customSentenceTokenizer)\n    : getSentencesFromTree(paper);\n  // Exact matching is requested when the keyphrase is enclosed in double quotes.\n  const isExactMatchRequested = isDoubleQuoted(paper.getKeyword());\n\n  /*\n   * Count the amount of keyphrase occurrences in the sentences.\n   * An occurrence is counted when all words of the keyphrase are contained within the sentence. Each sentence can contain multiple keyphrases.\n   * (e.g. \"The apple potato is an apple and a potato.\" has two occurrences of the keyphrase \"apple potato\").\n   */\n  const keyphraseFound = countKeyphraseInText(\n    sentences,\n    keyphraseForms,\n    locale,\n    matchWordCustomHelper,\n    isExactMatchRequested,\n    customSplitIntoTokensHelper\n  );\n\n  result.count = keyphraseFound.count;\n  result.markings = flatten(keyphraseFound.markings);\n  result.keyphraseLength = keyphraseLength;\n\n  return result;\n}\n\n/**\n * Calculates the keyphrase count, takes morphology into account.\n *\n * @deprecated Use getKeyphraseCount instead.\n *\n * @param {Paper}       paper       The paper containing keyphrase and text.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {Object} An array of all the matches, markings and the keyphrase count.\n */\nexport function keywordCount(paper, researcher) {\n  console.warn('This function is deprecated, use getKeyphraseCount instead.');\n  return getKeyphraseCount(paper, researcher);\n}\n"],"mappings":";;;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AACA,IAAAI,sBAAA,GAAAF,sBAAA,CAAAF,OAAA;AACA,IAAAK,2BAAA,GAAAH,sBAAA,CAAAF,OAAA;AACA,IAAAM,eAAA,GAAAJ,sBAAA,CAAAF,OAAA;AACA,IAAAO,qBAAA,GAAAP,OAAA;AACA,IAAAQ,aAAA,GAAAN,sBAAA,CAAAF,OAAA;AACA,IAAAS,QAAA,GAAAT,OAAA;AACA,IAAAU,cAAA,GAAAR,sBAAA,CAAAF,OAAA;AAA8D,SAAAE,uBAAAS,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE9D;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASG,oBAAoBA,CAClCC,SAAS,EACTC,cAAc,EACdC,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BAA2B,EAC3B;EACA,MAAMC,MAAM,GAAG;IAACC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE;EAAE,CAAC;EAEvCR,SAAS,CAACS,OAAO,CAACC,QAAQ,IAAI;IAC5B,MAAMC,iBAAiB,GAAGV,cAAc,CAACW,GAAG,CAACC,SAAS,IACpD,IAAAC,mCAA0B,EACxBJ,QAAQ,EACRG,SAAS,EACTX,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BACF,CACF,CAAC;IACD;IACA,MAAMU,eAAe,GAAGJ,iBAAiB,CAACK,KAAK,CAACH,SAAS,IAAIA,SAAS,CAACN,KAAK,GAAG,CAAC,CAAC;IAEjF,IAAIQ,eAAe,EAAE;MACnB;AACN;AACA;AACA;AACA;MACM,MAAME,MAAM,GAAGN,iBAAiB,CAACC,GAAG,CAACM,KAAK,IAAIA,KAAK,CAACX,KAAK,CAAC;MAC1D;AACN;AACA;AACA;AACA;AACA;AACA;AACA;MACM,MAAMY,eAAe,GAAGC,IAAI,CAACC,GAAG,CAAC,GAAGJ,MAAM,CAAC;MAC3C,MAAMK,UAAU,GAAG,IAAAC,mBAAW,EAACZ,iBAAiB,CAACC,GAAG,CAACM,KAAK,IAAIA,KAAK,CAACM,OAAO,CAAC,CAAC;MAE7E,IAAIhB,QAAQ,GAAG,EAAE;MAEjB,IAAIL,qBAAqB,EAAE;QACzB;QACAK,QAAQ,GAAG,IAAAiB,0CAAoB,EAACf,QAAQ,EAAEY,UAAU,EAAEnB,qBAAqB,CAAC;MAC9E,CAAC,MAAM;QACLK,QAAQ,GAAG,IAAAkB,8BAAqB,EAAChB,QAAQ,EAAEY,UAAU,CAAC;MACxD;MAEAhB,MAAM,CAACC,KAAK,IAAIY,eAAe;MAC/Bb,MAAM,CAACE,QAAQ,CAACmB,IAAI,CAACnB,QAAQ,CAAC;IAChC;EACF,CAAC,CAAC;EAEF,OAAOF,MAAM;AACf;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASsB,iBAAiBA,CAACC,KAAK,EAAEC,UAAU,EAAE;EAC3D,MAAMxB,MAAM,GAAG;IAACC,KAAK,EAAE,CAAC;IAAEC,QAAQ,EAAE,EAAE;IAAEuB,eAAe,EAAE;EAAC,CAAC;EAC3D,MAAMC,UAAU,GAAGF,UAAU,CAACG,WAAW,CAAC,YAAY,CAAC;EACvD,IAAIhC,cAAc,GAAG+B,UAAU,CAAC/B,cAAc;EAC9C,MAAM8B,eAAe,GAAG9B,cAAc,CAACiC,MAAM;;EAE7C;AACF;AACA;AACA;EACEjC,cAAc,GAAGA,cAAc,CAACW,GAAG,CAACuB,IAAI,IAAIA,IAAI,CAACvB,GAAG,CAACwB,IAAI,IAAI,IAAAC,uBAAe,EAACD,IAAI,CAAC,CAAC,CAAC;EAEpF,IAAIL,eAAe,KAAK,CAAC,EAAE;IACzB,OAAOzB,MAAM;EACf;EAEA,IAAIH,qBAAqB,GAAG2B,UAAU,CAACQ,SAAS,CAAC,uBAAuB,CAAC;EACzE,MAAMC,uBAAuB,GAAGT,UAAU,CAACQ,SAAS,CAAC,mBAAmB,CAAC;EACzE,MAAMjC,2BAA2B,GAAGyB,UAAU,CAACQ,SAAS,CAAC,uBAAuB,CAAC;EACjF,MAAMpC,MAAM,GAAG2B,KAAK,CAACW,SAAS,CAAC,CAAC;EAChC,MAAMC,IAAI,GAAGtC,qBAAqB,GAC9B,IAAAuC,iCAAwB,EAACb,KAAK,CAACc,OAAO,CAAC,CAAC,EAAEd,KAAK,CAACe,WAAW,IAAIf,KAAK,CAACe,WAAW,CAACC,UAAU,CAAC,GAC5FhB,KAAK,CAACc,OAAO,CAAC,CAAC;;EAEnB;EACA,IAAI,CAACxC,qBAAqB,IAAI,IAAA2C,sBAAa,EAACjB,KAAK,CAACkB,UAAU,CAAC,CAAC,GAAG,GAAG,GAAGN,IAAI,CAAC,EAAE;IAC5E;IACA,IAAI;MACF;MACAtC,qBAAqB,GAAG,SAAAA,CAASO,QAAQ,EAAEyB,IAAI,EAAE;QAC/C,MAAMM,IAAI,GAAG,OAAO/B,QAAQ,KAAK,QAAQ,GAAGA,QAAQ,GAAGA,QAAQ,CAAC+B,IAAI,IAAI/B,QAAQ;QAChF,MAAMc,OAAO,GAAG,EAAE;QAElB,IAAI,CAACiB,IAAI,IAAI,CAACN,IAAI,EAAE;UAClB,OAAOX,OAAO;QAChB;QAEA,MAAMwB,SAAS,GAAGP,IAAI,CAACQ,WAAW,CAAC,CAAC;QACpC,MAAMC,SAAS,GAAGf,IAAI,CAACc,WAAW,CAAC,CAAC;QAEpC,IAAIE,UAAU,GAAG,CAAC;QAClB,IAAIC,KAAK;QAET,OAAO,CAACA,KAAK,GAAGJ,SAAS,CAACK,OAAO,CAACH,SAAS,EAAEC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE;UAChE3B,OAAO,CAACG,IAAI,CAACQ,IAAI,CAAC;UAClBgB,UAAU,GAAGC,KAAK,GAAGF,SAAS,CAAChB,MAAM;QACvC;QAEA,OAAOV,OAAO;MAChB,CAAC;IACH,CAAC,CAAC,OAAO8B,KAAK,EAAE;MACd;MACAC,OAAO,CAACC,IAAI,CAAC,mDAAmD,EAAEF,KAAK,CAACG,OAAO,CAAC;IAClF;EACF;;EAEA;EACA,MAAMzD,SAAS,GAAGG,qBAAqB,GACnC,IAAAuD,qBAAY,EAACjB,IAAI,EAAEF,uBAAuB,CAAC,GAC3C,IAAAoB,6BAAoB,EAAC9B,KAAK,CAAC;EAC/B;EACA,MAAMzB,qBAAqB,GAAG,IAAAwD,uBAAc,EAAC/B,KAAK,CAACkB,UAAU,CAAC,CAAC,CAAC;;EAEhE;AACF;AACA;AACA;AACA;EACE,MAAMc,cAAc,GAAG9D,oBAAoB,CACzCC,SAAS,EACTC,cAAc,EACdC,MAAM,EACNC,qBAAqB,EACrBC,qBAAqB,EACrBC,2BACF,CAAC;EAEDC,MAAM,CAACC,KAAK,GAAGsD,cAAc,CAACtD,KAAK;EACnCD,MAAM,CAACE,QAAQ,GAAG,IAAAsD,eAAO,EAACD,cAAc,CAACrD,QAAQ,CAAC;EAClDF,MAAM,CAACyB,eAAe,GAAGA,eAAe;EAExC,OAAOzB,MAAM;AACf;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASyD,YAAYA,CAAClC,KAAK,EAAEC,UAAU,EAAE;EAC9CyB,OAAO,CAACC,IAAI,CAAC,6DAA6D,CAAC;EAC3E,OAAO5B,iBAAiB,CAACC,KAAK,EAAEC,UAAU,CAAC;AAC7C","ignoreList":[]}