axyseo 2.0.0-alpha.1

package/build/languageProcessing/researches/getParagraphLength.js

@@ -0,0 +1,49 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _imageInText = require("../helpers/image/imageInText");
+var _sanitizeLineBreakTag = _interopRequireDefault(require("../helpers/sanitize/sanitizeLineBreakTag"));
+var _countWords = _interopRequireDefault(require("../helpers/word/countWords.js"));
+var _matchParagraphs = _interopRequireDefault(require("../helpers/html/matchParagraphs.js"));
+var _lodash = require("lodash");
+var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
+var _helpers = require("../helpers");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Gets all paragraphs and their word counts or character counts from the text.
+ *
+ * @param {Paper} 		paper 		The paper object to get the text from.
+ * @param {Researcher} 	researcher 	The researcher to use for analysis.
+ *
+ * @returns {Array} The array containing an object with the paragraph word or character count and paragraph text.
+ */
+function _default(paper, researcher) {
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+
+  // Remove images from text before retrieving the paragraphs.
+  // This step is done here so that applying highlight in captions is possible for ParagraphTooLongAssessment.
+  text = text.replace(_imageInText.imageRegex, "");
+
+  // Replace line break tags containing attribute(s) with paragraph tag.
+  text = (0, _sanitizeLineBreakTag.default)(text);
+  const paragraphs = (0, _matchParagraphs.default)(text);
+  const paragraphsLength = [];
+
+  // An optional custom helper to count length to use instead of countWords.
+  const customCountLength = researcher.getHelper("customCountLength");
+  paragraphs.map(function (paragraph) {
+    paragraphsLength.push({
+      countLength: customCountLength ? customCountLength(paragraph) : (0, _countWords.default)(paragraph),
+      text: paragraph
+    });
+  });
+  return (0, _lodash.filter)(paragraphsLength, function (paragraphLength) {
+    return paragraphLength.countLength > 0;
+  });
+}
+//# sourceMappingURL=getParagraphLength.js.map

package/build/languageProcessing/researches/getParagraphLength.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getParagraphLength.js","names":["_imageInText","require","_sanitizeLineBreakTag","_interopRequireDefault","_countWords","_matchParagraphs","_lodash","_htmlParser","_helpers","e","__esModule","default","_default","paper","researcher","text","getText","removeHtmlBlocks","filterShortcodesFromHTML","_attributes","shortcodes","replace","imageRegex","sanitizeLineBreakTag","paragraphs","matchParagraphs","paragraphsLength","customCountLength","getHelper","map","paragraph","push","countLength","countWords","filter","paragraphLength"],"sources":["../../../src/languageProcessing/researches/getParagraphLength.js"],"sourcesContent":["import { imageRegex } from \"../helpers/image/imageInText\";\nimport sanitizeLineBreakTag from \"../helpers/sanitize/sanitizeLineBreakTag\";\nimport countWords from \"../helpers/word/countWords.js\";\nimport matchParagraphs from \"../helpers/html/matchParagraphs.js\";\nimport { filter } from \"lodash\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers\";\n\n/**\n * Gets all paragraphs and their word counts or character counts from the text.\n *\n * @param {Paper} \t\tpaper \t\tThe paper object to get the text from.\n * @param {Researcher} \tresearcher \tThe researcher to use for analysis.\n *\n * @returns {Array} The array containing an object with the paragraph word or character count and paragraph text.\n */\nexport default function( paper, researcher ) {\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\n\t// Remove images from text before retrieving the paragraphs.\n\t// This step is done here so that applying highlight in captions is possible for ParagraphTooLongAssessment.\n\ttext = text.replace( imageRegex, \"\" );\n\n\t// Replace line break tags containing attribute(s) with paragraph tag.\n\ttext = sanitizeLineBreakTag( text );\n\tconst paragraphs = matchParagraphs( text );\n\tconst paragraphsLength = [];\n\n\t// An optional custom helper to count length to use instead of countWords.\n\tconst customCountLength = researcher.getHelper( \"customCountLength\" );\n\n\tparagraphs.map( function( paragraph ) {\n\t\tparagraphsLength.push( {\n\t\t\tcountLength: customCountLength ? customCountLength( paragraph ) : countWords( paragraph ),\n\t\t\ttext: paragraph,\n\t\t} );\n\t} );\n\n\treturn filter( paragraphsLength, function( paragraphLength ) {\n\t\treturn ( paragraphLength.countLength > 0 );\n\t} );\n}\n"],"mappings":";;;;;;AAAA,IAAAA,YAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,WAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,gBAAA,GAAAF,sBAAA,CAAAF,OAAA;AACA,IAAAK,OAAA,GAAAL,OAAA;AACA,IAAAM,WAAA,GAAAJ,sBAAA,CAAAF,OAAA;AACA,IAAAO,QAAA,GAAAP,OAAA;AAAsD,SAAAE,uBAAAM,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEtD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,KAAK,EAAEC,UAAU,EAAG;EAC5C,IAAIC,IAAI,GAAGF,KAAK,CAACG,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,iCAAwB,EAAEH,IAAI,EAAEF,KAAK,CAACM,WAAW,IAAIN,KAAK,CAACM,WAAW,CAACC,UAAW,CAAC;;EAE1F;EACA;EACAL,IAAI,GAAGA,IAAI,CAACM,OAAO,CAAEC,uBAAU,EAAE,EAAG,CAAC;;EAErC;EACAP,IAAI,GAAG,IAAAQ,6BAAoB,EAAER,IAAK,CAAC;EACnC,MAAMS,UAAU,GAAG,IAAAC,wBAAe,EAAEV,IAAK,CAAC;EAC1C,MAAMW,gBAAgB,GAAG,EAAE;;EAE3B;EACA,MAAMC,iBAAiB,GAAGb,UAAU,CAACc,SAAS,CAAE,mBAAoB,CAAC;EAErEJ,UAAU,CAACK,GAAG,CAAE,UAAUC,SAAS,EAAG;IACrCJ,gBAAgB,CAACK,IAAI,CAAE;MACtBC,WAAW,EAAEL,iBAAiB,GAAGA,iBAAiB,CAAEG,SAAU,CAAC,GAAG,IAAAG,mBAAU,EAAEH,SAAU,CAAC;MACzFf,IAAI,EAAEe;IACP,CAAE,CAAC;EACJ,CAAE,CAAC;EAEH,OAAO,IAAAI,cAAM,EAAER,gBAAgB,EAAE,UAAUS,eAAe,EAAG;IAC5D,OAASA,eAAe,CAACH,WAAW,GAAG,CAAC;EACzC,CAAE,CAAC;AACJ","ignoreList":[]}

package/build/languageProcessing/researches/getParagraphs.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+/**
+ * Returns all paragraphs in a given Paper.
+ * Remove paragraphs that do not contain sentences or only consist of links.
+ *
+ * @param {Paper} paper The current paper.
+ * @returns {Paragraph[]} All paragraphs in the paper.
+ */
+function _default(paper) {
+  let paragraphs = paper.getTree().findAll(node => node.name === "p");
+
+  // Remove empty paragraphs without sentences and paragraphs only consisting of links.
+  paragraphs = (0, _lodash.reject)(paragraphs, paragraph => paragraph.sentences.length === 0);
+  paragraphs = (0, _lodash.reject)(paragraphs, paragraph => paragraph.childNodes.every(node => node.name === "a"));
+  return paragraphs;
+}
+//# sourceMappingURL=getParagraphs.js.map

package/build/languageProcessing/researches/getParagraphs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getParagraphs.js","names":["_lodash","require","_default","paper","paragraphs","getTree","findAll","node","name","reject","paragraph","sentences","length","childNodes","every"],"sources":["../../../src/languageProcessing/researches/getParagraphs.js"],"sourcesContent":["import { reject } from \"lodash\";\n\n/**\n * Returns all paragraphs in a given Paper.\n * Remove paragraphs that do not contain sentences or only consist of links.\n *\n * @param {Paper} paper The current paper.\n * @returns {Paragraph[]} All paragraphs in the paper.\n */\nexport default function( paper ) {\n\tlet paragraphs = paper.getTree().findAll( node => node.name === \"p\" );\n\n\t// Remove empty paragraphs without sentences and paragraphs only consisting of links.\n\tparagraphs = reject( paragraphs, paragraph => paragraph.sentences.length === 0 );\n\tparagraphs = reject( paragraphs, paragraph => paragraph.childNodes.every( node => node.name === \"a\" ) );\n\n\treturn paragraphs;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAAUC,KAAK,EAAG;EAChC,IAAIC,UAAU,GAAGD,KAAK,CAACE,OAAO,CAAC,CAAC,CAACC,OAAO,CAAEC,IAAI,IAAIA,IAAI,CAACC,IAAI,KAAK,GAAI,CAAC;;EAErE;EACAJ,UAAU,GAAG,IAAAK,cAAM,EAAEL,UAAU,EAAEM,SAAS,IAAIA,SAAS,CAACC,SAAS,CAACC,MAAM,KAAK,CAAE,CAAC;EAChFR,UAAU,GAAG,IAAAK,cAAM,EAAEL,UAAU,EAAEM,SAAS,IAAIA,SAAS,CAACG,UAAU,CAACC,KAAK,CAAEP,IAAI,IAAIA,IAAI,CAACC,IAAI,KAAK,GAAI,CAAE,CAAC;EAEvG,OAAOJ,UAAU;AAClB","ignoreList":[]}

package/build/languageProcessing/researches/getPassiveVoiceResult.js

@@ -0,0 +1,127 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = getPassiveVoice;
+exports.getPeriphrasticPassives = exports.getMorphologicalPassives = void 0;
+var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences.js"));
+var _stripHTMLTags = require("../helpers/sanitize/stripHTMLTags.js");
+var _Sentence = _interopRequireDefault(require("../../languageProcessing/values/Sentence.js"));
+var _lodash = require("lodash");
+var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
+var _helpers = require("../helpers");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Looks for morphological passive voice.
+ * Supported morphological languages: "ru", "sv", "id", "ar", "he", "tr", "fa".
+ * Farsi is implemented as morphological because the periphrastic passives are used as compound verbs (among other compound passives).
+ * @param {Paper}      paper      The paper object.
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Object} The found passive sentences.
+ */
+const getMorphologicalPassives = function (paper, researcher) {
+  const isPassiveSentence = researcher.getHelper("isPassiveSentence");
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+  const memoizedTokenizer = researcher.getHelper("memoizedTokenizer");
+  const sentences = (0, _getSentences.default)(text, memoizedTokenizer).map(function (sentence) {
+    return new _Sentence.default(sentence);
+  });
+  const totalNumberSentences = sentences.length;
+  const passiveSentences = [];
+  (0, _lodash.forEach)(sentences, function (sentence) {
+    const strippedSentence = (0, _stripHTMLTags.stripFullTags)(sentence.getSentenceText()).toLocaleLowerCase();
+    sentence.setPassive(isPassiveSentence(strippedSentence));
+    if (sentence.isPassive() === true) {
+      passiveSentences.push(sentence.getSentenceText());
+    }
+  });
+  return {
+    total: totalNumberSentences,
+    passives: passiveSentences
+  };
+};
+
+/**
+ * Looks for periphrastic passive voice.
+ * Supported periphrastic languages: "en", "de", "nl", "fr", "es", "it", "pt", "pl", "sk".
+ *
+ * @param {Paper}      paper      The paper object.
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Object} The found passive sentences.
+ */
+exports.getMorphologicalPassives = getMorphologicalPassives;
+const getPeriphrasticPassives = function (paper, researcher) {
+  const getClauses = researcher.getHelper("getClauses");
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+  const memoizedTokenizer = researcher.getHelper("memoizedTokenizer");
+  const sentences = (0, _getSentences.default)(text, memoizedTokenizer).map(function (sentence) {
+    return new _Sentence.default(sentence);
+  });
+  const totalNumberSentences = sentences.length;
+  const passiveSentences = [];
+  (0, _lodash.forEach)(sentences, function (sentence) {
+    const strippedSentence = (0, _stripHTMLTags.stripFullTags)(sentence.getSentenceText()).toLocaleLowerCase();
+
+    // The functionality based on sentencePart objects should be rewritten using array indices of stopwords and auxiliaries.
+
+    // Divide a sentence into clauses and return an array of clause objects that have been checked for passiveness.
+    const clauses = getClauses(strippedSentence);
+    sentence.setClauses(clauses);
+
+    // Check sentence passiveness based on its clause passiveness.
+    if (sentence.isPassive()) {
+      passiveSentences.push(sentence.getSentenceText());
+    }
+  });
+  return {
+    total: totalNumberSentences,
+    passives: passiveSentences
+  };
+};
+
+/**
+ * Looks for both morphological and periphrastic passive voice
+ * Supported languages with both morphological and periphrastic passives: "hu", "nb".
+ * Due to technical difficulties "nb" is only implemented as periphrastic at the moment. Languages that have not been implemented yet: "da".
+ *
+ * @param {Paper}      paper      The paper object.
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Object} The found passive sentences.
+ */
+exports.getPeriphrasticPassives = getPeriphrasticPassives;
+const getMorphologicalAndPeriphrasticPassive = function (paper, researcher) {
+  const morphologicalPassives = getMorphologicalPassives(paper, researcher);
+  const periphrasticPassives = getPeriphrasticPassives(paper, researcher).passives;
+  return {
+    total: morphologicalPassives.total,
+    passives: periphrasticPassives.concat(morphologicalPassives.passives)
+  };
+};
+
+/**
+ * Looks for passive voice.
+ *
+ * @param {Paper}      paper      The paper object.
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Object} The found passive sentences.
+ */
+function getPassiveVoice(paper, researcher) {
+  const passiveType = researcher.getConfig("passiveConstructionType");
+  if (passiveType === "periphrastic") {
+    return getPeriphrasticPassives(paper, researcher);
+  }
+  if (passiveType === "morphological") {
+    return getMorphologicalPassives(paper, researcher);
+  }
+  return getMorphologicalAndPeriphrasticPassive(paper, researcher);
+}
+//# sourceMappingURL=getPassiveVoiceResult.js.map

package/build/languageProcessing/researches/getPassiveVoiceResult.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getPassiveVoiceResult.js","names":["_getSentences","_interopRequireDefault","require","_stripHTMLTags","_Sentence","_lodash","_htmlParser","_helpers","e","__esModule","default","getMorphologicalPassives","paper","researcher","isPassiveSentence","getHelper","text","getText","removeHtmlBlocks","filterShortcodesFromHTML","_attributes","shortcodes","memoizedTokenizer","sentences","getSentences","map","sentence","Sentence","totalNumberSentences","length","passiveSentences","forEach","strippedSentence","stripHTMLTags","getSentenceText","toLocaleLowerCase","setPassive","isPassive","push","total","passives","exports","getPeriphrasticPassives","getClauses","clauses","setClauses","getMorphologicalAndPeriphrasticPassive","morphologicalPassives","periphrasticPassives","concat","getPassiveVoice","passiveType","getConfig"],"sources":["../../../src/languageProcessing/researches/getPassiveVoiceResult.js"],"sourcesContent":["import getSentences from \"../helpers/sentence/getSentences.js\";\nimport { stripFullTags as stripHTMLTags } from \"../helpers/sanitize/stripHTMLTags.js\";\nimport Sentence from \"../../languageProcessing/values/Sentence.js\";\n\nimport { forEach } from \"lodash\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers\";\n\n/**\n * Looks for morphological passive voice.\n * Supported morphological languages: \"ru\", \"sv\", \"id\", \"ar\", \"he\", \"tr\", \"fa\".\n * Farsi is implemented as morphological because the periphrastic passives are used as compound verbs (among other compound passives).\n * @param {Paper}      paper      The paper object.\n * @param {Researcher} researcher The researcher.\n *\n * @returns {Object} The found passive sentences.\n */\nexport const getMorphologicalPassives = function( paper, researcher ) {\n\tconst isPassiveSentence = researcher.getHelper( \"isPassiveSentence\" );\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\tconst sentences = getSentences( text, memoizedTokenizer )\n\t\t.map( function( sentence ) {\n\t\t\treturn new Sentence( sentence );\n\t\t} );\n\tconst totalNumberSentences = sentences.length;\n\tconst passiveSentences = [];\n\n\tforEach( sentences, function( sentence ) {\n\t\tconst strippedSentence = stripHTMLTags( sentence.getSentenceText() ).toLocaleLowerCase();\n\n\t\tsentence.setPassive( isPassiveSentence( strippedSentence ) );\n\n\t\tif ( sentence.isPassive() === true ) {\n\t\t\tpassiveSentences.push( sentence.getSentenceText() );\n\t\t}\n\t} );\n\n\treturn {\n\t\ttotal: totalNumberSentences,\n\t\tpassives: passiveSentences,\n\t};\n};\n\n/**\n * Looks for periphrastic passive voice.\n * Supported periphrastic languages: \"en\", \"de\", \"nl\", \"fr\", \"es\", \"it\", \"pt\", \"pl\", \"sk\".\n *\n * @param {Paper}      paper      The paper object.\n * @param {Researcher} researcher The researcher.\n *\n * @returns {Object} The found passive sentences.\n */\nexport const getPeriphrasticPassives = function( paper, researcher ) {\n\tconst getClauses = researcher.getHelper( \"getClauses\" );\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\tconst sentences = getSentences( text, memoizedTokenizer )\n\t\t.map( function( sentence ) {\n\t\t\treturn new Sentence( sentence );\n\t\t} );\n\tconst totalNumberSentences = sentences.length;\n\tconst passiveSentences = [];\n\n\tforEach( sentences, function( sentence ) {\n\t\tconst strippedSentence = stripHTMLTags( sentence.getSentenceText() ).toLocaleLowerCase();\n\n\t\t// The functionality based on sentencePart objects should be rewritten using array indices of stopwords and auxiliaries.\n\n\t\t// Divide a sentence into clauses and return an array of clause objects that have been checked for passiveness.\n\t\tconst clauses = getClauses( strippedSentence );\n\t\tsentence.setClauses( clauses );\n\n\t\t// Check sentence passiveness based on its clause passiveness.\n\t\tif ( sentence.isPassive() ) {\n\t\t\tpassiveSentences.push( sentence.getSentenceText() );\n\t\t}\n\t} );\n\n\treturn {\n\t\ttotal: totalNumberSentences,\n\t\tpassives: passiveSentences,\n\t};\n};\n\n/**\n * Looks for both morphological and periphrastic passive voice\n * Supported languages with both morphological and periphrastic passives: \"hu\", \"nb\".\n * Due to technical difficulties \"nb\" is only implemented as periphrastic at the moment. Languages that have not been implemented yet: \"da\".\n *\n * @param {Paper}      paper      The paper object.\n * @param {Researcher} researcher The researcher.\n *\n * @returns {Object} The found passive sentences.\n */\nconst getMorphologicalAndPeriphrasticPassive = function( paper, researcher ) {\n\tconst morphologicalPassives = getMorphologicalPassives( paper, researcher );\n\tconst periphrasticPassives = getPeriphrasticPassives( paper, researcher ).passives;\n\n\treturn {\n\t\ttotal: morphologicalPassives.total,\n\t\tpassives: periphrasticPassives.concat( morphologicalPassives.passives ),\n\t};\n};\n\n/**\n * Looks for passive voice.\n *\n * @param {Paper}      paper      The paper object.\n * @param {Researcher} researcher The researcher.\n *\n * @returns {Object} The found passive sentences.\n */\nexport default function getPassiveVoice( paper, researcher ) {\n\tconst passiveType = researcher.getConfig( \"passiveConstructionType\" );\n\n\tif ( passiveType === \"periphrastic\" ) {\n\t\treturn getPeriphrasticPassives( paper, researcher );\n\t}\n\tif ( passiveType === \"morphological\" ) {\n\t\treturn getMorphologicalPassives( paper, researcher );\n\t}\n\n\treturn getMorphologicalAndPeriphrasticPassive( paper, researcher );\n}\n"],"mappings":";;;;;;;AAAA,IAAAA,aAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,cAAA,GAAAD,OAAA;AACA,IAAAE,SAAA,GAAAH,sBAAA,CAAAC,OAAA;AAEA,IAAAG,OAAA,GAAAH,OAAA;AACA,IAAAI,WAAA,GAAAL,sBAAA,CAAAC,OAAA;AACA,IAAAK,QAAA,GAAAL,OAAA;AAAsD,SAAAD,uBAAAO,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEtD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,MAAMG,wBAAwB,GAAG,SAAAA,CAAUC,KAAK,EAAEC,UAAU,EAAG;EACrE,MAAMC,iBAAiB,GAAGD,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;EACrE,IAAIC,IAAI,GAAGJ,KAAK,CAACK,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,iCAAwB,EAAEH,IAAI,EAAEJ,KAAK,CAACQ,WAAW,IAAIR,KAAK,CAACQ,WAAW,CAACC,UAAW,CAAC;EAC1F,MAAMC,iBAAiB,GAAGT,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;EACrE,MAAMQ,SAAS,GAAG,IAAAC,qBAAY,EAAER,IAAI,EAAEM,iBAAkB,CAAC,CACvDG,GAAG,CAAE,UAAUC,QAAQ,EAAG;IAC1B,OAAO,IAAIC,iBAAQ,CAAED,QAAS,CAAC;EAChC,CAAE,CAAC;EACJ,MAAME,oBAAoB,GAAGL,SAAS,CAACM,MAAM;EAC7C,MAAMC,gBAAgB,GAAG,EAAE;EAE3B,IAAAC,eAAO,EAAER,SAAS,EAAE,UAAUG,QAAQ,EAAG;IACxC,MAAMM,gBAAgB,GAAG,IAAAC,4BAAa,EAAEP,QAAQ,CAACQ,eAAe,CAAC,CAAE,CAAC,CAACC,iBAAiB,CAAC,CAAC;IAExFT,QAAQ,CAACU,UAAU,CAAEtB,iBAAiB,CAAEkB,gBAAiB,CAAE,CAAC;IAE5D,IAAKN,QAAQ,CAACW,SAAS,CAAC,CAAC,KAAK,IAAI,EAAG;MACpCP,gBAAgB,CAACQ,IAAI,CAAEZ,QAAQ,CAACQ,eAAe,CAAC,CAAE,CAAC;IACpD;EACD,CAAE,CAAC;EAEH,OAAO;IACNK,KAAK,EAAEX,oBAAoB;IAC3BY,QAAQ,EAAEV;EACX,CAAC;AACF,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AARAW,OAAA,CAAA9B,wBAAA,GAAAA,wBAAA;AASO,MAAM+B,uBAAuB,GAAG,SAAAA,CAAU9B,KAAK,EAAEC,UAAU,EAAG;EACpE,MAAM8B,UAAU,GAAG9B,UAAU,CAACE,SAAS,CAAE,YAAa,CAAC;EACvD,IAAIC,IAAI,GAAGJ,KAAK,CAACK,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,iCAAwB,EAAEH,IAAI,EAAEJ,KAAK,CAACQ,WAAW,IAAIR,KAAK,CAACQ,WAAW,CAACC,UAAW,CAAC;EAC1F,MAAMC,iBAAiB,GAAGT,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;EACrE,MAAMQ,SAAS,GAAG,IAAAC,qBAAY,EAAER,IAAI,EAAEM,iBAAkB,CAAC,CACvDG,GAAG,CAAE,UAAUC,QAAQ,EAAG;IAC1B,OAAO,IAAIC,iBAAQ,CAAED,QAAS,CAAC;EAChC,CAAE,CAAC;EACJ,MAAME,oBAAoB,GAAGL,SAAS,CAACM,MAAM;EAC7C,MAAMC,gBAAgB,GAAG,EAAE;EAE3B,IAAAC,eAAO,EAAER,SAAS,EAAE,UAAUG,QAAQ,EAAG;IACxC,MAAMM,gBAAgB,GAAG,IAAAC,4BAAa,EAAEP,QAAQ,CAACQ,eAAe,CAAC,CAAE,CAAC,CAACC,iBAAiB,CAAC,CAAC;;IAExF;;IAEA;IACA,MAAMS,OAAO,GAAGD,UAAU,CAAEX,gBAAiB,CAAC;IAC9CN,QAAQ,CAACmB,UAAU,CAAED,OAAQ,CAAC;;IAE9B;IACA,IAAKlB,QAAQ,CAACW,SAAS,CAAC,CAAC,EAAG;MAC3BP,gBAAgB,CAACQ,IAAI,CAAEZ,QAAQ,CAACQ,eAAe,CAAC,CAAE,CAAC;IACpD;EACD,CAAE,CAAC;EAEH,OAAO;IACNK,KAAK,EAAEX,oBAAoB;IAC3BY,QAAQ,EAAEV;EACX,CAAC;AACF,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AATAW,OAAA,CAAAC,uBAAA,GAAAA,uBAAA;AAUA,MAAMI,sCAAsC,GAAG,SAAAA,CAAUlC,KAAK,EAAEC,UAAU,EAAG;EAC5E,MAAMkC,qBAAqB,GAAGpC,wBAAwB,CAAEC,KAAK,EAAEC,UAAW,CAAC;EAC3E,MAAMmC,oBAAoB,GAAGN,uBAAuB,CAAE9B,KAAK,EAAEC,UAAW,CAAC,CAAC2B,QAAQ;EAElF,OAAO;IACND,KAAK,EAAEQ,qBAAqB,CAACR,KAAK;IAClCC,QAAQ,EAAEQ,oBAAoB,CAACC,MAAM,CAAEF,qBAAqB,CAACP,QAAS;EACvE,CAAC;AACF,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASU,eAAeA,CAAEtC,KAAK,EAAEC,UAAU,EAAG;EAC5D,MAAMsC,WAAW,GAAGtC,UAAU,CAACuC,SAAS,CAAE,yBAA0B,CAAC;EAErE,IAAKD,WAAW,KAAK,cAAc,EAAG;IACrC,OAAOT,uBAAuB,CAAE9B,KAAK,EAAEC,UAAW,CAAC;EACpD;EACA,IAAKsC,WAAW,KAAK,eAAe,EAAG;IACtC,OAAOxC,wBAAwB,CAAEC,KAAK,EAAEC,UAAW,CAAC;EACrD;EAEA,OAAOiC,sCAAsC,CAAElC,KAAK,EAAEC,UAAW,CAAC;AACnE","ignoreList":[]}

package/build/languageProcessing/researches/getProminentWordsForInsights.js

@@ -0,0 +1,45 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _determineProminentWords = require("../helpers/prominentWords/determineProminentWords");
+var _removeURLs = _interopRequireDefault(require("../helpers/sanitize/removeURLs.js"));
+var _removeEmailAddresses = _interopRequireDefault(require("../helpers/sanitize/removeEmailAddresses"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Retrieves the prominent words from the given paper.
+ *
+ * @param {Paper} paper             The paper to determine the prominent words of.
+ * @param {Researcher} researcher   The researcher to use for analysis.
+ *
+ * @returns {WordCombination[]} Prominent words for this paper, filtered and sorted.
+ */
+function getProminentWordsForInsights(paper, researcher) {
+  const functionWords = researcher.getConfig("functionWords");
+  // An optional custom helper to return custom function to return the stem of a word.
+  const customStemmer = researcher.getHelper("customGetStemmer");
+  const stemmer = customStemmer ? customStemmer(researcher) : researcher.getHelper("getStemmer")(researcher);
+  // An optional custom helper to get words from the text.
+  const getWordsCustomHelper = researcher.getHelper("getWordsCustomHelper");
+  let text = paper.getText();
+  // We don't want to include URLs or email addresses in prominent words.
+  text = (0, _removeURLs.default)(text);
+  text = (0, _removeEmailAddresses.default)(text);
+
+  // If the language has a custom helper to get words from the text, we don't retrieve the abbreviation.
+  const abbreviations = getWordsCustomHelper ? [] : (0, _determineProminentWords.retrieveAbbreviations)(text);
+  const prominentWordsFromText = (0, _determineProminentWords.getProminentWords)(text, abbreviations, stemmer, functionWords, getWordsCustomHelper);
+  const collapsedWords = (0, _determineProminentWords.collapseProminentWordsOnStem)(prominentWordsFromText);
+  (0, _determineProminentWords.sortProminentWords)(collapsedWords);
+
+  /*
+   * Collapse the list of prominent words on stems, sort it, filter out all words that occur less than
+   * 5 times in the text. Return the 20 top items from this list.
+   */
+  return (0, _lodash.take)((0, _determineProminentWords.filterProminentWords)(collapsedWords, 5), 20);
+}
+var _default = exports.default = getProminentWordsForInsights;
+//# sourceMappingURL=getProminentWordsForInsights.js.map

package/build/languageProcessing/researches/getProminentWordsForInsights.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getProminentWordsForInsights.js","names":["_lodash","require","_determineProminentWords","_removeURLs","_interopRequireDefault","_removeEmailAddresses","e","__esModule","default","getProminentWordsForInsights","paper","researcher","functionWords","getConfig","customStemmer","getHelper","stemmer","getWordsCustomHelper","text","getText","removeURLs","removeEmailAddresses","abbreviations","retrieveAbbreviations","prominentWordsFromText","getProminentWords","collapsedWords","collapseProminentWordsOnStem","sortProminentWords","take","filterProminentWords","_default","exports"],"sources":["../../../src/languageProcessing/researches/getProminentWordsForInsights.js"],"sourcesContent":["import { take } from \"lodash\";\nimport {\n\tcollapseProminentWordsOnStem,\n\tfilterProminentWords,\n\tgetProminentWords,\n\tretrieveAbbreviations,\n\tsortProminentWords,\n} from \"../helpers/prominentWords/determineProminentWords\";\nimport removeURLs from \"../helpers/sanitize/removeURLs.js\";\nimport removeEmailAddresses from \"../helpers/sanitize/removeEmailAddresses\";\n\n/**\n * Retrieves the prominent words from the given paper.\n *\n * @param {Paper} paper             The paper to determine the prominent words of.\n * @param {Researcher} researcher   The researcher to use for analysis.\n *\n * @returns {WordCombination[]} Prominent words for this paper, filtered and sorted.\n */\nfunction getProminentWordsForInsights( paper, researcher ) {\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\t// An optional custom helper to return custom function to return the stem of a word.\n\tconst customStemmer = researcher.getHelper( \"customGetStemmer\" );\n\tconst stemmer = customStemmer ? customStemmer( researcher ) : researcher.getHelper( \"getStemmer\" )( researcher );\n\t// An optional custom helper to get words from the text.\n\tconst getWordsCustomHelper = researcher.getHelper( \"getWordsCustomHelper\" );\n\n\tlet text = paper.getText();\n\t// We don't want to include URLs or email addresses in prominent words.\n\ttext = removeURLs( text );\n\ttext = removeEmailAddresses( text );\n\n\t// If the language has a custom helper to get words from the text, we don't retrieve the abbreviation.\n\tconst abbreviations = getWordsCustomHelper ? [] : retrieveAbbreviations( text );\n\n\tconst prominentWordsFromText = getProminentWords( text, abbreviations, stemmer, functionWords, getWordsCustomHelper );\n\n\tconst collapsedWords = collapseProminentWordsOnStem( prominentWordsFromText );\n\tsortProminentWords( collapsedWords );\n\n\t/*\n\t * Collapse the list of prominent words on stems, sort it, filter out all words that occur less than\n\t * 5 times in the text. Return the 20 top items from this list.\n\t */\n\treturn take( filterProminentWords( collapsedWords, 5 ), 20 );\n}\n\nexport default getProminentWordsForInsights;\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,wBAAA,GAAAD,OAAA;AAOA,IAAAE,WAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAD,sBAAA,CAAAH,OAAA;AAA4E,SAAAG,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE5E;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASG,4BAA4BA,CAAEC,KAAK,EAAEC,UAAU,EAAG;EAC1D,MAAMC,aAAa,GAAGD,UAAU,CAACE,SAAS,CAAE,eAAgB,CAAC;EAC7D;EACA,MAAMC,aAAa,GAAGH,UAAU,CAACI,SAAS,CAAE,kBAAmB,CAAC;EAChE,MAAMC,OAAO,GAAGF,aAAa,GAAGA,aAAa,CAAEH,UAAW,CAAC,GAAGA,UAAU,CAACI,SAAS,CAAE,YAAa,CAAC,CAAEJ,UAAW,CAAC;EAChH;EACA,MAAMM,oBAAoB,GAAGN,UAAU,CAACI,SAAS,CAAE,sBAAuB,CAAC;EAE3E,IAAIG,IAAI,GAAGR,KAAK,CAACS,OAAO,CAAC,CAAC;EAC1B;EACAD,IAAI,GAAG,IAAAE,mBAAU,EAAEF,IAAK,CAAC;EACzBA,IAAI,GAAG,IAAAG,6BAAoB,EAAEH,IAAK,CAAC;;EAEnC;EACA,MAAMI,aAAa,GAAGL,oBAAoB,GAAG,EAAE,GAAG,IAAAM,8CAAqB,EAAEL,IAAK,CAAC;EAE/E,MAAMM,sBAAsB,GAAG,IAAAC,0CAAiB,EAAEP,IAAI,EAAEI,aAAa,EAAEN,OAAO,EAAEJ,aAAa,EAAEK,oBAAqB,CAAC;EAErH,MAAMS,cAAc,GAAG,IAAAC,qDAA4B,EAAEH,sBAAuB,CAAC;EAC7E,IAAAI,2CAAkB,EAAEF,cAAe,CAAC;;EAEpC;AACD;AACA;AACA;EACC,OAAO,IAAAG,YAAI,EAAE,IAAAC,6CAAoB,EAAEJ,cAAc,EAAE,CAAE,CAAC,EAAE,EAAG,CAAC;AAC7D;AAAC,IAAAK,QAAA,GAAAC,OAAA,CAAAxB,OAAA,GAEcC,4BAA4B","ignoreList":[]}

package/build/languageProcessing/researches/getProminentWordsForInternalLinking.js

@@ -0,0 +1,103 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _countWords = _interopRequireDefault(require("../helpers/word/countWords"));
+var _determineProminentWords = require("../helpers/prominentWords/determineProminentWords");
+var _getSubheadings = require("../helpers/html/getSubheadings");
+var _baseStemmer = _interopRequireDefault(require("../helpers/morphology/baseStemmer"));
+var _removeURLs = _interopRequireDefault(require("../helpers/sanitize/removeURLs.js"));
+var _removeEmailAddresses = _interopRequireDefault(require("../helpers/sanitize/removeEmailAddresses"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Removes URLs and email addresses from the text.
+ *
+ * @param {string}	text	The text to sanitize.
+ *
+ * @returns {string} The text without URLs and email addresses.
+ */
+const sanitizeText = function (text) {
+  text = (0, _removeURLs.default)(text);
+  return (0, _removeEmailAddresses.default)(text);
+};
+
+/**
+ * Retrieves the prominent words from the given paper.
+ *
+ * @param {Paper}       paper       The paper to determine the prominent words of.
+ * @param {Researcher}  researcher  The researcher to use for analysis.
+ *
+ * @returns {Object}          result                    A compound result object.
+ * @returns {ProminentWord[]} result.prominentWords     Prominent words for this paper, filtered and sorted.
+ * @returns {boolean}         result.hasMetaDescription Whether the metadescription is available in the input paper.
+ * @returns {boolean}         result.hasTitle           Whether the title is available in the input paper.
+ */
+function getProminentWordsForInternalLinking(paper, researcher) {
+  const functionWords = researcher.getConfig("functionWords");
+  // An optional custom helper to return custom function to return the stem of a word.
+  const customStemmer = researcher.getHelper("customGetStemmer");
+  const stemmer = customStemmer ? customStemmer(researcher) : researcher.getHelper("getStemmer")(researcher);
+  // An optional custom helper to get words from the text.
+  const getWordsCustomHelper = researcher.getHelper("getWordsCustomHelper");
+  // An optional custom helper to count length to use instead of countWords.
+  const customCountLength = researcher.getHelper("customCountLength");
+  const text = sanitizeText(paper.getText());
+  const metadescription = sanitizeText(paper.getDescription());
+  const title = sanitizeText(paper.getTitle());
+  const result = {};
+  result.hasMetaDescription = metadescription !== "";
+  result.hasTitle = title !== "";
+  result.prominentWords = [];
+
+  /**
+   * We only want to return suggestions (and spend time calculating prominent words) if the text is at least 100 words.
+   * And when a customCountLength is available, we only want to return the suggestions if the text has at least 200 characters.
+  	 */
+  if (customCountLength) {
+    if (customCountLength(text) < 200) {
+      return result;
+    }
+  } else if ((0, _countWords.default)(text) < 100) {
+    return result;
+  }
+  const subheadings = (0, _getSubheadings.getSubheadingsTopLevel)(text).map(subheading => subheading[2]);
+  const attributes = [paper.getKeyword(), paper.getSynonyms(), title, metadescription, subheadings.join(" ")];
+
+  // If the language has a custom helper to get words from the text, we don't retrieve the abbreviation.
+  const abbreviations = getWordsCustomHelper ? [] : (0, _determineProminentWords.retrieveAbbreviations)(text.concat(attributes.join(" ")));
+  const removedSubheadingText = (0, _getSubheadings.removeSubheadingsTopLevel)(text);
+  const prominentWordsFromText = (0, _determineProminentWords.getProminentWords)(removedSubheadingText, abbreviations, stemmer, functionWords, getWordsCustomHelper);
+  const prominentWordsFromPaperAttributes = (0, _determineProminentWords.getProminentWordsFromPaperAttributes)(attributes, abbreviations, stemmer, functionWords, getWordsCustomHelper);
+
+  /*
+   * If a word is used in any of the attributes, its weight is automatically high.
+   * To make sure the word survives weight filters and gets saved in the database, make the number of occurrences times-3.
+   */
+  prominentWordsFromPaperAttributes.forEach(relevantWord => relevantWord.setOccurrences(relevantWord.getOccurrences() * 3));
+  const collapsedWords = (0, _determineProminentWords.collapseProminentWordsOnStem)(prominentWordsFromPaperAttributes.concat(prominentWordsFromText));
+  (0, _determineProminentWords.sortProminentWords)(collapsedWords);
+
+  /*
+   * If morphology data are available for a language, the minimum number of occurrences to consider a word to be prominent is 4.
+   * This minimum number was chosen in order to avoid premature suggestions of words from the paper attributes.
+   * These get a times-3 boost and would therefore be prominent with just 1 occurrence.
+   *
+   * If morphology data are not available, and therefore word forms are not recognized, the minimum threshold is lowered to 2.
+   */
+  let minimumNumberOfOccurrences = 4;
+  if (stemmer === _baseStemmer.default) {
+    minimumNumberOfOccurrences = 2;
+  }
+
+  /*
+   * Return the 100 top items from the collapsed and sorted list. The number is picked deliberately to prevent larger
+   * articles from getting too long of lists.
+   */
+  result.prominentWords = (0, _lodash.take)((0, _determineProminentWords.filterProminentWords)(collapsedWords, minimumNumberOfOccurrences), 100);
+  return result;
+}
+var _default = exports.default = getProminentWordsForInternalLinking;
+//# sourceMappingURL=getProminentWordsForInternalLinking.js.map

package/build/languageProcessing/researches/getProminentWordsForInternalLinking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getProminentWordsForInternalLinking.js","names":["_lodash","require","_countWords","_interopRequireDefault","_determineProminentWords","_getSubheadings","_baseStemmer","_removeURLs","_removeEmailAddresses","e","__esModule","default","sanitizeText","text","removeURLs","removeEmailAddresses","getProminentWordsForInternalLinking","paper","researcher","functionWords","getConfig","customStemmer","getHelper","stemmer","getWordsCustomHelper","customCountLength","getText","metadescription","getDescription","title","getTitle","result","hasMetaDescription","hasTitle","prominentWords","countWords","subheadings","getSubheadingsTopLevel","map","subheading","attributes","getKeyword","getSynonyms","join","abbreviations","retrieveAbbreviations","concat","removedSubheadingText","removeSubheadingsTopLevel","prominentWordsFromText","getProminentWords","prominentWordsFromPaperAttributes","getProminentWordsFromPaperAttributes","forEach","relevantWord","setOccurrences","getOccurrences","collapsedWords","collapseProminentWordsOnStem","sortProminentWords","minimumNumberOfOccurrences","baseStemmer","take","filterProminentWords","_default","exports"],"sources":["../../../src/languageProcessing/researches/getProminentWordsForInternalLinking.js"],"sourcesContent":["import { take } from \"lodash\";\nimport countWords from \"../helpers/word/countWords\";\nimport {\n\tcollapseProminentWordsOnStem,\n\tfilterProminentWords,\n\tgetProminentWords,\n\tgetProminentWordsFromPaperAttributes,\n\tretrieveAbbreviations,\n\tsortProminentWords,\n} from \"../helpers/prominentWords/determineProminentWords\";\nimport { getSubheadingsTopLevel, removeSubheadingsTopLevel } from \"../helpers/html/getSubheadings\";\nimport baseStemmer from \"../helpers/morphology/baseStemmer\";\nimport removeURLs from \"../helpers/sanitize/removeURLs.js\";\nimport removeEmailAddresses from \"../helpers/sanitize/removeEmailAddresses\";\n\n/**\n * Removes URLs and email addresses from the text.\n *\n * @param {string}\ttext\tThe text to sanitize.\n *\n * @returns {string} The text without URLs and email addresses.\n */\nconst sanitizeText = function( text ) {\n\ttext = removeURLs( text );\n\treturn removeEmailAddresses( text );\n};\n\n/**\n * Retrieves the prominent words from the given paper.\n *\n * @param {Paper}       paper       The paper to determine the prominent words of.\n * @param {Researcher}  researcher  The researcher to use for analysis.\n *\n * @returns {Object}          result                    A compound result object.\n * @returns {ProminentWord[]} result.prominentWords     Prominent words for this paper, filtered and sorted.\n * @returns {boolean}         result.hasMetaDescription Whether the metadescription is available in the input paper.\n * @returns {boolean}         result.hasTitle           Whether the title is available in the input paper.\n */\nfunction getProminentWordsForInternalLinking( paper, researcher ) {\n\tconst functionWords = researcher.getConfig( \"functionWords\" );\n\t// An optional custom helper to return custom function to return the stem of a word.\n\tconst customStemmer = researcher.getHelper( \"customGetStemmer\" );\n\tconst stemmer = customStemmer ? customStemmer( researcher ) : researcher.getHelper( \"getStemmer\" )( researcher );\n\t// An optional custom helper to get words from the text.\n\tconst getWordsCustomHelper = researcher.getHelper( \"getWordsCustomHelper\" );\n\t// An optional custom helper to count length to use instead of countWords.\n\tconst customCountLength = researcher.getHelper( \"customCountLength\" );\n\n\tconst text = sanitizeText( paper.getText() );\n\tconst metadescription = sanitizeText( paper.getDescription() );\n\tconst title = sanitizeText( paper.getTitle() );\n\n\tconst result = {};\n\tresult.hasMetaDescription = metadescription !== \"\";\n\tresult.hasTitle = title !== \"\";\n\tresult.prominentWords = [];\n\n\t/**\n\t * We only want to return suggestions (and spend time calculating prominent words) if the text is at least 100 words.\n\t * And when a customCountLength is available, we only want to return the suggestions if the text has at least 200 characters.\n \t */\n\tif ( customCountLength ) {\n\t\tif ( customCountLength( text ) < 200 ) {\n\t\t\treturn result;\n\t\t}\n\t} else if ( countWords( text ) < 100 ) {\n\t\treturn result;\n\t}\n\n\tconst subheadings = getSubheadingsTopLevel( text ).map( subheading => subheading[ 2 ] );\n\tconst attributes = [\n\t\tpaper.getKeyword(),\n\t\tpaper.getSynonyms(),\n\t\ttitle,\n\t\tmetadescription,\n\t\tsubheadings.join( \" \" ),\n\t];\n\n\t// If the language has a custom helper to get words from the text, we don't retrieve the abbreviation.\n\tconst abbreviations = getWordsCustomHelper ? [] : retrieveAbbreviations( text.concat( attributes.join( \" \" ) ) );\n\n\tconst removedSubheadingText = removeSubheadingsTopLevel( text );\n\tconst prominentWordsFromText = getProminentWords( removedSubheadingText, abbreviations, stemmer, functionWords, getWordsCustomHelper );\n\n\tconst prominentWordsFromPaperAttributes = getProminentWordsFromPaperAttributes(\n\t\tattributes, abbreviations, stemmer, functionWords, getWordsCustomHelper );\n\n\t/*\n\t * If a word is used in any of the attributes, its weight is automatically high.\n\t * To make sure the word survives weight filters and gets saved in the database, make the number of occurrences times-3.\n\t */\n\tprominentWordsFromPaperAttributes.forEach( relevantWord => relevantWord.setOccurrences( relevantWord.getOccurrences() * 3 ) );\n\n\tconst collapsedWords = collapseProminentWordsOnStem( prominentWordsFromPaperAttributes.concat( prominentWordsFromText ) );\n\tsortProminentWords( collapsedWords );\n\n\t/*\n\t * If morphology data are available for a language, the minimum number of occurrences to consider a word to be prominent is 4.\n\t * This minimum number was chosen in order to avoid premature suggestions of words from the paper attributes.\n\t * These get a times-3 boost and would therefore be prominent with just 1 occurrence.\n\t *\n\t * If morphology data are not available, and therefore word forms are not recognized, the minimum threshold is lowered to 2.\n\t */\n\tlet minimumNumberOfOccurrences = 4;\n\n\tif ( stemmer === baseStemmer ) {\n\t\tminimumNumberOfOccurrences = 2;\n\t}\n\n\t/*\n\t * Return the 100 top items from the collapsed and sorted list. The number is picked deliberately to prevent larger\n\t * articles from getting too long of lists.\n\t */\n\tresult.prominentWords = take( filterProminentWords( collapsedWords, minimumNumberOfOccurrences ), 100 );\n\n\treturn result;\n}\n\nexport default getProminentWordsForInternalLinking;\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,WAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,wBAAA,GAAAH,OAAA;AAQA,IAAAI,eAAA,GAAAJ,OAAA;AACA,IAAAK,YAAA,GAAAH,sBAAA,CAAAF,OAAA;AACA,IAAAM,WAAA,GAAAJ,sBAAA,CAAAF,OAAA;AACA,IAAAO,qBAAA,GAAAL,sBAAA,CAAAF,OAAA;AAA4E,SAAAE,uBAAAM,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE5E;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,YAAY,GAAG,SAAAA,CAAUC,IAAI,EAAG;EACrCA,IAAI,GAAG,IAAAC,mBAAU,EAAED,IAAK,CAAC;EACzB,OAAO,IAAAE,6BAAoB,EAAEF,IAAK,CAAC;AACpC,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASG,mCAAmCA,CAAEC,KAAK,EAAEC,UAAU,EAAG;EACjE,MAAMC,aAAa,GAAGD,UAAU,CAACE,SAAS,CAAE,eAAgB,CAAC;EAC7D;EACA,MAAMC,aAAa,GAAGH,UAAU,CAACI,SAAS,CAAE,kBAAmB,CAAC;EAChE,MAAMC,OAAO,GAAGF,aAAa,GAAGA,aAAa,CAAEH,UAAW,CAAC,GAAGA,UAAU,CAACI,SAAS,CAAE,YAAa,CAAC,CAAEJ,UAAW,CAAC;EAChH;EACA,MAAMM,oBAAoB,GAAGN,UAAU,CAACI,SAAS,CAAE,sBAAuB,CAAC;EAC3E;EACA,MAAMG,iBAAiB,GAAGP,UAAU,CAACI,SAAS,CAAE,mBAAoB,CAAC;EAErE,MAAMT,IAAI,GAAGD,YAAY,CAAEK,KAAK,CAACS,OAAO,CAAC,CAAE,CAAC;EAC5C,MAAMC,eAAe,GAAGf,YAAY,CAAEK,KAAK,CAACW,cAAc,CAAC,CAAE,CAAC;EAC9D,MAAMC,KAAK,GAAGjB,YAAY,CAAEK,KAAK,CAACa,QAAQ,CAAC,CAAE,CAAC;EAE9C,MAAMC,MAAM,GAAG,CAAC,CAAC;EACjBA,MAAM,CAACC,kBAAkB,GAAGL,eAAe,KAAK,EAAE;EAClDI,MAAM,CAACE,QAAQ,GAAGJ,KAAK,KAAK,EAAE;EAC9BE,MAAM,CAACG,cAAc,GAAG,EAAE;;EAE1B;AACD;AACA;AACA;EACC,IAAKT,iBAAiB,EAAG;IACxB,IAAKA,iBAAiB,CAAEZ,IAAK,CAAC,GAAG,GAAG,EAAG;MACtC,OAAOkB,MAAM;IACd;EACD,CAAC,MAAM,IAAK,IAAAI,mBAAU,EAAEtB,IAAK,CAAC,GAAG,GAAG,EAAG;IACtC,OAAOkB,MAAM;EACd;EAEA,MAAMK,WAAW,GAAG,IAAAC,sCAAsB,EAAExB,IAAK,CAAC,CAACyB,GAAG,CAAEC,UAAU,IAAIA,UAAU,CAAE,CAAC,CAAG,CAAC;EACvF,MAAMC,UAAU,GAAG,CAClBvB,KAAK,CAACwB,UAAU,CAAC,CAAC,EAClBxB,KAAK,CAACyB,WAAW,CAAC,CAAC,EACnBb,KAAK,EACLF,eAAe,EACfS,WAAW,CAACO,IAAI,CAAE,GAAI,CAAC,CACvB;;EAED;EACA,MAAMC,aAAa,GAAGpB,oBAAoB,GAAG,EAAE,GAAG,IAAAqB,8CAAqB,EAAEhC,IAAI,CAACiC,MAAM,CAAEN,UAAU,CAACG,IAAI,CAAE,GAAI,CAAE,CAAE,CAAC;EAEhH,MAAMI,qBAAqB,GAAG,IAAAC,yCAAyB,EAAEnC,IAAK,CAAC;EAC/D,MAAMoC,sBAAsB,GAAG,IAAAC,0CAAiB,EAAEH,qBAAqB,EAAEH,aAAa,EAAErB,OAAO,EAAEJ,aAAa,EAAEK,oBAAqB,CAAC;EAEtI,MAAM2B,iCAAiC,GAAG,IAAAC,6DAAoC,EAC7EZ,UAAU,EAAEI,aAAa,EAAErB,OAAO,EAAEJ,aAAa,EAAEK,oBAAqB,CAAC;;EAE1E;AACD;AACA;AACA;EACC2B,iCAAiC,CAACE,OAAO,CAAEC,YAAY,IAAIA,YAAY,CAACC,cAAc,CAAED,YAAY,CAACE,cAAc,CAAC,CAAC,GAAG,CAAE,CAAE,CAAC;EAE7H,MAAMC,cAAc,GAAG,IAAAC,qDAA4B,EAAEP,iCAAiC,CAACL,MAAM,CAAEG,sBAAuB,CAAE,CAAC;EACzH,IAAAU,2CAAkB,EAAEF,cAAe,CAAC;;EAEpC;AACD;AACA;AACA;AACA;AACA;AACA;EACC,IAAIG,0BAA0B,GAAG,CAAC;EAElC,IAAKrC,OAAO,KAAKsC,oBAAW,EAAG;IAC9BD,0BAA0B,GAAG,CAAC;EAC/B;;EAEA;AACD;AACA;AACA;EACC7B,MAAM,CAACG,cAAc,GAAG,IAAA4B,YAAI,EAAE,IAAAC,6CAAoB,EAAEN,cAAc,EAAEG,0BAA2B,CAAC,EAAE,GAAI,CAAC;EAEvG,OAAO7B,MAAM;AACd;AAAC,IAAAiC,QAAA,GAAAC,OAAA,CAAAtD,OAAA,GAEcK,mCAAmC","ignoreList":[]}

package/build/languageProcessing/researches/getSentenceBeginnings.js

@@ -0,0 +1,122 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _getWords = _interopRequireDefault(require("../helpers/word/getWords.js"));
+var _getSentences = _interopRequireDefault(require("../helpers/sentence/getSentences"));
+var _stripSpaces = _interopRequireDefault(require("../helpers/sanitize/stripSpaces.js"));
+var _stripHTMLTags = require("../helpers/sanitize/stripHTMLTags.js");
+var _lodash = require("lodash");
+var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
+var _helpers = require("../helpers");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Compares the first word of each sentence with the first word of the following sentence.
+ *
+ * @param {string} currentSentenceBeginning The first word of the current sentence.
+ * @param {string} nextSentenceBeginning The first word of the next sentence.
+ * @returns {boolean} Returns true if sentence beginnings match.
+ */
+const startsWithSameWord = function (currentSentenceBeginning, nextSentenceBeginning) {
+  return !(0, _lodash.isEmpty)(currentSentenceBeginning) && currentSentenceBeginning === nextSentenceBeginning;
+};
+
+/**
+ * Counts the number of similar sentence beginnings.
+ *
+ * @param {Array} sentenceBeginnings The array containing the first word of each sentence.
+ * @param {Array} sentences The array containing all sentences.
+ * @returns {Array} The array containing the objects containing the first words and the corresponding counts.
+ */
+const compareFirstWords = function (sentenceBeginnings, sentences) {
+  const consecutiveFirstWords = [];
+  let foundSentences = [];
+  let sameBeginnings = 1;
+  (0, _lodash.forEach)(sentenceBeginnings, function (beginning, i) {
+    const currentSentenceBeginning = beginning;
+    const nextSentenceBeginning = sentenceBeginnings[i + 1];
+    foundSentences.push(sentences[i]);
+    if (startsWithSameWord(currentSentenceBeginning, nextSentenceBeginning)) {
+      sameBeginnings++;
+    } else {
+      consecutiveFirstWords.push({
+        word: currentSentenceBeginning,
+        count: sameBeginnings,
+        sentences: foundSentences
+      });
+      sameBeginnings = 1;
+      foundSentences = [];
+    }
+  });
+  return consecutiveFirstWords;
+};
+
+/**
+ * Retrieves the first word from the sentence. If the first or second word is on an exception list of words that should not be considered as sentence
+ * beginnings, the following word is also retrieved.
+ *
+ * @param {string}  sentence                The sentence to retrieve the first word from.
+ * @param {Array}   firstWordExceptions     First word exceptions to match against.
+ * @param {Array}   secondWordExceptions    Second word exceptions to match against.
+ * @param {function}	getWordsCustomHelper   The language-specific helper function to retrieve words from text.
+ *
+ * @returns {string} The first word of the sentence.
+ */
+function getSentenceBeginning(sentence, firstWordExceptions, secondWordExceptions, getWordsCustomHelper) {
+  const stripped = (0, _stripHTMLTags.stripFullTags)((0, _stripSpaces.default)(sentence));
+  const words = getWordsCustomHelper ? getWordsCustomHelper(stripped) : (0, _getWords.default)(stripped);
+  if (words.length === 0) {
+    return "";
+  }
+  let firstWord = words[0].toLocaleLowerCase();
+  if (firstWordExceptions.indexOf(firstWord) > -1 && words.length > 1) {
+    firstWord = firstWord + " " + words[1];
+    if (secondWordExceptions) {
+      if (secondWordExceptions.includes(words[1])) {
+        firstWord = firstWord + " " + words[2];
+      }
+    }
+  }
+  return firstWord;
+}
+
+/**
+ * Gets the first word of each sentence from the text, and returns an object containing the first word of each sentence and the corresponding counts.
+ *
+ * @param {Paper}       paper       The Paper object to get the text from.
+ * @param {Researcher}  researcher  The researcher this research is a part of.
+ *
+ * @returns {Object} The object containing the first word of each sentence and the corresponding counts.
+ */
+function _default(paper, researcher) {
+  const firstWordExceptions = researcher.getConfig("firstWordExceptions");
+  const secondWordExceptions = researcher.getConfig("secondWordExceptions");
+  const getWordsCustomHelper = researcher.getHelper("getWordsCustomHelper");
+  const memoizedTokenizer = researcher.getHelper("memoizedTokenizer");
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+
+  // Remove any HTML whitespace padding and replace it with a single whitespace.
+  text = text.replace(/[\s\n]+/g, " ");
+
+  // Exclude text inside tables.
+  text = text.replace(/<figure class='wp-block-table'>.*<\/figure>/sg, "");
+
+  // Exclude text inside list items.
+  text = text.replace(/<li(?:[^>]+)?>(.*?)<\/li>/ig, "");
+  let sentences = (0, _getSentences.default)(text, memoizedTokenizer);
+  let sentenceBeginnings = sentences.map(function (sentence) {
+    return getSentenceBeginning(sentence, firstWordExceptions, secondWordExceptions, getWordsCustomHelper);
+  });
+  sentences = sentences.filter(function (sentence) {
+    const stripped = (0, _stripSpaces.default)(sentence);
+    const words = getWordsCustomHelper ? getWordsCustomHelper(stripped) : (0, _getWords.default)(stripped);
+    return words.length > 0;
+  });
+  sentenceBeginnings = (0, _lodash.filter)(sentenceBeginnings);
+  return compareFirstWords(sentenceBeginnings, sentences);
+}
+//# sourceMappingURL=getSentenceBeginnings.js.map

package/build/languageProcessing/researches/getSentenceBeginnings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getSentenceBeginnings.js","names":["_getWords","_interopRequireDefault","require","_getSentences","_stripSpaces","_stripHTMLTags","_lodash","_htmlParser","_helpers","e","__esModule","default","startsWithSameWord","currentSentenceBeginning","nextSentenceBeginning","isEmpty","compareFirstWords","sentenceBeginnings","sentences","consecutiveFirstWords","foundSentences","sameBeginnings","forEach","beginning","i","push","word","count","getSentenceBeginning","sentence","firstWordExceptions","secondWordExceptions","getWordsCustomHelper","stripped","stripTags","stripSpaces","words","getWords","length","firstWord","toLocaleLowerCase","indexOf","includes","_default","paper","researcher","getConfig","getHelper","memoizedTokenizer","text","getText","removeHtmlBlocks","filterShortcodesFromHTML","_attributes","shortcodes","replace","getSentences","map","filter"],"sources":["../../../src/languageProcessing/researches/getSentenceBeginnings.js"],"sourcesContent":["import getWords from \"../helpers/word/getWords.js\";\nimport getSentences from \"../helpers/sentence/getSentences\";\nimport stripSpaces from \"../helpers/sanitize/stripSpaces.js\";\nimport { stripFullTags as stripTags } from \"../helpers/sanitize/stripHTMLTags.js\";\n\nimport { filter, forEach, isEmpty } from \"lodash\";\nimport removeHtmlBlocks from \"../helpers/html/htmlParser\";\nimport { filterShortcodesFromHTML } from \"../helpers\";\n\n/**\n * Compares the first word of each sentence with the first word of the following sentence.\n *\n * @param {string} currentSentenceBeginning The first word of the current sentence.\n * @param {string} nextSentenceBeginning The first word of the next sentence.\n * @returns {boolean} Returns true if sentence beginnings match.\n */\nconst startsWithSameWord = function( currentSentenceBeginning, nextSentenceBeginning ) {\n\treturn ! isEmpty( currentSentenceBeginning ) && currentSentenceBeginning === nextSentenceBeginning;\n};\n\n/**\n * Counts the number of similar sentence beginnings.\n *\n * @param {Array} sentenceBeginnings The array containing the first word of each sentence.\n * @param {Array} sentences The array containing all sentences.\n * @returns {Array} The array containing the objects containing the first words and the corresponding counts.\n */\nconst compareFirstWords = function( sentenceBeginnings, sentences ) {\n\tconst consecutiveFirstWords = [];\n\tlet foundSentences = [];\n\tlet sameBeginnings = 1;\n\n\tforEach( sentenceBeginnings, function( beginning, i ) {\n\t\tconst currentSentenceBeginning = beginning;\n\t\tconst nextSentenceBeginning = sentenceBeginnings[ i + 1 ];\n\t\tfoundSentences.push( sentences[ i ] );\n\n\t\tif ( startsWithSameWord( currentSentenceBeginning, nextSentenceBeginning ) ) {\n\t\t\tsameBeginnings++;\n\t\t} else {\n\t\t\tconsecutiveFirstWords.push( { word: currentSentenceBeginning, count: sameBeginnings, sentences: foundSentences } );\n\t\t\tsameBeginnings = 1;\n\t\t\tfoundSentences = [];\n\t\t}\n\t} );\n\n\treturn consecutiveFirstWords;\n};\n\n/**\n * Retrieves the first word from the sentence. If the first or second word is on an exception list of words that should not be considered as sentence\n * beginnings, the following word is also retrieved.\n *\n * @param {string}  sentence                The sentence to retrieve the first word from.\n * @param {Array}   firstWordExceptions     First word exceptions to match against.\n * @param {Array}   secondWordExceptions    Second word exceptions to match against.\n * @param {function}\tgetWordsCustomHelper   The language-specific helper function to retrieve words from text.\n *\n * @returns {string} The first word of the sentence.\n */\nfunction getSentenceBeginning( sentence, firstWordExceptions, secondWordExceptions, getWordsCustomHelper ) {\n\tconst stripped = stripTags( stripSpaces( sentence ) );\n\tconst words = getWordsCustomHelper ? getWordsCustomHelper( stripped ) : getWords( stripped );\n\n\tif ( words.length === 0 ) {\n\t\treturn \"\";\n\t}\n\n\tlet firstWord = words[ 0 ].toLocaleLowerCase();\n\n\tif ( firstWordExceptions.indexOf( firstWord ) > -1 && words.length > 1 ) {\n\t\tfirstWord = firstWord + \" \" + words[ 1 ];\n\t\tif ( secondWordExceptions ) {\n\t\t\tif ( secondWordExceptions.includes( words[ 1 ] ) ) {\n\t\t\t\tfirstWord = firstWord + \" \" + words[ 2 ];\n\t\t\t}\n\t\t}\n\t}\n\n\treturn firstWord;\n}\n\n/**\n * Gets the first word of each sentence from the text, and returns an object containing the first word of each sentence and the corresponding counts.\n *\n * @param {Paper}       paper       The Paper object to get the text from.\n * @param {Researcher}  researcher  The researcher this research is a part of.\n *\n * @returns {Object} The object containing the first word of each sentence and the corresponding counts.\n */\nexport default function( paper, researcher ) {\n\tconst firstWordExceptions = researcher.getConfig( \"firstWordExceptions\" );\n\tconst secondWordExceptions = researcher.getConfig( \"secondWordExceptions\" );\n\tconst getWordsCustomHelper = researcher.getHelper( \"getWordsCustomHelper\" );\n\tconst memoizedTokenizer = researcher.getHelper( \"memoizedTokenizer\" );\n\n\tlet text = paper.getText();\n\ttext = removeHtmlBlocks( text );\n\ttext = filterShortcodesFromHTML( text, paper._attributes && paper._attributes.shortcodes );\n\n\t// Remove any HTML whitespace padding and replace it with a single whitespace.\n\ttext = text.replace( /[\\s\\n]+/g, \" \" );\n\n\t// Exclude text inside tables.\n\ttext = text.replace( /<figure class='wp-block-table'>.*<\\/figure>/sg, \"\" );\n\n\t// Exclude text inside list items.\n\ttext = text.replace( /<li(?:[^>]+)?>(.*?)<\\/li>/ig, \"\" );\n\n\tlet sentences = getSentences( text, memoizedTokenizer );\n\n\tlet sentenceBeginnings = sentences.map( function( sentence ) {\n\t\treturn getSentenceBeginning( sentence, firstWordExceptions, secondWordExceptions, getWordsCustomHelper );\n\t} );\n\n\tsentences = sentences.filter( function( sentence ) {\n\t\tconst stripped = stripSpaces( sentence );\n\t\tconst words = getWordsCustomHelper ? getWordsCustomHelper( stripped ) : getWords( stripped );\n\t\treturn words.length > 0;\n\t} );\n\tsentenceBeginnings = filter( sentenceBeginnings );\n\n\treturn compareFirstWords( sentenceBeginnings, sentences );\n}\n"],"mappings":";;;;;;AAAA,IAAAA,SAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,aAAA,GAAAF,sBAAA,CAAAC,OAAA;AACA,IAAAE,YAAA,GAAAH,sBAAA,CAAAC,OAAA;AACA,IAAAG,cAAA,GAAAH,OAAA;AAEA,IAAAI,OAAA,GAAAJ,OAAA;AACA,IAAAK,WAAA,GAAAN,sBAAA,CAAAC,OAAA;AACA,IAAAM,QAAA,GAAAN,OAAA;AAAsD,SAAAD,uBAAAQ,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEtD;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,kBAAkB,GAAG,SAAAA,CAAUC,wBAAwB,EAAEC,qBAAqB,EAAG;EACtF,OAAO,CAAE,IAAAC,eAAO,EAAEF,wBAAyB,CAAC,IAAIA,wBAAwB,KAAKC,qBAAqB;AACnG,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAME,iBAAiB,GAAG,SAAAA,CAAUC,kBAAkB,EAAEC,SAAS,EAAG;EACnE,MAAMC,qBAAqB,GAAG,EAAE;EAChC,IAAIC,cAAc,GAAG,EAAE;EACvB,IAAIC,cAAc,GAAG,CAAC;EAEtB,IAAAC,eAAO,EAAEL,kBAAkB,EAAE,UAAUM,SAAS,EAAEC,CAAC,EAAG;IACrD,MAAMX,wBAAwB,GAAGU,SAAS;IAC1C,MAAMT,qBAAqB,GAAGG,kBAAkB,CAAEO,CAAC,GAAG,CAAC,CAAE;IACzDJ,cAAc,CAACK,IAAI,CAAEP,SAAS,CAAEM,CAAC,CAAG,CAAC;IAErC,IAAKZ,kBAAkB,CAAEC,wBAAwB,EAAEC,qBAAsB,CAAC,EAAG;MAC5EO,cAAc,EAAE;IACjB,CAAC,MAAM;MACNF,qBAAqB,CAACM,IAAI,CAAE;QAAEC,IAAI,EAAEb,wBAAwB;QAAEc,KAAK,EAAEN,cAAc;QAAEH,SAAS,EAAEE;MAAe,CAAE,CAAC;MAClHC,cAAc,GAAG,CAAC;MAClBD,cAAc,GAAG,EAAE;IACpB;EACD,CAAE,CAAC;EAEH,OAAOD,qBAAqB;AAC7B,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASS,oBAAoBA,CAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,oBAAoB,EAAEC,oBAAoB,EAAG;EAC1G,MAAMC,QAAQ,GAAG,IAAAC,4BAAS,EAAE,IAAAC,oBAAW,EAAEN,QAAS,CAAE,CAAC;EACrD,MAAMO,KAAK,GAAGJ,oBAAoB,GAAGA,oBAAoB,CAAEC,QAAS,CAAC,GAAG,IAAAI,iBAAQ,EAAEJ,QAAS,CAAC;EAE5F,IAAKG,KAAK,CAACE,MAAM,KAAK,CAAC,EAAG;IACzB,OAAO,EAAE;EACV;EAEA,IAAIC,SAAS,GAAGH,KAAK,CAAE,CAAC,CAAE,CAACI,iBAAiB,CAAC,CAAC;EAE9C,IAAKV,mBAAmB,CAACW,OAAO,CAAEF,SAAU,CAAC,GAAG,CAAC,CAAC,IAAIH,KAAK,CAACE,MAAM,GAAG,CAAC,EAAG;IACxEC,SAAS,GAAGA,SAAS,GAAG,GAAG,GAAGH,KAAK,CAAE,CAAC,CAAE;IACxC,IAAKL,oBAAoB,EAAG;MAC3B,IAAKA,oBAAoB,CAACW,QAAQ,CAAEN,KAAK,CAAE,CAAC,CAAG,CAAC,EAAG;QAClDG,SAAS,GAAGA,SAAS,GAAG,GAAG,GAAGH,KAAK,CAAE,CAAC,CAAE;MACzC;IACD;EACD;EAEA,OAAOG,SAAS;AACjB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAI,SAAUC,KAAK,EAAEC,UAAU,EAAG;EAC5C,MAAMf,mBAAmB,GAAGe,UAAU,CAACC,SAAS,CAAE,qBAAsB,CAAC;EACzE,MAAMf,oBAAoB,GAAGc,UAAU,CAACC,SAAS,CAAE,sBAAuB,CAAC;EAC3E,MAAMd,oBAAoB,GAAGa,UAAU,CAACE,SAAS,CAAE,sBAAuB,CAAC;EAC3E,MAAMC,iBAAiB,GAAGH,UAAU,CAACE,SAAS,CAAE,mBAAoB,CAAC;EAErE,IAAIE,IAAI,GAAGL,KAAK,CAACM,OAAO,CAAC,CAAC;EAC1BD,IAAI,GAAG,IAAAE,mBAAgB,EAAEF,IAAK,CAAC;EAC/BA,IAAI,GAAG,IAAAG,iCAAwB,EAAEH,IAAI,EAAEL,KAAK,CAACS,WAAW,IAAIT,KAAK,CAACS,WAAW,CAACC,UAAW,CAAC;;EAE1F;EACAL,IAAI,GAAGA,IAAI,CAACM,OAAO,CAAE,UAAU,EAAE,GAAI,CAAC;;EAEtC;EACAN,IAAI,GAAGA,IAAI,CAACM,OAAO,CAAE,+CAA+C,EAAE,EAAG,CAAC;;EAE1E;EACAN,IAAI,GAAGA,IAAI,CAACM,OAAO,CAAE,6BAA6B,EAAE,EAAG,CAAC;EAExD,IAAIrC,SAAS,GAAG,IAAAsC,qBAAY,EAAEP,IAAI,EAAED,iBAAkB,CAAC;EAEvD,IAAI/B,kBAAkB,GAAGC,SAAS,CAACuC,GAAG,CAAE,UAAU5B,QAAQ,EAAG;IAC5D,OAAOD,oBAAoB,CAAEC,QAAQ,EAAEC,mBAAmB,EAAEC,oBAAoB,EAAEC,oBAAqB,CAAC;EACzG,CAAE,CAAC;EAEHd,SAAS,GAAGA,SAAS,CAACwC,MAAM,CAAE,UAAU7B,QAAQ,EAAG;IAClD,MAAMI,QAAQ,GAAG,IAAAE,oBAAW,EAAEN,QAAS,CAAC;IACxC,MAAMO,KAAK,GAAGJ,oBAAoB,GAAGA,oBAAoB,CAAEC,QAAS,CAAC,GAAG,IAAAI,iBAAQ,EAAEJ,QAAS,CAAC;IAC5F,OAAOG,KAAK,CAACE,MAAM,GAAG,CAAC;EACxB,CAAE,CAAC;EACHrB,kBAAkB,GAAG,IAAAyC,cAAM,EAAEzC,kBAAmB,CAAC;EAEjD,OAAOD,iBAAiB,CAAEC,kBAAkB,EAAEC,SAAU,CAAC;AAC1D","ignoreList":[]}

package/build/languageProcessing/researches/getSubheadingTextLengths.js

@@ -0,0 +1,60 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _getSubheadingTexts = _interopRequireDefault(require("../helpers/html/getSubheadingTexts"));
+var _countWords = _interopRequireDefault(require("../helpers/word/countWords"));
+var _lodash = require("lodash");
+var _htmlParser = _interopRequireDefault(require("../helpers/html/htmlParser"));
+var _helpers = require("../helpers");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Gets the subheadings from the text and returns the length of these subheading in an array.
+ *
+ * @param {Paper}       paper       The Paper object to get the text from.
+ * @param {Researcher}  researcher  The researcher to use for analysis.
+ *
+ * @returns {Object} The object containing the array of found subheadings and the length of the text before the first subheading.
+ */
+function _default(paper, researcher) {
+  let text = paper.getText();
+  text = (0, _htmlParser.default)(text);
+  text = (0, _helpers.filterShortcodesFromHTML)(text, paper._attributes && paper._attributes.shortcodes);
+  const matches = (0, _getSubheadingTexts.default)(text);
+
+  // An optional custom helper to count length to use instead of countWords.
+  const customCountLength = researcher.getHelper("customCountLength");
+  const foundSubheadings = [];
+  (0, _lodash.forEach)(matches, function (match) {
+    foundSubheadings.push({
+      subheading: match.subheading,
+      text: match.text,
+      countLength: customCountLength ? customCountLength(match.text) : (0, _countWords.default)(match.text),
+      index: match.index
+    });
+  });
+  let textBeforeFirstSubheadingLength = 0;
+  let textBeforeFirstSubheading = "";
+  if (foundSubheadings.length > 0) {
+    // Find first subheading.
+    const firstSubheading = foundSubheadings[0];
+    // Retrieve text preceding first subheading.
+    textBeforeFirstSubheading = text.slice(0, firstSubheading.index);
+    textBeforeFirstSubheadingLength = customCountLength ? customCountLength(textBeforeFirstSubheading) : (0, _countWords.default)(textBeforeFirstSubheading);
+  }
+
+  // Check if there is a text before the first subheading.
+  if (textBeforeFirstSubheadingLength > 0 && textBeforeFirstSubheading !== "") {
+    // Also add the text before the first subheading to the array.
+    foundSubheadings.unshift({
+      // Assign an empty string for the subheading for text that comes before the first subheading.
+      subheading: "",
+      text: textBeforeFirstSubheading,
+      countLength: textBeforeFirstSubheadingLength
+    });
+  }
+  return foundSubheadings;
+}
+//# sourceMappingURL=getSubheadingTextLengths.js.map