axyseo 2.1.11 → 2.1.13

package/build/cjs/languageProcessing/languages/ja/customResearches/getWordForms.js

@@ -0,0 +1,57 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+var _parseSynonyms = _interopRequireDefault(require("../../../helpers/sanitize/parseSynonyms"));
+var _getContentWords = _interopRequireDefault(require("../helpers/getContentWords"));
+var _createWordForms = _interopRequireDefault(require("../helpers/internal/createWordForms"));
+var _doubleQuotes = _interopRequireDefault(require("../../../helpers/sanitize/doubleQuotes"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Creates word forms for each word in the given keyphrase.
+ *
+ * @param {string}      keyphrase   The keyphrase to generate word forms for.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {Array<string[]>} The word forms for each word in the keyphrase.
+ */
+function getKeyphraseForms(keyphrase, researcher) {
+  // The keyphrase is in double quotes: use it as an exact match keyphrase.
+  if (_doubleQuotes.default.includes(keyphrase[0]) && _doubleQuotes.default.includes(keyphrase[keyphrase.length - 1])) {
+    return [[keyphrase]];
+  }
+  const keyphraseWords = (0, _getContentWords.default)(keyphrase);
+
+  // If the keyphrase does not contain content words, return an empty list.
+  if (keyphraseWords.length === 0) {
+    return [[]];
+  }
+  const morphologyData = (0, _lodash.get)(researcher.getData("morphology"), "ja", false);
+  return keyphraseWords.map(word => morphologyData ? (0, _createWordForms.default)(word, morphologyData) : [word]);
+}
+
+/**
+ * Gets all matching word forms for the keyphrase and synonyms.
+ *
+ * @param {Paper}       paper       The paper.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {Object} Object with an array of keyphrase forms and an array of arrays of synonyms forms, based on the forms
+ * found in the text or created forms.
+ */
+function _default(paper, researcher) {
+  let keyphrase = paper.getKeyword().toLocaleLowerCase("ja").trim();
+  // Remove spaces from the keyphrase.
+  keyphrase = keyphrase.replace(/\s/g, "");
+  const synonyms = (0, _parseSynonyms.default)(paper.getSynonyms().toLocaleLowerCase("ja").trim());
+  const keyphraseForms = getKeyphraseForms(keyphrase, researcher);
+  const synonymsForms = synonyms.map(synonym => getKeyphraseForms(synonym, researcher));
+  return {
+    keyphraseForms,
+    synonymsForms
+  };
+}
+//# sourceMappingURL=getWordForms.js.map

package/build/cjs/languageProcessing/languages/ja/customResearches/getWordForms.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getWordForms.js","names":["_lodash","require","_parseSynonyms","_interopRequireDefault","_getContentWords","_createWordForms","_doubleQuotes","e","__esModule","default","getKeyphraseForms","keyphrase","researcher","doubleQuotes","includes","length","keyphraseWords","getContentWords","morphologyData","get","getData","map","word","createWordForms","_default","paper","getKeyword","toLocaleLowerCase","trim","replace","synonyms","parseSynonyms","getSynonyms","keyphraseForms","synonymsForms","synonym"],"sources":["../../../../../../src/languageProcessing/languages/ja/customResearches/getWordForms.js"],"sourcesContent":["import { get } from \"lodash\";\nimport parseSynonyms from \"../../../helpers/sanitize/parseSynonyms\";\nimport getContentWords from \"../helpers/getContentWords\";\nimport createWordForms from \"../helpers/internal/createWordForms\";\nimport doubleQuotes from \"../../../helpers/sanitize/doubleQuotes\";\n\n/**\n * Creates word forms for each word in the given keyphrase.\n *\n * @param {string}      keyphrase   The keyphrase to generate word forms for.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {Array<string[]>} The word forms for each word in the keyphrase.\n */\nfunction getKeyphraseForms( keyphrase, researcher ) {\n\t// The keyphrase is in double quotes: use it as an exact match keyphrase.\n\tif ( doubleQuotes.includes( keyphrase[ 0 ] ) && doubleQuotes.includes( keyphrase[ keyphrase.length - 1 ] ) ) {\n\t\treturn [ [ keyphrase ] ];\n\t}\n\n\tconst keyphraseWords = getContentWords( keyphrase );\n\n\t// If the keyphrase does not contain content words, return an empty list.\n\tif ( keyphraseWords.length === 0 ) {\n\t\treturn [ [] ];\n\t}\n\n\tconst morphologyData = get( researcher.getData( \"morphology\" ), \"ja\", false );\n\treturn keyphraseWords.map( word => morphologyData ? createWordForms( word, morphologyData ) : [ word ] );\n}\n\n/**\n * Gets all matching word forms for the keyphrase and synonyms.\n *\n * @param {Paper}       paper       The paper.\n * @param {Researcher}  researcher  The researcher.\n *\n * @returns {Object} Object with an array of keyphrase forms and an array of arrays of synonyms forms, based on the forms\n * found in the text or created forms.\n */\nexport default function( paper, researcher ) {\n\tlet keyphrase = paper.getKeyword().toLocaleLowerCase( \"ja\" ).trim();\n\t// Remove spaces from the keyphrase.\n\tkeyphrase = keyphrase.replace( /\\s/g, \"\" );\n\tconst synonyms = parseSynonyms( paper.getSynonyms().toLocaleLowerCase( \"ja\" ).trim() );\n\tconst keyphraseForms = getKeyphraseForms( keyphrase, researcher );\n\tconst synonymsForms = synonyms.map( synonym => getKeyphraseForms( synonym, researcher ) );\n\n\treturn { keyphraseForms, synonymsForms };\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,cAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,gBAAA,GAAAD,sBAAA,CAAAF,OAAA;AACA,IAAAI,gBAAA,GAAAF,sBAAA,CAAAF,OAAA;AACA,IAAAK,aAAA,GAAAH,sBAAA,CAAAF,OAAA;AAAkE,SAAAE,uBAAAI,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAElE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASG,iBAAiBA,CAAEC,SAAS,EAAEC,UAAU,EAAG;EACnD;EACA,IAAKC,qBAAY,CAACC,QAAQ,CAAEH,SAAS,CAAE,CAAC,CAAG,CAAC,IAAIE,qBAAY,CAACC,QAAQ,CAAEH,SAAS,CAAEA,SAAS,CAACI,MAAM,GAAG,CAAC,CAAG,CAAC,EAAG;IAC5G,OAAO,CAAE,CAAEJ,SAAS,CAAE,CAAE;EACzB;EAEA,MAAMK,cAAc,GAAG,IAAAC,wBAAe,EAAEN,SAAU,CAAC;;EAEnD;EACA,IAAKK,cAAc,CAACD,MAAM,KAAK,CAAC,EAAG;IAClC,OAAO,CAAE,EAAE,CAAE;EACd;EAEA,MAAMG,cAAc,GAAG,IAAAC,WAAG,EAAEP,UAAU,CAACQ,OAAO,CAAE,YAAa,CAAC,EAAE,IAAI,EAAE,KAAM,CAAC;EAC7E,OAAOJ,cAAc,CAACK,GAAG,CAAEC,IAAI,IAAIJ,cAAc,GAAG,IAAAK,wBAAe,EAAED,IAAI,EAAEJ,cAAe,CAAC,GAAG,CAAEI,IAAI,CAAG,CAAC;AACzG;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAE,SAAUC,KAAK,EAAEb,UAAU,EAAG;EAC5C,IAAID,SAAS,GAAGc,KAAK,CAACC,UAAU,CAAC,CAAC,CAACC,iBAAiB,CAAE,IAAK,CAAC,CAACC,IAAI,CAAC,CAAC;EACnE;EACAjB,SAAS,GAAGA,SAAS,CAACkB,OAAO,CAAE,KAAK,EAAE,EAAG,CAAC;EAC1C,MAAMC,QAAQ,GAAG,IAAAC,sBAAa,EAAEN,KAAK,CAACO,WAAW,CAAC,CAAC,CAACL,iBAAiB,CAAE,IAAK,CAAC,CAACC,IAAI,CAAC,CAAE,CAAC;EACtF,MAAMK,cAAc,GAAGvB,iBAAiB,CAAEC,SAAS,EAAEC,UAAW,CAAC;EACjE,MAAMsB,aAAa,GAAGJ,QAAQ,CAACT,GAAG,CAAEc,OAAO,IAAIzB,iBAAiB,CAAEyB,OAAO,EAAEvB,UAAW,CAAE,CAAC;EAEzF,OAAO;IAAEqB,cAAc;IAAEC;EAAc,CAAC;AACzC","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/customResearches/textLength.js

@@ -0,0 +1,32 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _countCharacters = _interopRequireDefault(require("../helpers/countCharacters"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * A result of the character count calculation.
+ *
+ * @typedef CharacterCountResult
+ * @param {number} count The number of characters found in the text.
+ * @param {string} unit The unit used in the text length calculations, always "character".
+ */
+
+/**
+ * Count the characters in the text.
+ *
+ * @param {Paper} paper The Paper object.
+ *
+ * @returns {CharacterCountResult} The number of characters found in the text, plus "character" as the unit used in calculating the text length.
+ */
+function _default(paper) {
+  const text = paper?.getText?.() || '';
+  return {
+    text,
+    count: (0, _countCharacters.default)(text),
+    unit: "character"
+  };
+}
+//# sourceMappingURL=textLength.js.map

package/build/cjs/languageProcessing/languages/ja/customResearches/textLength.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textLength.js","names":["_countCharacters","_interopRequireDefault","require","e","__esModule","default","_default","paper","text","getText","count","countCharacters","unit"],"sources":["../../../../../../src/languageProcessing/languages/ja/customResearches/textLength.js"],"sourcesContent":["import countCharacters from \"../helpers/countCharacters\";\n\n/**\n * A result of the character count calculation.\n *\n * @typedef CharacterCountResult\n * @param {number} count The number of characters found in the text.\n * @param {string} unit The unit used in the text length calculations, always \"character\".\n */\n\n/**\n * Count the characters in the text.\n *\n * @param {Paper} paper The Paper object.\n *\n * @returns {CharacterCountResult} The number of characters found in the text, plus \"character\" as the unit used in calculating the text length.\n */\nexport default function( paper ) {\n  const text = paper?.getText?.() || '';\n\treturn {\n\t\ttext,\n\t\tcount: countCharacters(text),\n\t\tunit: \"character\",\n\t};\n}\n"],"mappings":";;;;;;AAAA,IAAAA,gBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAyD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEzD;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,KAAK,EAAG;EAC/B,MAAMC,IAAI,GAAGD,KAAK,EAAEE,OAAO,GAAG,CAAC,IAAI,EAAE;EACtC,OAAO;IACND,IAAI;IACJE,KAAK,EAAE,IAAAC,wBAAe,EAACH,IAAI,CAAC;IAC5BI,IAAI,EAAE;EACP,CAAC;AACF","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/countCharacters.js

@@ -0,0 +1,27 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _index = require("../../../../index.js");
+var _removeURLs = _interopRequireDefault(require("../../../helpers/sanitize/removeURLs.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  sanitizeString
+} = _index.languageProcessing;
+/**
+ * Calculates the character count which serves as a measure of text length.
+ * The character count includes letters, punctuation, and numbers. It doesn't include URLs, HTML tags, and spaces.
+ *
+ * @param {string} text The text to be counted.
+ *
+ * @returns {number} The character count of the given text.
+ */
+function _default(text) {
+  text = (0, _removeURLs.default)(text);
+  text = sanitizeString(text);
+  text = text.replace(/\s/g, '');
+  return text.length;
+}
+//# sourceMappingURL=countCharacters.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/countCharacters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"countCharacters.js","names":["_index","require","_removeURLs","_interopRequireDefault","e","__esModule","default","sanitizeString","languageProcessing","_default","text","removeURLs","replace","length"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/countCharacters.js"],"sourcesContent":["import {languageProcessing} from '@axyseo/index.js';\nconst {sanitizeString} = languageProcessing;\nimport removeURLs from '../../../helpers/sanitize/removeURLs.js';\n\n/**\n * Calculates the character count which serves as a measure of text length.\n * The character count includes letters, punctuation, and numbers. It doesn't include URLs, HTML tags, and spaces.\n *\n * @param {string} text The text to be counted.\n *\n * @returns {number} The character count of the given text.\n */\nexport default function(text) {\n  text = removeURLs(text);\n  text = sanitizeString(text);\n  text = text.replace(/\\s/g, '');\n\n  return text.length;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAEA,IAAAC,WAAA,GAAAC,sBAAA,CAAAF,OAAA;AAAiE,SAAAE,uBAAAC,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AADjE,MAAM;EAACG;AAAc,CAAC,GAAGC,yBAAkB;AAG3C;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAASC,IAAI,EAAE;EAC5BA,IAAI,GAAG,IAAAC,mBAAU,EAACD,IAAI,CAAC;EACvBA,IAAI,GAAGH,cAAc,CAACG,IAAI,CAAC;EAC3BA,IAAI,GAAGA,IAAI,CAACE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;EAE9B,OAAOF,IAAI,CAACG,MAAM;AACpB","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/customGetStemmer.js

@@ -0,0 +1,28 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = getStemmer;
+var _lodash = require("lodash");
+var _index = require("../../../../index.js");
+var _determineStem = _interopRequireDefault(require("./internal/determineStem"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  baseStemmer
+} = _index.languageProcessing;
+/**
+ * Returns the stemmer for a researcher. This helper is currently only used for Prominent Words functionality.
+ *
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Function} The stemmer.
+ */
+function getStemmer(researcher) {
+  const morphologyData = (0, _lodash.get)(researcher.getData('morphology'), 'ja', false);
+  if (morphologyData) {
+    return word => (0, _determineStem.default)(word, morphologyData);
+  }
+  return baseStemmer;
+}
+//# sourceMappingURL=customGetStemmer.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/customGetStemmer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"customGetStemmer.js","names":["_lodash","require","_index","_determineStem","_interopRequireDefault","e","__esModule","default","baseStemmer","languageProcessing","getStemmer","researcher","morphologyData","get","getData","word","determineStem"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/customGetStemmer.js"],"sourcesContent":["import {get} from 'lodash';\nimport {languageProcessing} from '@axyseo/index.js';\nconst {baseStemmer} = languageProcessing;\nimport determineStem from './internal/determineStem';\n\n/**\n * Returns the stemmer for a researcher. This helper is currently only used for Prominent Words functionality.\n *\n * @param {Researcher} researcher The researcher.\n *\n * @returns {Function} The stemmer.\n */\nexport default function getStemmer(researcher) {\n  const morphologyData = get(researcher.getData('morphology'), 'ja', false);\n\n  if (morphologyData) {\n    return word => determineStem(word, morphologyData);\n  }\n\n  return baseStemmer;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAD,OAAA;AAEA,IAAAE,cAAA,GAAAC,sBAAA,CAAAH,OAAA;AAAqD,SAAAG,uBAAAC,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AADrD,MAAM;EAACG;AAAW,CAAC,GAAGC,yBAAkB;AAGxC;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASC,UAAUA,CAACC,UAAU,EAAE;EAC7C,MAAMC,cAAc,GAAG,IAAAC,WAAG,EAACF,UAAU,CAACG,OAAO,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC;EAEzE,IAAIF,cAAc,EAAE;IAClB,OAAOG,IAAI,IAAI,IAAAC,sBAAa,EAACD,IAAI,EAAEH,cAAc,CAAC;EACpD;EAEA,OAAOJ,WAAW;AACpB","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/getContentWords.js

@@ -0,0 +1,27 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _getWords = _interopRequireDefault(require("./getWords"));
+var _functionWords = _interopRequireDefault(require("../config/functionWords"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Filters out function words and removes じゃ ending if a words ends in it.
+ *
+ * @param {string} text The input text.
+ *
+ * @returns {Array} The array words with function words filtered out and じゃ removed.
+ */
+function _default(text) {
+  let words = (0, _getWords.default)(text);
+  // Filter function words and morphemes.
+  words = words.filter(word => !_functionWords.default.includes(word));
+
+  // Check if the segment ends in -じゃ, and remove the ending if it does.
+  const ending = "じゃ";
+  words = words.map(word => word.endsWith(ending) ? word.slice(0, -ending.length) : word);
+  return words;
+}
+//# sourceMappingURL=getContentWords.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/getContentWords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getContentWords.js","names":["_getWords","_interopRequireDefault","require","_functionWords","e","__esModule","default","_default","text","words","getWords","filter","word","functionWords","includes","ending","map","endsWith","slice","length"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/getContentWords.js"],"sourcesContent":["import getWords from \"./getWords\";\nimport functionWords from \"../config/functionWords\";\n\n/**\n * Filters out function words and removes じゃ ending if a words ends in it.\n *\n * @param {string} text The input text.\n *\n * @returns {Array} The array words with function words filtered out and じゃ removed.\n */\nexport default function( text ) {\n\tlet words = getWords( text );\n\t// Filter function words and morphemes.\n\twords = words.filter( word => ! functionWords.includes( word ) );\n\n\t// Check if the segment ends in -じゃ, and remove the ending if it does.\n\tconst ending = \"じゃ\";\n\twords = words.map( word => word.endsWith( ending ) ? word.slice( 0, -ending.length ) : word );\n\n\treturn words;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,SAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,cAAA,GAAAF,sBAAA,CAAAC,OAAA;AAAoD,SAAAD,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEpD;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,IAAI,EAAG;EAC/B,IAAIC,KAAK,GAAG,IAAAC,iBAAQ,EAAEF,IAAK,CAAC;EAC5B;EACAC,KAAK,GAAGA,KAAK,CAACE,MAAM,CAAEC,IAAI,IAAI,CAAEC,sBAAa,CAACC,QAAQ,CAAEF,IAAK,CAAE,CAAC;;EAEhE;EACA,MAAMG,MAAM,GAAG,IAAI;EACnBN,KAAK,GAAGA,KAAK,CAACO,GAAG,CAAEJ,IAAI,IAAIA,IAAI,CAACK,QAAQ,CAAEF,MAAO,CAAC,GAAGH,IAAI,CAACM,KAAK,CAAE,CAAC,EAAE,CAACH,MAAM,CAACI,MAAO,CAAC,GAAGP,IAAK,CAAC;EAE7F,OAAOH,KAAK;AACb","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/getWords.js

@@ -0,0 +1,38 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+var _index = require("../../../../index.js");
+var _tinySegmenter = _interopRequireDefault(require("tiny-segmenter"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  sanitizeString,
+  removePunctuation
+} = _index.languageProcessing;
+const segmenter = new _tinySegmenter.default();
+
+/**
+ * Returns an array with words used in the text.
+ *
+ * @param {string} text The text to be counted.
+ *
+ * @returns {Array} The array with all words.
+ */
+function _default(text) {
+  // Strips HTML tags.
+  text = sanitizeString(text);
+  if (text === '') {
+    return [];
+  }
+  let words = segmenter.segment(text);
+  words = (0, _lodash.map)(words, function (word) {
+    return removePunctuation(word);
+  });
+  return (0, _lodash.filter)(words, function (word) {
+    return word.trim() !== '';
+  });
+}
+//# sourceMappingURL=getWords.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/getWords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getWords.js","names":["_lodash","require","_index","_tinySegmenter","_interopRequireDefault","e","__esModule","default","sanitizeString","removePunctuation","languageProcessing","segmenter","TinySegmenter","_default","text","words","segment","map","word","filter","trim"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/getWords.js"],"sourcesContent":["import {filter, map} from 'lodash';\nimport {languageProcessing} from '@axyseo/index.js';\nconst {sanitizeString, removePunctuation} = languageProcessing;\nimport TinySegmenter from 'tiny-segmenter';\n\nconst segmenter = new TinySegmenter();\n\n/**\n * Returns an array with words used in the text.\n *\n * @param {string} text The text to be counted.\n *\n * @returns {Array} The array with all words.\n */\nexport default function(text) {\n  // Strips HTML tags.\n  text = sanitizeString(text);\n  if (text === '') {\n    return [];\n  }\n\n  let words = segmenter.segment(text);\n\n  words = map(words, function(word) {\n    return removePunctuation(word);\n  });\n\n  return filter(words, function(word) {\n    return word.trim() !== '';\n  });\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAD,OAAA;AAEA,IAAAE,cAAA,GAAAC,sBAAA,CAAAH,OAAA;AAA2C,SAAAG,uBAAAC,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAD3C,MAAM;EAACG,cAAc;EAAEC;AAAiB,CAAC,GAAGC,yBAAkB;AAG9D,MAAMC,SAAS,GAAG,IAAIC,sBAAa,CAAC,CAAC;;AAErC;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAC,SAASC,IAAI,EAAE;EAC5B;EACAA,IAAI,GAAGN,cAAc,CAACM,IAAI,CAAC;EAC3B,IAAIA,IAAI,KAAK,EAAE,EAAE;IACf,OAAO,EAAE;EACX;EAEA,IAAIC,KAAK,GAAGJ,SAAS,CAACK,OAAO,CAACF,IAAI,CAAC;EAEnCC,KAAK,GAAG,IAAAE,WAAG,EAACF,KAAK,EAAE,UAASG,IAAI,EAAE;IAChC,OAAOT,iBAAiB,CAACS,IAAI,CAAC;EAChC,CAAC,CAAC;EAEF,OAAO,IAAAC,cAAM,EAACJ,KAAK,EAAE,UAASG,IAAI,EAAE;IAClC,OAAOA,IAAI,CAACE,IAAI,CAAC,CAAC,KAAK,EAAE;EAC3B,CAAC,CAAC;AACJ","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/internal/SentenceTokenizer.js

@@ -0,0 +1,99 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _quotes = require("../../../../helpers/sanitize/quotes.js");
+var _SentenceTokenizer = _interopRequireDefault(require("../../../../helpers/sentence/SentenceTokenizer"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/*
+ * \u2026 - ellipsis.
+ * \u3002 - Japanese ideographic full stop.
+ * \uFF61 - Japanese half-width ideographic full stop.
+ * \uFF01 - Japanese full-width exclamation mark.
+ * \u203C - Japanese double exclamation mark.
+ * \uFF1F - Japanese fullwidth question mark.
+ * \u2047 - Japanese double question mark.
+ * \u2049 - Japanese exclamation question mark.
+ * \u2048 - Japanese question exclamation mark.
+ * \u2025 - Japanese two dot leader.
+ */
+const sentenceDelimiters = "?!\u2026\u3002\uFF61\uFF01\u203C\uFF1F\u2047\u2049\u2048\u2049\u2025";
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+class JapaneseSentenceTokenizer extends _SentenceTokenizer.default {
+  /**
+   * Constructor
+   * @constructor
+   */
+  constructor() {
+    super();
+    this.sentenceDelimiters = sentenceDelimiters;
+  }
+
+  /**
+   * Returns whether a certain character is a number.
+   *
+   * @param {string} character The character to check.
+   * @returns {boolean} Whether or not the character is a capital letter.
+   */
+  isNumber(character) {
+    const japaneseNumbers = [
+    // Full-width.
+    /^[\uFF10-\uFF19]+$/i,
+    // Circled digit, parenthesized digit, and digit with full stop.
+    /^[\u2460-\u249B]+$/i,
+    // Parenthesized ideograph.
+    /^[\u3220-\u3229]+$/i,
+    // Circled ideograph.
+    /^[\u3280-\u3289]+$/i];
+    return !(0, _lodash.isNaN)(parseInt(character, 10)) || japaneseNumbers.some(numberRange => numberRange.test(character));
+  }
+
+  /**
+   * Returns whether a given character is quotation mark.
+   *
+   * @param {string} character The character to check.
+   * @returns {boolean} Whether or not the given character is a quotation mark.
+   */
+  isQuotation(character) {
+    character = (0, _quotes.normalize)(character);
+    const japaneseOpeningQuotationMark = /^[\u300C\u300E\u3008\u3014\u3010\uFF5B\uFF3B]+$/i;
+    return "'" === character || "\"" === character || japaneseOpeningQuotationMark.test(character);
+  }
+
+  /**
+   * Checks whether a character is from Japanese language that could be sentence beginning.
+   *
+   * @param {string} letter The letter to check.
+   *
+   * @returns {boolean} Whether the letter is from Japanese language that could be sentence beginning.
+   */
+  isLetterFromSpecificLanguage(letter) {
+    const japaneseLetterRanges = [
+    // Hiragana.
+    /^[\u3040-\u3096]+$/i,
+    // Katakana full-width.
+    /^[\u30A1-\u30FA]+$/i, /^[\u31F0-\u31FF]+$/i,
+    // Katakana half-width.
+    /^[\uFF66-\uFF9D]+$/i,
+    // Kanji.
+    /^[\u4E00-\u9FFC]+$/i];
+    return japaneseLetterRanges.some(ltrLetterRange => ltrLetterRange.test(letter));
+  }
+
+  /**
+   * Always returns true as Japanese sentence beginning doesn't need to be preceded by a whitespace to be a valid one.
+   *
+   * @returns {true}  Always true.
+   */
+  isCharacterASpace() {
+    return true;
+  }
+}
+exports.default = JapaneseSentenceTokenizer;
+//# sourceMappingURL=SentenceTokenizer.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/internal/SentenceTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTokenizer.js","names":["_lodash","require","_quotes","_SentenceTokenizer","_interopRequireDefault","e","__esModule","default","sentenceDelimiters","JapaneseSentenceTokenizer","SentenceTokenizer","constructor","isNumber","character","japaneseNumbers","isNaN","parseInt","some","numberRange","test","isQuotation","normalizeQuotes","japaneseOpeningQuotationMark","isLetterFromSpecificLanguage","letter","japaneseLetterRanges","ltrLetterRange","isCharacterASpace","exports"],"sources":["../../../../../../../src/languageProcessing/languages/ja/helpers/internal/SentenceTokenizer.js"],"sourcesContent":["import { isNaN } from \"lodash\";\n\nimport { normalize as normalizeQuotes } from \"../../../../helpers/sanitize/quotes.js\";\nimport SentenceTokenizer from \"../../../../helpers/sentence/SentenceTokenizer\";\n\n/*\n * \\u2026 - ellipsis.\n * \\u3002 - Japanese ideographic full stop.\n * \\uFF61 - Japanese half-width ideographic full stop.\n * \\uFF01 - Japanese full-width exclamation mark.\n * \\u203C - Japanese double exclamation mark.\n * \\uFF1F - Japanese fullwidth question mark.\n * \\u2047 - Japanese double question mark.\n * \\u2049 - Japanese exclamation question mark.\n * \\u2048 - Japanese question exclamation mark.\n * \\u2025 - Japanese two dot leader.\n */\nconst sentenceDelimiters = \"?!\\u2026\\u3002\\uFF61\\uFF01\\u203C\\uFF1F\\u2047\\u2049\\u2048\\u2049\\u2025\";\n\n/**\n * Class for tokenizing a (html) text into sentences.\n */\nexport default class JapaneseSentenceTokenizer extends SentenceTokenizer {\n\t/**\n\t * Constructor\n\t * @constructor\n\t */\n\tconstructor() {\n\t\tsuper();\n\t\tthis.sentenceDelimiters = sentenceDelimiters;\n\t}\n\n\t/**\n\t * Returns whether a certain character is a number.\n\t *\n\t * @param {string} character The character to check.\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisNumber( character ) {\n\t\tconst japaneseNumbers = [\n\t\t\t// Full-width.\n\t\t\t/^[\\uFF10-\\uFF19]+$/i,\n\t\t\t// Circled digit, parenthesized digit, and digit with full stop.\n\t\t\t/^[\\u2460-\\u249B]+$/i,\n\t\t\t// Parenthesized ideograph.\n\t\t\t/^[\\u3220-\\u3229]+$/i,\n\t\t\t// Circled ideograph.\n\t\t\t/^[\\u3280-\\u3289]+$/i,\n\t\t];\n\n\t\treturn ( ! isNaN( parseInt( character, 10 ) ) || japaneseNumbers.some( numberRange => numberRange.test( character ) ) );\n\t}\n\n\t/**\n\t * Returns whether a given character is quotation mark.\n\t *\n\t * @param {string} character The character to check.\n\t * @returns {boolean} Whether or not the given character is a quotation mark.\n\t */\n\tisQuotation( character ) {\n\t\tcharacter = normalizeQuotes( character );\n\n\t\tconst japaneseOpeningQuotationMark = /^[\\u300C\\u300E\\u3008\\u3014\\u3010\\uFF5B\\uFF3B]+$/i;\n\n\t\treturn \"'\" === character || \"\\\"\" === character || japaneseOpeningQuotationMark.test( character );\n\t}\n\n\t/**\n\t * Checks whether a character is from Japanese language that could be sentence beginning.\n\t *\n\t * @param {string} letter The letter to check.\n\t *\n\t * @returns {boolean} Whether the letter is from Japanese language that could be sentence beginning.\n\t */\n\tisLetterFromSpecificLanguage( letter ) {\n\t\tconst japaneseLetterRanges = [\n\t\t\t// Hiragana.\n\t\t\t/^[\\u3040-\\u3096]+$/i,\n\t\t\t// Katakana full-width.\n\t\t\t/^[\\u30A1-\\u30FA]+$/i,\n\t\t\t/^[\\u31F0-\\u31FF]+$/i,\n\t\t\t// Katakana half-width.\n\t\t\t/^[\\uFF66-\\uFF9D]+$/i,\n\t\t\t// Kanji.\n\t\t\t/^[\\u4E00-\\u9FFC]+$/i,\n\t\t];\n\n\t\treturn (\n\t\t\tjapaneseLetterRanges.some( ltrLetterRange => ltrLetterRange.test( letter ) )\n\t\t);\n\t}\n\n\t/**\n\t * Always returns true as Japanese sentence beginning doesn't need to be preceded by a whitespace to be a valid one.\n\t *\n\t * @returns {true}  Always true.\n\t */\n\tisCharacterASpace() {\n\t\treturn true;\n\t}\n}\n\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA,IAAAC,OAAA,GAAAD,OAAA;AACA,IAAAE,kBAAA,GAAAC,sBAAA,CAAAH,OAAA;AAA+E,SAAAG,uBAAAC,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE/E;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,kBAAkB,GAAG,sEAAsE;;AAEjG;AACA;AACA;AACe,MAAMC,yBAAyB,SAASC,0BAAiB,CAAC;EACxE;AACD;AACA;AACA;EACCC,WAAWA,CAAA,EAAG;IACb,KAAK,CAAC,CAAC;IACP,IAAI,CAACH,kBAAkB,GAAGA,kBAAkB;EAC7C;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCI,QAAQA,CAAEC,SAAS,EAAG;IACrB,MAAMC,eAAe,GAAG;IACvB;IACA,qBAAqB;IACrB;IACA,qBAAqB;IACrB;IACA,qBAAqB;IACrB;IACA,qBAAqB,CACrB;IAED,OAAS,CAAE,IAAAC,aAAK,EAAEC,QAAQ,CAAEH,SAAS,EAAE,EAAG,CAAE,CAAC,IAAIC,eAAe,CAACG,IAAI,CAAEC,WAAW,IAAIA,WAAW,CAACC,IAAI,CAAEN,SAAU,CAAE,CAAC;EACtH;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCO,WAAWA,CAAEP,SAAS,EAAG;IACxBA,SAAS,GAAG,IAAAQ,iBAAe,EAAER,SAAU,CAAC;IAExC,MAAMS,4BAA4B,GAAG,kDAAkD;IAEvF,OAAO,GAAG,KAAKT,SAAS,IAAI,IAAI,KAAKA,SAAS,IAAIS,4BAA4B,CAACH,IAAI,CAAEN,SAAU,CAAC;EACjG;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCU,4BAA4BA,CAAEC,MAAM,EAAG;IACtC,MAAMC,oBAAoB,GAAG;IAC5B;IACA,qBAAqB;IACrB;IACA,qBAAqB,EACrB,qBAAqB;IACrB;IACA,qBAAqB;IACrB;IACA,qBAAqB,CACrB;IAED,OACCA,oBAAoB,CAACR,IAAI,CAAES,cAAc,IAAIA,cAAc,CAACP,IAAI,CAAEK,MAAO,CAAE,CAAC;EAE9E;;EAEA;AACD;AACA;AACA;AACA;EACCG,iBAAiBA,CAAA,EAAG;IACnB,OAAO,IAAI;EACZ;AACD;AAACC,OAAA,CAAArB,OAAA,GAAAE,yBAAA","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/internal/createWordForms.js

@@ -0,0 +1,70 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _lodash = require("lodash");
+/**
+ * Checks if the word matches any paradigm and creates the forms from the matched paradigm(s).
+ * If no matches found, returns an array with the original word.
+ *
+ * @param {string}  word            The word to check.
+ * @param {Object}  morphologyData  The morphology data.
+ *
+ * @returns {Array} The array of created forms.
+ */
+function createForms(word, morphologyData) {
+  const paradigmEndingsGroups = morphologyData.paradigmGroups;
+
+  // The endings in the array are sorted from the longest to the shortest.
+  // This way, if there are multiple matches, the first element in the matched array is always the longest one.
+  let allEndings = (0, _lodash.uniq)((0, _lodash.flatten)(paradigmEndingsGroups));
+  allEndings = allEndings.sort((a, b) => b.length - a.length || a.localeCompare(b));
+
+  // Check if the word matches any ending(s) and save the ending(s).
+  const matchedEndings = allEndings.filter(ending => word.endsWith(ending));
+  const forms = [];
+  if (matchedEndings.length === 0) {
+    // If there is no match found, add the original word to the forms array.
+    forms.push(word);
+  } else {
+    // Pick the longest ending.
+    const matchedEnding = matchedEndings[0];
+
+    // Extract the stem.
+    const matchedStem = word.slice(0, -matchedEnding.length);
+
+    // Loop over each endings group in the endings group array.
+    for (const endingsGroup of paradigmEndingsGroups) {
+      // Check if the ending of the matched word can be found in the endings group and create all the forms with all the endings in the group.
+      if (endingsGroup.includes(matchedEnding)) {
+        forms.push(endingsGroup.map(ending => matchedStem + ending));
+      }
+    }
+  }
+
+  // If the final character of the input word ends in る, also create an extra form where the character is removed.
+  const ruEnding = "る";
+  if (word.endsWith(ruEnding)) {
+    forms.push(word.slice(0, -ruEnding.length));
+  }
+  return (0, _lodash.uniq)((0, _lodash.flatten)(forms));
+}
+
+/**
+ * Creates forms for Japanese word.
+ *
+ * @param {string}  word            The word to check. Assume that the input word is a segment as outputted from `getContentWords` helper.
+ * @param {Object}  morphologyData  The morphology data.
+ *
+ * @returns {Array} The array of created forms.
+ */
+function _default(word, morphologyData) {
+  // Check if the word is longer than 1 character.
+  if (word.length <= 1) {
+    return [word];
+  }
+  return createForms(word, morphologyData);
+}
+//# sourceMappingURL=createWordForms.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/internal/createWordForms.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createWordForms.js","names":["_lodash","require","createForms","word","morphologyData","paradigmEndingsGroups","paradigmGroups","allEndings","uniq","flatten","sort","a","b","length","localeCompare","matchedEndings","filter","ending","endsWith","forms","push","matchedEnding","matchedStem","slice","endingsGroup","includes","map","ruEnding","_default"],"sources":["../../../../../../../src/languageProcessing/languages/ja/helpers/internal/createWordForms.js"],"sourcesContent":["import { flatten, uniq } from \"lodash\";\n\n/**\n * Checks if the word matches any paradigm and creates the forms from the matched paradigm(s).\n * If no matches found, returns an array with the original word.\n *\n * @param {string}  word            The word to check.\n * @param {Object}  morphologyData  The morphology data.\n *\n * @returns {Array} The array of created forms.\n */\nfunction createForms( word, morphologyData ) {\n\tconst paradigmEndingsGroups = morphologyData.paradigmGroups;\n\n\t// The endings in the array are sorted from the longest to the shortest.\n\t// This way, if there are multiple matches, the first element in the matched array is always the longest one.\n\tlet allEndings = uniq( flatten( paradigmEndingsGroups ) );\n\tallEndings = allEndings.sort( ( a, b ) => b.length - a.length || a.localeCompare( b ) );\n\n\t// Check if the word matches any ending(s) and save the ending(s).\n\tconst matchedEndings = allEndings.filter( ending => word.endsWith( ending ) );\n\n\tconst forms = [];\n\n\tif ( matchedEndings.length === 0 ) {\n\t\t// If there is no match found, add the original word to the forms array.\n\t\tforms.push( word );\n\t} else {\n\t\t// Pick the longest ending.\n\t\tconst matchedEnding = matchedEndings[ 0 ];\n\n\t\t// Extract the stem.\n\t\tconst matchedStem = word.slice( 0, -matchedEnding.length );\n\n\t\t// Loop over each endings group in the endings group array.\n\t\tfor ( const endingsGroup of paradigmEndingsGroups ) {\n\t\t\t// Check if the ending of the matched word can be found in the endings group and create all the forms with all the endings in the group.\n\t\t\tif ( endingsGroup.includes( matchedEnding ) ) {\n\t\t\t\tforms.push( endingsGroup.map( ending => matchedStem + ending ) );\n\t\t\t}\n\t\t}\n\t}\n\n\t// If the final character of the input word ends in る, also create an extra form where the character is removed.\n\tconst ruEnding = \"る\";\n\tif ( word.endsWith( ruEnding ) ) {\n\t\tforms.push( word.slice( 0, -ruEnding.length ) );\n\t}\n\n\treturn uniq( flatten( forms ) );\n}\n\n/**\n * Creates forms for Japanese word.\n *\n * @param {string}  word            The word to check. Assume that the input word is a segment as outputted from `getContentWords` helper.\n * @param {Object}  morphologyData  The morphology data.\n *\n * @returns {Array} The array of created forms.\n */\nexport default function( word, morphologyData ) {\n\t// Check if the word is longer than 1 character.\n\tif ( word.length <= 1 ) {\n\t\treturn [ word ];\n\t}\n\n\treturn createForms( word, morphologyData );\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,WAAWA,CAAEC,IAAI,EAAEC,cAAc,EAAG;EAC5C,MAAMC,qBAAqB,GAAGD,cAAc,CAACE,cAAc;;EAE3D;EACA;EACA,IAAIC,UAAU,GAAG,IAAAC,YAAI,EAAE,IAAAC,eAAO,EAAEJ,qBAAsB,CAAE,CAAC;EACzDE,UAAU,GAAGA,UAAU,CAACG,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMA,CAAC,CAACC,MAAM,GAAGF,CAAC,CAACE,MAAM,IAAIF,CAAC,CAACG,aAAa,CAAEF,CAAE,CAAE,CAAC;;EAEvF;EACA,MAAMG,cAAc,GAAGR,UAAU,CAACS,MAAM,CAAEC,MAAM,IAAId,IAAI,CAACe,QAAQ,CAAED,MAAO,CAAE,CAAC;EAE7E,MAAME,KAAK,GAAG,EAAE;EAEhB,IAAKJ,cAAc,CAACF,MAAM,KAAK,CAAC,EAAG;IAClC;IACAM,KAAK,CAACC,IAAI,CAAEjB,IAAK,CAAC;EACnB,CAAC,MAAM;IACN;IACA,MAAMkB,aAAa,GAAGN,cAAc,CAAE,CAAC,CAAE;;IAEzC;IACA,MAAMO,WAAW,GAAGnB,IAAI,CAACoB,KAAK,CAAE,CAAC,EAAE,CAACF,aAAa,CAACR,MAAO,CAAC;;IAE1D;IACA,KAAM,MAAMW,YAAY,IAAInB,qBAAqB,EAAG;MACnD;MACA,IAAKmB,YAAY,CAACC,QAAQ,CAAEJ,aAAc,CAAC,EAAG;QAC7CF,KAAK,CAACC,IAAI,CAAEI,YAAY,CAACE,GAAG,CAAET,MAAM,IAAIK,WAAW,GAAGL,MAAO,CAAE,CAAC;MACjE;IACD;EACD;;EAEA;EACA,MAAMU,QAAQ,GAAG,GAAG;EACpB,IAAKxB,IAAI,CAACe,QAAQ,CAAES,QAAS,CAAC,EAAG;IAChCR,KAAK,CAACC,IAAI,CAAEjB,IAAI,CAACoB,KAAK,CAAE,CAAC,EAAE,CAACI,QAAQ,CAACd,MAAO,CAAE,CAAC;EAChD;EAEA,OAAO,IAAAL,YAAI,EAAE,IAAAC,eAAO,EAAEU,KAAM,CAAE,CAAC;AAChC;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAS,SAAUzB,IAAI,EAAEC,cAAc,EAAG;EAC/C;EACA,IAAKD,IAAI,CAACU,MAAM,IAAI,CAAC,EAAG;IACvB,OAAO,CAAEV,IAAI,CAAE;EAChB;EAEA,OAAOD,WAAW,CAAEC,IAAI,EAAEC,cAAe,CAAC;AAC3C","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/internal/determineStem.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = determineStem;
+var _createWordForms = _interopRequireDefault(require("./createWordForms"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Determines the canonical stem from the word forms.
+ *
+ * @param {string} word             The word input.
+ * @param {object} morphologyData   The morphology data file.
+ *
+ * @returns {string}    The shortest form from the array as the canonical stem.
+ */
+function determineStem(word, morphologyData) {
+  let createdWordForms = (0, _createWordForms.default)(word, morphologyData);
+  // Sort from the shortest to the longest form.
+  createdWordForms = createdWordForms.sort((a, b) => a.length - b.length || a.localeCompare(b));
+  return createdWordForms[0];
+}
+//# sourceMappingURL=determineStem.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/internal/determineStem.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"determineStem.js","names":["_createWordForms","_interopRequireDefault","require","e","__esModule","default","determineStem","word","morphologyData","createdWordForms","createWordForms","sort","a","b","length","localeCompare"],"sources":["../../../../../../../src/languageProcessing/languages/ja/helpers/internal/determineStem.js"],"sourcesContent":["import createWordForms from \"./createWordForms\";\n\n/**\n * Determines the canonical stem from the word forms.\n *\n * @param {string} word             The word input.\n * @param {object} morphologyData   The morphology data file.\n *\n * @returns {string}    The shortest form from the array as the canonical stem.\n */\nexport default function determineStem( word, morphologyData ) {\n\tlet createdWordForms = createWordForms( word, morphologyData );\n\t// Sort from the shortest to the longest form.\n\tcreatedWordForms = createdWordForms.sort( ( a, b ) => a.length - b.length || a.localeCompare( b ) );\n\n\treturn createdWordForms[ 0 ];\n}\n"],"mappings":";;;;;;AAAA,IAAAA,gBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAgD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEhD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASG,aAAaA,CAAEC,IAAI,EAAEC,cAAc,EAAG;EAC7D,IAAIC,gBAAgB,GAAG,IAAAC,wBAAe,EAAEH,IAAI,EAAEC,cAAe,CAAC;EAC9D;EACAC,gBAAgB,GAAGA,gBAAgB,CAACE,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMD,CAAC,CAACE,MAAM,GAAGD,CAAC,CAACC,MAAM,IAAIF,CAAC,CAACG,aAAa,CAAEF,CAAE,CAAE,CAAC;EAEnG,OAAOJ,gBAAgB,CAAE,CAAC,CAAE;AAC7B","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/matchTextWithWord.js

@@ -0,0 +1,58 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _getContentWords = _interopRequireDefault(require("./getContentWords"));
+var _processExactMatchRequest = _interopRequireDefault(require("../../../helpers/match/processExactMatchRequest"));
+var _quotes = require("../../../helpers/sanitize/quotes");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Checks for word matches in a text and returns an array containing the matched word(s).
+ *
+ * @param {string}  text               The text to find the word to match.
+ * @param {string}  wordToMatch        The word to match.
+ *
+ * @returns {Array} An array of the matched word(s).
+ */
+function _default(text, wordToMatch) {
+  /*
+      * Lowercase the text so that it matches the wordToMatch which is already lowercased.
+      * Note that Japanese doesn't differentiate between upper and lower case, so this is only needed in case
+      * the text contains non-Japanese characters.
+      */
+  text = text.toLowerCase();
+
+  // Check if the exact match is requested.
+  const isExactMatchRequested = (0, _processExactMatchRequest.default)(wordToMatch);
+  if (isExactMatchRequested.exactMatchRequested) {
+    /*
+     * Normalize single quotes in case they differ between the text and the word to match.
+     * Normalizing is only needed for exact matching, because with non-exact matching single quotes are considered
+     * word boundaries.
+     * Quotes in wordToMatch are already normalized at an earlier point.
+    */
+    text = (0, _quotes.normalizeSingle)(text);
+    const keyphrase = isExactMatchRequested.keyphrase;
+    const matches = [];
+
+    // Return the index of the match. It returns -1 if there is no match.
+    let index = text.indexOf(keyphrase);
+    while (index !== -1) {
+      // Push the match to the array.
+      matches.push(keyphrase);
+      // Look for the next match after the previous one and adjust the index.
+      index = text.indexOf(keyphrase, index + keyphrase.length);
+    }
+    return matches;
+  }
+
+  /*
+   * `getContentWords` is used here to retrieve the words from the text instead of `getWords` because it has an additional step
+   * to remove this ending -じゃ from the segmented words which means that using this method will improve matching possibility.
+   */
+  const words = (0, _getContentWords.default)(text);
+  return words.filter(word => wordToMatch === word);
+}
+//# sourceMappingURL=matchTextWithWord.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/matchTextWithWord.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchTextWithWord.js","names":["_getContentWords","_interopRequireDefault","require","_processExactMatchRequest","_quotes","e","__esModule","default","_default","text","wordToMatch","toLowerCase","isExactMatchRequested","processExactMatchRequest","exactMatchRequested","normalizeSingle","keyphrase","matches","index","indexOf","push","length","words","getContentWords","filter","word"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/matchTextWithWord.js"],"sourcesContent":["import getContentWords from \"./getContentWords\";\nimport processExactMatchRequest from \"../../../helpers/match/processExactMatchRequest\";\nimport { normalizeSingle } from \"../../../helpers/sanitize/quotes\";\n\n/**\n * Checks for word matches in a text and returns an array containing the matched word(s).\n *\n * @param {string}  text               The text to find the word to match.\n * @param {string}  wordToMatch        The word to match.\n *\n * @returns {Array} An array of the matched word(s).\n */\nexport default function( text, wordToMatch ) {\n\t/*\n     * Lowercase the text so that it matches the wordToMatch which is already lowercased.\n     * Note that Japanese doesn't differentiate between upper and lower case, so this is only needed in case\n     * the text contains non-Japanese characters.\n     */\n\ttext = text.toLowerCase();\n\n\t// Check if the exact match is requested.\n\tconst isExactMatchRequested = processExactMatchRequest( wordToMatch );\n\tif ( isExactMatchRequested.exactMatchRequested ) {\n\t\t/*\n\t\t * Normalize single quotes in case they differ between the text and the word to match.\n\t\t * Normalizing is only needed for exact matching, because with non-exact matching single quotes are considered\n\t\t * word boundaries.\n\t\t * Quotes in wordToMatch are already normalized at an earlier point.\n\t\t*/\n\t\ttext = normalizeSingle( text );\n\n\t\tconst keyphrase = isExactMatchRequested.keyphrase;\n\t\tconst matches = [];\n\n\t\t// Return the index of the match. It returns -1 if there is no match.\n\t\tlet index = text.indexOf( keyphrase );\n\t\twhile ( index !== -1 ) {\n\t\t\t// Push the match to the array.\n\t\t\tmatches.push( keyphrase );\n\t\t\t// Look for the next match after the previous one and adjust the index.\n\t\t\tindex = text.indexOf( keyphrase, index + keyphrase.length );\n\t\t}\n\t\treturn matches;\n\t}\n\n\t/*\n\t * `getContentWords` is used here to retrieve the words from the text instead of `getWords` because it has an additional step\n\t * to remove this ending -じゃ from the segmented words which means that using this method will improve matching possibility.\n\t */\n\tconst words = getContentWords( text );\n\n\treturn words.filter( word => wordToMatch === word );\n}\n"],"mappings":";;;;;;AAAA,IAAAA,gBAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,yBAAA,GAAAF,sBAAA,CAAAC,OAAA;AACA,IAAAE,OAAA,GAAAF,OAAA;AAAmE,SAAAD,uBAAAI,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEnE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,IAAI,EAAEC,WAAW,EAAG;EAC5C;AACD;AACA;AACA;AACA;EACCD,IAAI,GAAGA,IAAI,CAACE,WAAW,CAAC,CAAC;;EAEzB;EACA,MAAMC,qBAAqB,GAAG,IAAAC,iCAAwB,EAAEH,WAAY,CAAC;EACrE,IAAKE,qBAAqB,CAACE,mBAAmB,EAAG;IAChD;AACF;AACA;AACA;AACA;AACA;IACEL,IAAI,GAAG,IAAAM,uBAAe,EAAEN,IAAK,CAAC;IAE9B,MAAMO,SAAS,GAAGJ,qBAAqB,CAACI,SAAS;IACjD,MAAMC,OAAO,GAAG,EAAE;;IAElB;IACA,IAAIC,KAAK,GAAGT,IAAI,CAACU,OAAO,CAAEH,SAAU,CAAC;IACrC,OAAQE,KAAK,KAAK,CAAC,CAAC,EAAG;MACtB;MACAD,OAAO,CAACG,IAAI,CAAEJ,SAAU,CAAC;MACzB;MACAE,KAAK,GAAGT,IAAI,CAACU,OAAO,CAAEH,SAAS,EAAEE,KAAK,GAAGF,SAAS,CAACK,MAAO,CAAC;IAC5D;IACA,OAAOJ,OAAO;EACf;;EAEA;AACD;AACA;AACA;EACC,MAAMK,KAAK,GAAG,IAAAC,wBAAe,EAAEd,IAAK,CAAC;EAErC,OAAOa,KAAK,CAACE,MAAM,CAAEC,IAAI,IAAIf,WAAW,KAAKe,IAAK,CAAC;AACpD","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/matchTransitionWords.js

@@ -0,0 +1,29 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+var _getWords = _interopRequireDefault(require("./getWords"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Matches the sentence against transition words.
+ *
+ * @param {string} sentence The sentence to match against.
+ * @param {Array} transitionWords The array containing transition words.
+ * @returns {Array} The found transitional words.
+ */
+function _default(sentence, transitionWords) {
+  // Retrieve the words from the segmenter, join into string for easier comparison through the contains function.
+  const words = (0, _getWords.default)(sentence).join("|");
+  const results = [];
+  transitionWords.forEach(function (transitionWord) {
+    // The transitionWords are segmented, like the sentence, so join here as well.
+    const target = transitionWord.join("|");
+    if (words.includes(target)) {
+      results.push(transitionWord);
+    }
+  });
+  return results;
+}
+//# sourceMappingURL=matchTransitionWords.js.map

package/build/cjs/languageProcessing/languages/ja/helpers/matchTransitionWords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchTransitionWords.js","names":["_getWords","_interopRequireDefault","require","e","__esModule","default","_default","sentence","transitionWords","words","getWords","join","results","forEach","transitionWord","target","includes","push"],"sources":["../../../../../../src/languageProcessing/languages/ja/helpers/matchTransitionWords.js"],"sourcesContent":["import getWords from \"./getWords\";\n\n/**\n * Matches the sentence against transition words.\n *\n * @param {string} sentence The sentence to match against.\n * @param {Array} transitionWords The array containing transition words.\n * @returns {Array} The found transitional words.\n */\nexport default function( sentence, transitionWords ) {\n\t// Retrieve the words from the segmenter, join into string for easier comparison through the contains function.\n\tconst words = getWords( sentence ).join( \"|\" );\n\n\tconst results = [];\n\n\ttransitionWords.forEach( function( transitionWord ) {\n\t\t// The transitionWords are segmented, like the sentence, so join here as well.\n\t\tconst target = transitionWord.join( \"|\" );\n\t\tif ( words.includes( target ) ) {\n\t\t\tresults.push( transitionWord );\n\t\t}\n\t} );\n\n\treturn results;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,SAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAkC,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAElC;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAG,SAAUC,QAAQ,EAAEC,eAAe,EAAG;EACpD;EACA,MAAMC,KAAK,GAAG,IAAAC,iBAAQ,EAAEH,QAAS,CAAC,CAACI,IAAI,CAAE,GAAI,CAAC;EAE9C,MAAMC,OAAO,GAAG,EAAE;EAElBJ,eAAe,CAACK,OAAO,CAAE,UAAUC,cAAc,EAAG;IACnD;IACA,MAAMC,MAAM,GAAGD,cAAc,CAACH,IAAI,CAAE,GAAI,CAAC;IACzC,IAAKF,KAAK,CAACO,QAAQ,CAAED,MAAO,CAAC,EAAG;MAC/BH,OAAO,CAACK,IAAI,CAAEH,cAAe,CAAC;IAC/B;EACD,CAAE,CAAC;EAEH,OAAOF,OAAO;AACf","ignoreList":[]}

package/build/cjs/languageProcessing/languages/ja/helpers/memoizedSentenceTokenizer.js

@@ -0,0 +1,37 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _SentenceTokenizer = _interopRequireDefault(require("./internal/SentenceTokenizer"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Returns the sentences from a certain text.
+ *
+ * @param {string} text 					The text to retrieve sentences from..
+ * @param {boolean} [trimSentences=true] 	Whether to trim whitespace from the beginning and end of the sentences or not.
+ *
+ * @returns {Array<string>} The list of sentences in the text.
+ */
+function getSentenceTokenizer(text, trimSentences = true) {
+  const sentenceTokenizer = new _SentenceTokenizer.default();
+  const {
+    tokenizer,
+    tokens
+  } = sentenceTokenizer.createTokenizer();
+  sentenceTokenizer.tokenize(tokenizer, text);
+  return tokens.length === 0 ? [] : sentenceTokenizer.getSentencesFromTokens(tokens, trimSentences);
+}
+
+/*
+ * The second argument to the memoize function is a so-called resolver function.
+ * It creates a cache key consisting of a combination of all arguments to a function.
+ * This is needed because by default, only the first argument to a function is used as the map cache key by the memoize function.
+ * This means that a function is only re-run if the value of the first argument changes.
+ * We want to re-run the getSentenceTokenizer function also when only the second argument changes to prevent cache collisions.
+ * @see https://lodash.com/docs/4.17.15#memoize
+ */
+var _default = exports.default = (0, _lodash.memoize)(getSentenceTokenizer, (...args) => JSON.stringify(args));
+//# sourceMappingURL=memoizedSentenceTokenizer.js.map