axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/languages/ja/config/transitionWords.js

@@ -0,0 +1,354 @@
+/** @module config/transitionWords */
+
+export default [
+	[ "だから" ],
+	[ "その", "ため" ],
+	[ "この", "ため" ],
+	[ "それ", "で" ],
+	[ "そこ", "で" ],
+	[ "よって" ],
+	[ "する", "と" ],
+	[ "だと", "する", "と" ],
+	[ "ゆえ", "に" ],
+	[ "それゆえ", "に" ],
+	[ "し", "た", "がっ", "て" ],
+	[ "それゆえ" ],
+	[ "それ", "なら" ],
+	[ "それ", "で", "は" ],
+	[ "ならば" ],
+	[ "だ", "と", "し", "たら" ],
+	[ "そう", "する", "と" ],
+	[ "そう", "し", "たら" ],
+	[ "さも", "ない", "と" ],
+	[ "そうし", "ない", "と" ],
+	[ "そう", "で", "ない", "なら" ],
+	[ "だと", "すれ", "ば" ],
+	[ "そう", "なる", "と" ],
+	[ "と", "なる", "と" ],
+	[ "と", "なれ", "ば" ],
+	[ "そうして", "みる", "と" ],
+	[ "そう", "なれ", "ば" ],
+	[ "そうして" ],
+	[ "そのけっか" ],
+	[ "その", "結果" ],
+	[ "しかし" ],
+	[ "けど" ],
+	[ "ただ" ],
+	[ "だ", "が" ],
+	[ "しかし", "ながら" ],
+	[ "けれど" ],
+	[ "けれども" ],
+	[ "だけど" ],
+	[ "だけども" ],
+	[ "そう", "で", "は", "ある", "が" ],
+	[ "それ", "でも" ],
+	[ "でも" ],
+	[ "で", "は", "ある", "が" ],
+	[ "に", "も", "かかわらず" ],
+	[ "それ", "に", "も", "かかわらず" ],
+	[ "ところ", "が" ],
+	[ "しかる", "に" ],
+	[ "と", "はいう", "もの", "の" ],
+	[ "と", "は", "言う", "もの", "の" ],
+	[ "な", "の", "に" ],
+	[ "それ", "な", "の", "に" ],
+	[ "と", "は", "いえ" ],
+	[ "そう", "はいう", "もの", "の" ],
+	[ "そう", "は", "言う", "もの", "の" ],
+	[ "そのくせ" ],
+	[ "さり", "と", "て" ],
+	[ "さ", "れど" ],
+	[ "これ", "に", "はんし", "て" ],
+	[ "これ", "に", "反し", "て" ],
+	[ "それ", "に", "し", "て", "は" ],
+	[ "そのわり", "に", "は" ],
+	[ "そのわり", "に" ],
+	[ "それ", "なら" ],
+	[ "ならび", "に" ],
+	[ "おなじく" ],
+	[ "同じく" ],
+	[ "また" ],
+	[ "どう", "よう", "に" ],
+	[ "同様", "に" ],
+	[ "さ", "れど", "も" ],
+	[ "さらに" ],
+	[ "おなじ", "よう", "に" ],
+	[ "同じ", "よう", "に" ],
+	[ "のみ", "なら", "ず" ],
+	[ "しかも" ],
+	[ "おまけ", "に" ],
+	[ "そのうえ" ],
+	[ "その", "上" ],
+	[ "そして" ],
+	[ "それ", "から" ],
+	[ "それどころか" ],
+	[ "どころか" ],
+	[ "それ", "に" ],
+	[ "それ", "に", "し", "て", "も" ],
+	[ "くわえ", "て" ],
+	[ "加え", "て" ],
+	[ "それ", "にくわえ", "て" ],
+	[ "それ", "に", "加え", "て" ],
+	[ "ひいて", "は" ],
+	[ "なお" ],
+	[ "それ", "ばかり", "で", "なく" ],
+	[ "それ", "ばかりか" ],
+	[ "とも", "あれ" ],
+	[ "その", "うえ", "に" ],
+	[ "その", "上", "に" ],
+	[ "その", "うえ", "で" ],
+	[ "その", "上", "で" ],
+	[ "あまつさえ" ],
+	[ "いっぽう" ],
+	[ "一方" ],
+	[ "たほう" ],
+	[ "他方" ],
+	[ "ぎゃく", "に" ],
+	[ "逆", "に" ],
+	[ "それ", "に", "たいし", "て" ],
+	[ "それ", "に対して" ],
+	[ "たいし", "て" ],
+	[ "対して" ],
+	[ "はん", "たい", "に" ],
+	[ "反対", "に" ],
+	[ "はんめん" ],
+	[ "反面" ],
+	[ "その", "は", "んめん" ],
+	[ "その", "反面" ],
+	[ "また", "は" ],
+	[ "もしく", "は" ],
+	[ "あるい", "は" ],
+	[ "それとも" ],
+	[ "ほか", "に", "は" ],
+	[ "他", "に", "は" ],
+	[ "ほか", "に", "も" ],
+	[ "他", "に", "も" ],
+	[ "だいいち", "に" ],
+	[ "第一", "に" ],
+	[ "だい", "に", "に" ],
+	[ "第二", "に" ],
+	[ "だい", "さん", "に" ],
+	[ "第三", "に" ],
+	[ "だい", "よん", "に" ],
+	[ "第四", "に" ],
+	[ "ひとつめ", "は" ],
+	[ "一つめ", "は" ],
+	[ "一つ", "目", "は" ],
+	[ "１つめ", "は" ],
+	[ "1つ", "目", "は" ],
+	[ "ふたつめ", "は" ],
+	[ "二つめ", "は" ],
+	[ "二つ", "目", "は" ],
+	[ "２つめ", "は" ],
+	[ "2つ", "目", "は" ],
+	[ "みっつめ", "は" ],
+	[ "三つめ", "は" ],
+	[ "三つ", "目", "は" ],
+	[ "３つめ", "は" ],
+	[ "3つ", "目", "は" ],
+	[ "よっつめ", "は" ],
+	[ "四つめ", "は" ],
+	[ "四つ", "目", "は" ],
+	[ "４つめ", "は" ],
+	[ "4つ", "目", "は" ],
+	[ "いって", "んめ", "は" ],
+	[ "一点目", "は" ],
+	[ "1", "点目", "は" ],
+	[ "に", "てんめ", "は" ],
+	[ "二点目", "は" ],
+	[ "2", "点目", "は" ],
+	[ "さん", "て", "んめ", "は" ],
+	[ "三点目", "は" ],
+	[ "3", "点目", "は" ],
+	[ "よん", "て", "んめ", "は" ],
+	[ "四点目", "は" ],
+	[ "4", "点目", "は" ],
+	[ "ひとつ", "は" ],
+	[ "一つ", "は" ],
+	[ "１つ", "は" ],
+	[ "もう", "ひ", "とつ", "は" ],
+	[ "もう", "一つ", "は" ],
+	[ "もう", "１", "つ", "は" ],
+	[ "いってん", "は" ],
+	[ "一点", "は" ],
+	[ "1点", "は" ],
+	[ "もういってん", "は" ],
+	[ "もう", "一点", "は" ],
+	[ "もう", "1", "点", "は" ],
+	[ "はじめ", "に" ],
+	[ "さいしょ", "に" ],
+	[ "最初", "に" ],
+	[ "つづい", "て" ],
+	[ "続い", "て" ],
+	[ "つい", "で" ],
+	[ "次い", "で" ],
+	[ "さいごに" ],
+	[ "最後", "に" ],
+	[ "おわり", "に" ],
+	[ "終わり", "に" ],
+	[ "終り", "に" ],
+	[ "その", "ご" ],
+	[ "その", "後" ],
+	[ "まず" ],
+	[ "つぎ", "に" ],
+	[ "次", "に" ],
+	[ "さらに" ],
+	[ "その", "あと" ],
+	[ "その", "あと", "に" ],
+	[ "その後", "に" ],
+	[ "なぜ", "なら" ],
+	[ "なぜ", "なら", "ば" ],
+	[ "なぜか", "という", "と" ],
+	[ "という", "の", "は" ],
+	[ "という", "の", "も" ],
+	[ "だっ", "て" ],
+	[ "なに", "しろ" ],
+	[ "なにせ" ],
+	[ "どう", "して", "か", "という", "と" ],
+	[ "なん", "で", "か", "という", "と" ],
+	[ "ち", "なみ", "に" ],
+	[ "ただ" ],
+	[ "もっとも" ],
+	[ "その", "かわり" ],
+	[ "ただし" ],
+	[ "そも", "そも" ],
+	[ "じつ", "は" ],
+	[ "実", "は" ],
+	[ "じつ", "の", "ところ" ],
+	[ "実", "の", "ところ" ],
+	[ "じつ", "は", "という", "と" ],
+	[ "実", "は", "という", "と" ],
+	[ "実", "は", "と", "言う", "と" ],
+	[ "つまり" ],
+	[ "いいかえる", "と" ],
+	[ "言い", "かえる", "と" ],
+	[ "言い換える", "と" ],
+	[ "す", "なわち" ],
+	[ "よう", "は" ],
+	[ "要", "は" ],
+	[ "とど", "の", "つまり" ],
+	[ "よう", "する", "に" ],
+	[ "要", "する", "に" ],
+	[ "むしろ" ],
+	[ "かんげん", "する", "と" ],
+	[ "換言", "する", "と" ],
+	[ "かえっ", "て" ],
+	[ "かわり", "に" ],
+	[ "その", "かわり" ],
+	[ "いわば" ],
+	[ "いって", "みれ", "ば" ],
+	[ "言っ", "て", "みれ", "ば" ],
+	[ "という", "より" ],
+	[ "と言う", "より" ],
+	[ "という", "より", "は" ],
+	[ "と言う", "より", "は" ],
+	[ "という", "より", "も" ],
+	[ "と言う", "より", "も" ],
+	[ "という", "か" ],
+	[ "ぐたい", "て", "き", "に", "は" ],
+	[ "具体", "的", "に", "は" ],
+	[ "た", "とえば" ],
+	[ "例え", "ば" ],
+	[ "とりわけ" ],
+	[ "なか", "で", "も" ],
+	[ "こと", "に" ],
+	[ "殊", "に" ],
+	[ "とく", "に" ],
+	[ "特に" ],
+	[ "それ", "に", "は" ],
+	[ "その", "ため", "に", "は" ],
+	[ "そう", "する", "ば", "あい" ],
+	[ "そう", "する", "場合" ],
+	[ "その", "ば", "あい" ],
+	[ "その", "場合" ],
+	[ "そうすれ", "ば" ],
+	[ "それ", "によって" ],
+	[ "そう", "する", "こと", "で" ],
+	[ "さ", "れ", "ば" ],
+	[ "さすれ", "ば" ],
+	[ "さて" ],
+	[ "それ", "につけて", "も" ],
+	[ "ところ", "で" ],
+	[ "とき", "に" ],
+	[ "時", "に" ],
+	[ "それ", "は", "さ", "て", "おき" ],
+	[ "で", "は" ],
+	[ "それ", "で", "は" ],
+	[ "じゃ", "あ" ],
+	[ "とも", "あれ" ],
+	[ "それ", "は", "そう", "と" ],
+	[ "そういえ", "ば" ],
+	[ "それ", "に", "し", "たっ", "て" ],
+	[ "しょせん" ],
+	[ "所詮" ],
+	[ "ど", "の", "みち" ],
+	[ "どの", "道" ],
+	[ "どっちみち" ],
+	[ "どっち道" ],
+	[ "この", "よう", "に" ],
+	[ "こうして" ],
+	[ "いずれ", "に", "せ", "よ" ],
+	[ "いずれ", "に", "し", "て", "も" ],
+	[ "どちら", "に", "せ", "よ" ],
+	[ "どっち", "に", "し", "て", "も" ],
+	[ "どっち", "に", "せ", "よ" ],
+	[ "どちら", "に", "し", "て", "も" ],
+	[ "とも", "あれ" ],
+	[ "いじょう", "の", "よう", "に" ],
+	[ "以上", "の", "よう", "に" ],
+	[ "たしか", "に" ],
+	[ "確か", "に" ],
+	[ "いっぽう", "で" ],
+	[ "一方", "で" ],
+	[ "いっぽう", "で", "は" ],
+	[ "一方", "で", "は" ],
+	[ "たほう", "で" ],
+	[ "他方", "で" ],
+	[ "たほう", "で", "は" ],
+	[ "他方", "で", "は" ],
+	[ "かり", "に" ],
+	[ "仮", "に" ],
+	[ "た", "とえ" ],
+	[ "よしんば" ],
+	[ "と", "する", "と" ],
+	[ "とすれ", "ば" ],
+	[ "し", "から", "ば" ],
+	[ "に", "も", "かかわらず" ],
+	[ "に", "も", "拘わらず" ],
+	[ "といえども" ],
+	[ "と", "言え", "ど", "も" ],
+	[ "といって", "も" ],
+	[ "と言っ", "て", "も" ],
+	[ "と", "はいう", "もの", "の" ],
+	[ "と", "は", "言う", "もの", "の" ],
+	[ "こと", "に", "は" ],
+	[ "まも", "なく" ],
+	[ "やが", "て" ],
+	[ "と", "たん", "に" ],
+	[ "つづい", "て" ],
+	[ "続い", "て" ],
+	[ "ひきつづき" ],
+	[ "引き続き" ],
+	[ "かと", "おも", "う", "と" ],
+	[ "かと", "思う", "と" ],
+	[ "かと", "おもえば" ],
+	[ "かと", "思え", "ば" ],
+	[ "かと", "おもうまも", "なく" ],
+	[ "かと", "思う間も", "なく" ],
+	[ "かと", "思う", "まも", "なく" ],
+	[ "と", "たん" ],
+	[ "そのしゅんかん" ],
+	[ "その", "瞬間" ],
+	[ "どうじ", "に" ],
+	[ "同", "時に" ],
+	[ "まし", "て" ],
+	[ "まし", "て", "や" ],
+	[ "これ", "に", "たいし", "て" ],
+	[ "これ", "に対して" ],
+	[ "もちろん" ],
+	[ "も", "とより" ],
+	[ "それ", "だ", "から" ],
+	[ "これ", "だ", "から" ],
+	[ "とうぜん" ],
+	[ "当然" ],
+];

package/src/languageProcessing/languages/ja/customResearches/findKeyphraseInSEOTitle.js

@@ -0,0 +1,98 @@
+import { isEmpty } from "lodash";
+import { findWordFormsInString } from "../../../helpers/match/findKeywordFormsInString";
+import wordMatch from "../../../helpers/match/matchTextWithArray";
+import processExactMatchRequest from "../../../helpers/match/processExactMatchRequest";
+import getContentWords from "../helpers/getContentWords";
+import japaneseWordMatchHelper from "../helpers/matchTextWithWord";
+import functionWords from "../config/functionWords";
+import getWords from "../helpers/getWords";
+
+/**
+ * Checks the position of the keyphrase in the SEO title.
+ *
+ * @param {string} title    The SEO title of the paper.
+ * @param {number} position The position of the keyphrase in the SEO title.
+ *
+ * @returns {number} Potentially adjusted position of the keyphrase in the SEO title.
+ */
+function adjustPosition( title, position ) {
+	// Don't do anything if position if already 0.
+	if ( position === 0 ) {
+		return position;
+	}
+
+	// Retrieve the SEO title words before the keyphrase.
+	let titleBeforeKeyword = title.substring( 0, position );
+	titleBeforeKeyword = getWords( titleBeforeKeyword );
+	// Retrieve the non-function words.
+	titleBeforeKeyword = titleBeforeKeyword.filter( word => ! functionWords.includes( word ) );
+
+	if ( isEmpty( titleBeforeKeyword ) ) {
+		/*
+		 * Return position 0 if there are no words left in the SEO title before the keyword after filtering
+		 * the function words (such that "楽しい旅行" in "その楽しい旅行" is still counted as position 0).
+ 		 */
+		return 0;
+	}
+
+	return position;
+}
+
+/**
+ * Checks the occurrences of the keyphrase in the SEO title. Returns a result that contains information on
+ * (1) whether all the keyphrase forms are found,
+ * (2) the lowest number of the positions of the matches, and
+ * (3) whether the exact match keyphrase is requested.
+ *
+ * @param {Object} paper 			The paper containing SEO title and keyword.
+ * @param {Researcher} researcher 	The researcher to use for analysis.
+ *
+ * @returns {Object} An object containing these info: (1) whether all the keyphrase forms are found,
+ * (2) the lowest number of the positions of the matches, and (3) whether the exact match keyphrase is requested.
+ */
+export default function( paper, researcher ) {
+	const title = paper.getTitle();
+	let keyphrase = paper.getKeyword();
+
+	const result = { allWordsFound: false, position: -1, exactMatchKeyphrase: false  };
+
+	/*
+	 * Check if the keyword is enclosed in quotation mark.
+	 * If yes, remove the quotation marks and check if the exact match of the keyphrase is found in the SEO title.
+	 */
+	const exactMatchRequest = processExactMatchRequest( keyphrase );
+	if ( exactMatchRequest.exactMatchRequested ) {
+		result.exactMatchKeyphrase = true;
+
+		// Check if the exact match of the keyphrase is found in the SEO title and directly return the result if there is no match.
+		if ( ! title.includes( exactMatchRequest.keyphrase ) ) {
+			return result;
+		}
+
+		keyphrase = getContentWords( exactMatchRequest.keyphrase );
+
+		const keyphraseMatched = wordMatch( title, keyphrase, "ja", japaneseWordMatchHelper );
+
+		// It's an exact match if the length of the matched keyphrase is the same as the keyphrase length.
+		if ( keyphraseMatched.matches.length === keyphrase.length ) {
+			result.allWordsFound = true;
+			result.position = adjustPosition( title, keyphraseMatched.position );
+		}
+		/*
+		 * When the exact match process is requested, we don't need to run the check for the different word forms,
+		 * and return the result here.
+		 */
+		return result;
+	}
+
+	// Get the forms of the keyphrase (using the morphology research).
+	const keyphraseForms = researcher.getResearch( "morphology" ).keyphraseForms;
+	const separateWordFormsMatched = findWordFormsInString( keyphraseForms, title, "ja", japaneseWordMatchHelper );
+
+	if ( separateWordFormsMatched.percentWordMatches === 100 ) {
+		result.allWordsFound = true;
+		result.position = adjustPosition( title, separateWordFormsMatched.position );
+	}
+
+	return result;
+}

package/src/languageProcessing/languages/ja/customResearches/getKeyphraseLength.js

@@ -0,0 +1,19 @@
+import getWords from "../helpers/getWords";
+import characterCountHelper from "../helpers/wordsCharacterCount";
+
+/**
+ * Calculates the character length of the keyphrase.
+ *
+ * @param {Paper} paper The paper to research
+ *
+ * @returns {Object} The length of the keyphrase and an empty array of function words.
+ */
+export default function( paper ) {
+	const keyphrase = getWords( paper.getKeyword() );
+
+	return {
+		keyphraseLength: characterCountHelper( keyphrase ),
+		// Returns empty array because we don't take function words into account for Japanese keyphrase length calculations.
+		functionWords: [],
+	};
+}

package/src/languageProcessing/languages/ja/customResearches/getWordForms.js

@@ -0,0 +1,50 @@
+import { get } from "lodash";
+import parseSynonyms from "../../../helpers/sanitize/parseSynonyms";
+import getContentWords from "../helpers/getContentWords";
+import createWordForms from "../helpers/internal/createWordForms";
+import doubleQuotes from "../../../helpers/sanitize/doubleQuotes";
+
+/**
+ * Creates word forms for each word in the given keyphrase.
+ *
+ * @param {string}      keyphrase   The keyphrase to generate word forms for.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {Array<string[]>} The word forms for each word in the keyphrase.
+ */
+function getKeyphraseForms( keyphrase, researcher ) {
+	// The keyphrase is in double quotes: use it as an exact match keyphrase.
+	if ( doubleQuotes.includes( keyphrase[ 0 ] ) && doubleQuotes.includes( keyphrase[ keyphrase.length - 1 ] ) ) {
+		return [ [ keyphrase ] ];
+	}
+
+	const keyphraseWords = getContentWords( keyphrase );
+
+	// If the keyphrase does not contain content words, return an empty list.
+	if ( keyphraseWords.length === 0 ) {
+		return [ [] ];
+	}
+
+	const morphologyData = get( researcher.getData( "morphology" ), "ja", false );
+	return keyphraseWords.map( word => morphologyData ? createWordForms( word, morphologyData ) : [ word ] );
+}
+
+/**
+ * Gets all matching word forms for the keyphrase and synonyms.
+ *
+ * @param {Paper}       paper       The paper.
+ * @param {Researcher}  researcher  The researcher.
+ *
+ * @returns {Object} Object with an array of keyphrase forms and an array of arrays of synonyms forms, based on the forms
+ * found in the text or created forms.
+ */
+export default function( paper, researcher ) {
+	let keyphrase = paper.getKeyword().toLocaleLowerCase( "ja" ).trim();
+	// Remove spaces from the keyphrase.
+	keyphrase = keyphrase.replace( /\s/g, "" );
+	const synonyms = parseSynonyms( paper.getSynonyms().toLocaleLowerCase( "ja" ).trim() );
+	const keyphraseForms = getKeyphraseForms( keyphrase, researcher );
+	const synonymsForms = synonyms.map( synonym => getKeyphraseForms( synonym, researcher ) );
+
+	return { keyphraseForms, synonymsForms };
+}

package/src/languageProcessing/languages/ja/customResearches/textLength.js

@@ -0,0 +1,24 @@
+import countCharacters from "../helpers/countCharacters";
+
+/**
+ * A result of the character count calculation.
+ *
+ * @typedef CharacterCountResult
+ * @param {number} count The number of characters found in the text.
+ * @param {string} unit The unit used in the text length calculations, always "character".
+ */
+
+/**
+ * Count the characters in the text.
+ *
+ * @param {Paper} paper The Paper object.
+ *
+ * @returns {CharacterCountResult} The number of characters found in the text, plus "character" as the unit used in calculating the text length.
+ */
+export default function( paper ) {
+	return {
+		text: paper.getText(),
+		count: countCharacters( paper.getText() ),
+		unit: "character",
+	};
+}

package/src/languageProcessing/languages/ja/helpers/countCharacters.js

@@ -0,0 +1,19 @@
+import {languageProcessing} from '../../../index';
+const {sanitizeString} = languageProcessing;
+import removeURLs from '../../../helpers/sanitize/removeURLs.js';
+
+/**
+ * Calculates the character count which serves as a measure of text length.
+ * The character count includes letters, punctuation, and numbers. It doesn't include URLs, HTML tags, and spaces.
+ *
+ * @param {string} text The text to be counted.
+ *
+ * @returns {number} The character count of the given text.
+ */
+export default function(text) {
+  text = removeURLs(text);
+  text = sanitizeString(text);
+  text = text.replace(/\s/g, '');
+
+  return text.length;
+}

package/src/languageProcessing/languages/ja/helpers/customGetStemmer.js

@@ -0,0 +1,21 @@
+import {get} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {baseStemmer} = languageProcessing;
+import determineStem from './internal/determineStem';
+
+/**
+ * Returns the stemmer for a researcher. This helper is currently only used for Prominent Words functionality.
+ *
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Function} The stemmer.
+ */
+export default function getStemmer(researcher) {
+  const morphologyData = get(researcher.getData('morphology'), 'ja', false);
+
+  if (morphologyData) {
+    return word => determineStem(word, morphologyData);
+  }
+
+  return baseStemmer;
+}

package/src/languageProcessing/languages/ja/helpers/getContentWords.js

@@ -0,0 +1,21 @@
+import getWords from "./getWords";
+import functionWords from "../config/functionWords";
+
+/**
+ * Filters out function words and removes じゃ ending if a words ends in it.
+ *
+ * @param {string} text The input text.
+ *
+ * @returns {Array} The array words with function words filtered out and じゃ removed.
+ */
+export default function( text ) {
+	let words = getWords( text );
+	// Filter function words and morphemes.
+	words = words.filter( word => ! functionWords.includes( word ) );
+
+	// Check if the segment ends in -じゃ, and remove the ending if it does.
+	const ending = "じゃ";
+	words = words.map( word => word.endsWith( ending ) ? word.slice( 0, -ending.length ) : word );
+
+	return words;
+}

package/src/languageProcessing/languages/ja/helpers/getWords.js

@@ -0,0 +1,31 @@
+import {filter, map} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {sanitizeString, removePunctuation} = languageProcessing;
+import TinySegmenter from 'tiny-segmenter';
+
+const segmenter = new TinySegmenter();
+
+/**
+ * Returns an array with words used in the text.
+ *
+ * @param {string} text The text to be counted.
+ *
+ * @returns {Array} The array with all words.
+ */
+export default function(text) {
+  // Strips HTML tags.
+  text = sanitizeString(text);
+  if (text === '') {
+    return [];
+  }
+
+  let words = segmenter.segment(text);
+
+  words = map(words, function(word) {
+    return removePunctuation(word);
+  });
+
+  return filter(words, function(word) {
+    return word.trim() !== '';
+  });
+}