axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/languages/ja/helpers/internal/SentenceTokenizer.js

@@ -0,0 +1,102 @@
+import { isNaN } from "lodash";
+
+import { normalize as normalizeQuotes } from "../../../../helpers/sanitize/quotes.js";
+import SentenceTokenizer from "../../../../helpers/sentence/SentenceTokenizer";
+
+/*
+ * \u2026 - ellipsis.
+ * \u3002 - Japanese ideographic full stop.
+ * \uFF61 - Japanese half-width ideographic full stop.
+ * \uFF01 - Japanese full-width exclamation mark.
+ * \u203C - Japanese double exclamation mark.
+ * \uFF1F - Japanese fullwidth question mark.
+ * \u2047 - Japanese double question mark.
+ * \u2049 - Japanese exclamation question mark.
+ * \u2048 - Japanese question exclamation mark.
+ * \u2025 - Japanese two dot leader.
+ */
+const sentenceDelimiters = "?!\u2026\u3002\uFF61\uFF01\u203C\uFF1F\u2047\u2049\u2048\u2049\u2025";
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+export default class JapaneseSentenceTokenizer extends SentenceTokenizer {
+	/**
+	 * Constructor
+	 * @constructor
+	 */
+	constructor() {
+		super();
+		this.sentenceDelimiters = sentenceDelimiters;
+	}
+
+	/**
+	 * Returns whether a certain character is a number.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the character is a capital letter.
+	 */
+	isNumber( character ) {
+		const japaneseNumbers = [
+			// Full-width.
+			/^[\uFF10-\uFF19]+$/i,
+			// Circled digit, parenthesized digit, and digit with full stop.
+			/^[\u2460-\u249B]+$/i,
+			// Parenthesized ideograph.
+			/^[\u3220-\u3229]+$/i,
+			// Circled ideograph.
+			/^[\u3280-\u3289]+$/i,
+		];
+
+		return ( ! isNaN( parseInt( character, 10 ) ) || japaneseNumbers.some( numberRange => numberRange.test( character ) ) );
+	}
+
+	/**
+	 * Returns whether a given character is quotation mark.
+	 *
+	 * @param {string} character The character to check.
+	 * @returns {boolean} Whether or not the given character is a quotation mark.
+	 */
+	isQuotation( character ) {
+		character = normalizeQuotes( character );
+
+		const japaneseOpeningQuotationMark = /^[\u300C\u300E\u3008\u3014\u3010\uFF5B\uFF3B]+$/i;
+
+		return "'" === character || "\"" === character || japaneseOpeningQuotationMark.test( character );
+	}
+
+	/**
+	 * Checks whether a character is from Japanese language that could be sentence beginning.
+	 *
+	 * @param {string} letter The letter to check.
+	 *
+	 * @returns {boolean} Whether the letter is from Japanese language that could be sentence beginning.
+	 */
+	isLetterFromSpecificLanguage( letter ) {
+		const japaneseLetterRanges = [
+			// Hiragana.
+			/^[\u3040-\u3096]+$/i,
+			// Katakana full-width.
+			/^[\u30A1-\u30FA]+$/i,
+			/^[\u31F0-\u31FF]+$/i,
+			// Katakana half-width.
+			/^[\uFF66-\uFF9D]+$/i,
+			// Kanji.
+			/^[\u4E00-\u9FFC]+$/i,
+		];
+
+		return (
+			japaneseLetterRanges.some( ltrLetterRange => ltrLetterRange.test( letter ) )
+		);
+	}
+
+	/**
+	 * Always returns true as Japanese sentence beginning doesn't need to be preceded by a whitespace to be a valid one.
+	 *
+	 * @returns {true}  Always true.
+	 */
+	isCharacterASpace() {
+		return true;
+	}
+}
+

package/src/languageProcessing/languages/ja/helpers/internal/createWordForms.js

@@ -0,0 +1,68 @@
+import { flatten, uniq } from "lodash";
+
+/**
+ * Checks if the word matches any paradigm and creates the forms from the matched paradigm(s).
+ * If no matches found, returns an array with the original word.
+ *
+ * @param {string}  word            The word to check.
+ * @param {Object}  morphologyData  The morphology data.
+ *
+ * @returns {Array} The array of created forms.
+ */
+function createForms( word, morphologyData ) {
+	const paradigmEndingsGroups = morphologyData.paradigmGroups;
+
+	// The endings in the array are sorted from the longest to the shortest.
+	// This way, if there are multiple matches, the first element in the matched array is always the longest one.
+	let allEndings = uniq( flatten( paradigmEndingsGroups ) );
+	allEndings = allEndings.sort( ( a, b ) => b.length - a.length || a.localeCompare( b ) );
+
+	// Check if the word matches any ending(s) and save the ending(s).
+	const matchedEndings = allEndings.filter( ending => word.endsWith( ending ) );
+
+	const forms = [];
+
+	if ( matchedEndings.length === 0 ) {
+		// If there is no match found, add the original word to the forms array.
+		forms.push( word );
+	} else {
+		// Pick the longest ending.
+		const matchedEnding = matchedEndings[ 0 ];
+
+		// Extract the stem.
+		const matchedStem = word.slice( 0, -matchedEnding.length );
+
+		// Loop over each endings group in the endings group array.
+		for ( const endingsGroup of paradigmEndingsGroups ) {
+			// Check if the ending of the matched word can be found in the endings group and create all the forms with all the endings in the group.
+			if ( endingsGroup.includes( matchedEnding ) ) {
+				forms.push( endingsGroup.map( ending => matchedStem + ending ) );
+			}
+		}
+	}
+
+	// If the final character of the input word ends in る, also create an extra form where the character is removed.
+	const ruEnding = "る";
+	if ( word.endsWith( ruEnding ) ) {
+		forms.push( word.slice( 0, -ruEnding.length ) );
+	}
+
+	return uniq( flatten( forms ) );
+}
+
+/**
+ * Creates forms for Japanese word.
+ *
+ * @param {string}  word            The word to check. Assume that the input word is a segment as outputted from `getContentWords` helper.
+ * @param {Object}  morphologyData  The morphology data.
+ *
+ * @returns {Array} The array of created forms.
+ */
+export default function( word, morphologyData ) {
+	// Check if the word is longer than 1 character.
+	if ( word.length <= 1 ) {
+		return [ word ];
+	}
+
+	return createForms( word, morphologyData );
+}

package/src/languageProcessing/languages/ja/helpers/internal/determineStem.js

@@ -0,0 +1,17 @@
+import createWordForms from "./createWordForms";
+
+/**
+ * Determines the canonical stem from the word forms.
+ *
+ * @param {string} word             The word input.
+ * @param {object} morphologyData   The morphology data file.
+ *
+ * @returns {string}    The shortest form from the array as the canonical stem.
+ */
+export default function determineStem( word, morphologyData ) {
+	let createdWordForms = createWordForms( word, morphologyData );
+	// Sort from the shortest to the longest form.
+	createdWordForms = createdWordForms.sort( ( a, b ) => a.length - b.length || a.localeCompare( b ) );
+
+	return createdWordForms[ 0 ];
+}

package/src/languageProcessing/languages/ja/helpers/matchTextWithWord.js

@@ -0,0 +1,53 @@
+import getContentWords from "./getContentWords";
+import processExactMatchRequest from "../../../helpers/match/processExactMatchRequest";
+import { normalizeSingle } from "../../../helpers/sanitize/quotes";
+
+/**
+ * Checks for word matches in a text and returns an array containing the matched word(s).
+ *
+ * @param {string}  text               The text to find the word to match.
+ * @param {string}  wordToMatch        The word to match.
+ *
+ * @returns {Array} An array of the matched word(s).
+ */
+export default function( text, wordToMatch ) {
+	/*
+     * Lowercase the text so that it matches the wordToMatch which is already lowercased.
+     * Note that Japanese doesn't differentiate between upper and lower case, so this is only needed in case
+     * the text contains non-Japanese characters.
+     */
+	text = text.toLowerCase();
+
+	// Check if the exact match is requested.
+	const isExactMatchRequested = processExactMatchRequest( wordToMatch );
+	if ( isExactMatchRequested.exactMatchRequested ) {
+		/*
+		 * Normalize single quotes in case they differ between the text and the word to match.
+		 * Normalizing is only needed for exact matching, because with non-exact matching single quotes are considered
+		 * word boundaries.
+		 * Quotes in wordToMatch are already normalized at an earlier point.
+		*/
+		text = normalizeSingle( text );
+
+		const keyphrase = isExactMatchRequested.keyphrase;
+		const matches = [];
+
+		// Return the index of the match. It returns -1 if there is no match.
+		let index = text.indexOf( keyphrase );
+		while ( index !== -1 ) {
+			// Push the match to the array.
+			matches.push( keyphrase );
+			// Look for the next match after the previous one and adjust the index.
+			index = text.indexOf( keyphrase, index + keyphrase.length );
+		}
+		return matches;
+	}
+
+	/*
+	 * `getContentWords` is used here to retrieve the words from the text instead of `getWords` because it has an additional step
+	 * to remove this ending -じゃ from the segmented words which means that using this method will improve matching possibility.
+	 */
+	const words = getContentWords( text );
+
+	return words.filter( word => wordToMatch === word );
+}

package/src/languageProcessing/languages/ja/helpers/matchTransitionWords.js

@@ -0,0 +1,25 @@
+import getWords from "./getWords";
+
+/**
+ * Matches the sentence against transition words.
+ *
+ * @param {string} sentence The sentence to match against.
+ * @param {Array} transitionWords The array containing transition words.
+ * @returns {Array} The found transitional words.
+ */
+export default function( sentence, transitionWords ) {
+	// Retrieve the words from the segmenter, join into string for easier comparison through the contains function.
+	const words = getWords( sentence ).join( "|" );
+
+	const results = [];
+
+	transitionWords.forEach( function( transitionWord ) {
+		// The transitionWords are segmented, like the sentence, so join here as well.
+		const target = transitionWord.join( "|" );
+		if ( words.includes( target ) ) {
+			results.push( transitionWord );
+		}
+	} );
+
+	return results;
+}

package/src/languageProcessing/languages/ja/helpers/memoizedSentenceTokenizer.js

@@ -0,0 +1,28 @@
+import { memoize } from "lodash";
+import SentenceTokenizer from "./internal/SentenceTokenizer";
+
+/**
+ * Returns the sentences from a certain text.
+ *
+ * @param {string} text 					The text to retrieve sentences from..
+ * @param {boolean} [trimSentences=true] 	Whether to trim whitespace from the beginning and end of the sentences or not.
+ *
+ * @returns {Array<string>} The list of sentences in the text.
+ */
+function getSentenceTokenizer( text, trimSentences = true ) {
+	const sentenceTokenizer = new SentenceTokenizer();
+	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
+	sentenceTokenizer.tokenize( tokenizer, text );
+
+	return ( tokens.length === 0 ? [] : sentenceTokenizer.getSentencesFromTokens( tokens, trimSentences ) );
+}
+
+/*
+ * The second argument to the memoize function is a so-called resolver function.
+ * It creates a cache key consisting of a combination of all arguments to a function.
+ * This is needed because by default, only the first argument to a function is used as the map cache key by the memoize function.
+ * This means that a function is only re-run if the value of the first argument changes.
+ * We want to re-run the getSentenceTokenizer function also when only the second argument changes to prevent cache collisions.
+ * @see https://lodash.com/docs/4.17.15#memoize
+ */
+export default memoize( getSentenceTokenizer, ( ...args ) => JSON.stringify( args ) );

package/src/languageProcessing/languages/ja/helpers/splitIntoTokensCustom.js

@@ -0,0 +1,20 @@
+import { map } from "lodash";
+import TinySegmenter from "tiny-segmenter";
+
+/**
+ * Split sentence into tokens.
+ *
+ * @param {string} sentenceText The sentence text to split.
+ *
+ * @returns {Token[]} The tokens.
+ */
+function splitIntoTokensCustom( sentenceText ) {
+	// Return empty string if sentence is empty.
+	if ( sentenceText === "" ) {
+		return [];
+	}
+	// Split sentences into words that are also tokens.
+	const words = new TinySegmenter().segment( sentenceText );
+	return map( words );
+}
+export default splitIntoTokensCustom;

package/src/languageProcessing/languages/ja/helpers/wordsCharacterCount.js

@@ -0,0 +1,13 @@
+/**
+ * Calculates the characters length of all the words in the array.
+ *
+ * @param {array} wordsArray    The array of words to calculate.
+ *
+ * @returns {number} The characters length of all words in the array. Returns 0 if the input is an empty array.
+ */
+export default function( wordsArray ) {
+	const counts = [];
+	wordsArray.map( form => counts.push( form.length ) );
+
+	return wordsArray.length === 0 ? 0 : counts.reduce( ( a, b ) => a + b );
+}

package/src/languageProcessing/languages/nb/Researcher.js

@@ -0,0 +1,45 @@
+import {languageProcessing} from '../../../index';
+const {AbstractResearcher} = languageProcessing;
+
+// All config
+import firstWordExceptions from './config/firstWordExceptions';
+import functionWords from './config/functionWords';
+import stopWords from './config/stopWords';
+import transitionWords from './config/transitionWords';
+import twoPartTransitionWords from './config/twoPartTransitionWords';
+
+// All helpers
+import getStemmer from './helpers/getStemmer';
+import getClauses from './helpers/getClauses';
+
+/**
+ * The researcher contains all the researches, helpers, data, and config.
+ */
+export default class Researcher extends AbstractResearcher {
+  /**
+   * Constructor
+   * @param {Paper} paper The Paper object that is needed within the researches.
+   * @constructor
+   */
+  constructor(paper) {
+    super(paper);
+
+    // Delete Flesch Reading Ease research since Norwegian doesn't have the support for it
+    delete this.defaultResearches.getFleschReadingScore;
+
+    Object.assign(this.config, {
+      language: 'nb',
+      passiveConstructionType: 'periphrastic',
+      functionWords,
+      firstWordExceptions,
+      transitionWords,
+      twoPartTransitionWords,
+      stopWords
+    });
+
+    Object.assign(this.helpers, {
+      getStemmer,
+      getClauses
+    });
+  }
+}

package/src/languageProcessing/languages/nb/config/firstWordExceptions.js

@@ -0,0 +1,12 @@
+/**
+ *  Returns an array with exceptions for the sentence beginning researcher.
+ *  @returns {Array} The array filled with exceptions.
+ *  */
+export default [
+	// Indefinite articles:
+	"ei", "et",
+	// Numbers 1-10:
+	"en", "ett", "to", "tre", "fire", "fem", "seks", "sju", "syv", "åtte", "ni", "ti",
+	// Demonstrative pronouns:
+	"denne", "dette", "disse", "den", "det", "de",
+];

package/src/languageProcessing/languages/nb/config/functionWords.js

@@ -0,0 +1,106 @@
+import { singleWords as transitionWords } from "./transitionWords";
+import transformWordsWithHyphens from "../../../helpers/transform/transformWordsWithHyphens";
+
+/**
+ * Returns an object with function words.
+ *
+ * @returns {Object} The object filled with various categories of function word arrays.
+ */
+// "En" is already listed among the cardinal numbers, "de", "det" and "den" among the personal pronouns.
+const articles = [ "ei", "et" ];
+
+const cardinalNumerals = [ "null", "en", "ett", "ene", "to", "tre", "fire", "fem", "seks", "syv", "åtte", "ni", "ti", "elleve", "tolv",
+	"tretten", "fjorten", "femten", "seksten", "sytten", "atten", "nitten", "tjue", "tyve", "tjueen", "enogtyve", "tretti", "tredve",
+	"førti", "førr", "femti", "seksti", "sytti", "åtti", "nitti", "hundre", "hundreogen", "etthundreogen", "tohundre", "tusen",
+	"tusenogen", "million", "millioner", "milliard", "milliarder" ];
+
+const ordinalNumerals = [ "nullte", "første", "først", "sekund", "tredje", "fjerde", "femte", "sjette", "syvende", "åttende", "niende", "tiende",
+	"ellevte", "tolvte", "trettende", "fjortende", "femtende", "sekstende", "syttende", "åttende", "nittende", "tjuende", "tjueførst", "tjueførste",
+	"trettiende", "førtiende", "femtiende", "sekstiende", "syttiende", "åttiende", "nittiende", "hundrede", "hundreogfemtiende", "to hundrede",
+	"tusende", "millionte", "millardte" ];
+
+const pronouns = [
+	// Personal pronouns.
+	"jeg", "du", "den", "det", "vi", "de", "han", "hun", "dere", "henne", "oss", "meg", "deg", "ham", "dem",
+	// Possessive pronouns.
+	"min", "din", "deres", "vår", "deres", "ditt", "mitt", "våre", "vårt", "hans", "hennes", "dens", "dets", "egen",
+	"egne", "mi", "di", "sin", "si", "sitt", "sine", "mine", "dine",
+	// Demonstrative pronouns. "De", "det" and "den" are among the personal pronouns.
+	"denne", "dette", "disse", "slik", "slikt", "slike", "sånn", "sånt", "sånne", "samme",
+	// Reciprocal pronouns.
+	"hverandre", "hvert",
+	// Relative pronouns.
+	"som" ];
+
+const interrogatives = [ "hvem", "hvordan", "hvorfor", "hvor", "hva", "hvilken", "hvilket", "hvilke" ];
+
+const quantifiers = [ "mange", "mye", "mang en", "mangt et", "hele", "mer", "ingen", "ingenting", "ikke noen", "ikke noe",
+	"alle", "all", "alt", "allting", "noen", "noe", "flere", "hver", "hvert", "annenhver", "ammethvert", "begge", "sov", "mest", "fleste",
+	"få", "fæst", "færrest", "flere", "flest" ];
+
+const reflexivePronouns = [ "seg", "selv" ];
+
+const indefinitePronouns = [ "ingenting", "annen", "annet", "andre" ];
+
+const prepositions = [ "sånn", "ved", "mot", "ned", "enn", "over", "inn", "i", "sa", "opp", "der", "fra", "din", "nei", "mellom", "di", "oppe",
+	"av", "med", "til", "å", "på", "du", "uten", "én", "under", "hos", "inne", "gjennom", "unna", "del", "nede", "til", "over", "under", "etter",
+	"kun", "blant", "for", "mellom", "blant" ];
+
+const conjunctions = [ "eller", "hvis", "ja", "et", "som", "i", "og", "både", "men", "mens", "enten", "verken", "at", "om", "da", "når", "før",
+	"idet", "etter at", "siden", "innen", "med det samme", "til", "inntil", "hver gang", "etter hvert som", "så lenge", "så lenge som", "så ofte",
+	"så ofte som", "så snart", "så snart som", "etter", "etterpå", "foran", "tidligere", "fordi", "ettersom", "derfor", "dersom", "hvis", "så fremt",
+	"så sant", "i fall", "i tilfelle", "med mindre", "uten at", "bare", "for så vidt som", "uten at", "uten å", "enda", "fordi om", "enda om",
+	"skjønt", "om enn", "hva så", "trass i at", "hvor så", "samme", "selv om", "hva enn", "til tross for at", "hvor enn", "uansett", "for at", "så",
+	"så at", "slik at", "sånn at", "for at, så", "slik som", "så som", "som om", "enn", "dess", "jo", "desto" ];
+
+const interviewVerbs = [ "tror", "fortelle", "fortell", "fortalte", "tenkte", "tenk" ];
+
+const intensifiers = [ "virkelig", "akkurat", "visst" ];
+
+const auxiliariesAndDelexicalizedVerbs = [ "ha", "har", "hadde", "gjør", "gjøre", "gjorde", "kaller", "kalte", "kalle",
+	"kalla", "virker", "virka", "virke", "virka", "går", "gikk", "gå", "leges", "lages", "legges", "lages", "består",
+	"bestod", "bestå", "bestått", "bety", "betyr" ];
+
+const generalAdjectivesAdverbs = [
+	// General adjective.
+	"helt", "andre", "litt", "lenge", "siste", "fint", "annet", "stor", "stort", "store", "neste", "lenger", "annen", "nye",
+	"alene", "flott", "gammel", "gammelt", "gamle", "klart", "liten", "langt", "gamle", "dårlig", "hyggelig", "gode", "sånt",
+	"nytt", "best", "lang", "små", "lot", "større", "vakker", "vakkert", "vakre", "ny", "bra", "bedre", "grei", "greit", "greie",
+	"høyt", "største", "størst", "slikt", "liten", "lita", "lite", "små", "mindre", "minst", "kort", "glad", "dårlig", "ille", "ond",
+	"vond", "verre", "verst", "eldre", "eldst", "lang", "lengre", "lengst", "nær", "næmerere", "nærere", "nærmest",
+	"nærest", "tung", "tyngre", "tyngst", "ung", "yngre", "yngst", "pen",
+	// General adverbs.
+	"alltid", "godt", "sammen", "tilbake", "etter", "igjen", "bare", "så", "veldig", "bedre", "samme", "far", "eneste", "enig",
+	"borte", "snart", "rundt", "beste", "bort", "vekk", "nesten", "ganske", "senere", "videre", "straks", "svært", "neste",
+	"bak", "bakre", "bakerst", "borte", "bortre", "bortest", "fremme", "fremre", "fremst", "foran", "forrest", "inne", "indre", "innerst",
+	"midt", "midtre", "midterst", "nede", "nedre", "nederst", "nord", "nordre", "nordligst", "øvre", "øverst", "sør", "søndre", "sørligst",
+	"vest", "vestre", "vestligst", "øst", "østre", "østligst", "ute", "ytre", "ytterst", "underst", "langt", "fram", "her", "der", "nok", "aldri",
+	"ut", "ned", "nede", "bort", "innom", "ingensteds", "sjelden", "sjeldnere", "sjeldnest", "raskt", "raskere", "raskest", "gjerne", "heller",
+	"helst", "dårligere", "dårligst", "vondt", "vondere", "vondest", "meget", "øverst", "enda", "neppe", "nokså", "nesten", "helt", "bitende",
+	"aller", "ganske", "aldeles", "derfra", "herfra", "utenlands", "noensteds", "oppå", "hjemme", "hit", "dit", "vekk", "fram", "fort", "hyggelig",
+	"hvorledes", "sånn", "således", "slik", "pent", "morsomt", "akkurat", "alt", "ofte", "nettopp", "bestandig", "noen gang", "noen ganger",
+	"fremdeles", "ennå", "da", "sjeldent" ];
+
+const interjections = [ "hei", "fy", "au", "hurra", "uff", "takk", "hm", "fanden", "pokker", "fillern", "åh", "isj", "hallo", "æsj" ];
+
+const recipeWords = [ "g" ];
+
+const timeWords = [ "år", "året", "går", "dag", "nå", "tid", "tiden", "morgen", "dager", "minutt", "minutter", "dagen", "uke",
+	"uker", "måneder", "stund", "timer", "time", "morges", "ettermiddag", "tidlig", "fjor", "kveld", "natt", "fogårs", "vinter",
+	"sommer", "vår", "høst" ];
+
+const vagueNouns = [ "ting", "tingene" ];
+
+const miscellaneous = [ "ok", "okay", "ja", "jo", "jaså", "nei", "ikke", "unnskyld", "beklager", "herr", "altså", "grader", "grad", "kr",
+	// Fractions.
+	"en halvdel", "en halv", "to halve", "en tredel", "tredjedel", "to tredeler", "tredjedeler", "en firedel", "fjerdedel",
+	"kvart", "en trettendedel", "en fjortendedel", "en promille", "en tusendel", "halvannen", "en og en halv" ];
+
+export const cannotBeBetweenPassiveAuxiliaryAndParticiple = transformWordsWithHyphens( auxiliariesAndDelexicalizedVerbs.concat( interviewVerbs ) );
+
+export const all = transformWordsWithHyphens( [].concat( articles, cardinalNumerals, ordinalNumerals, pronouns, interrogatives,
+	quantifiers, reflexivePronouns, indefinitePronouns, prepositions, conjunctions, interviewVerbs,
+	intensifiers, auxiliariesAndDelexicalizedVerbs, generalAdjectivesAdverbs, interjections, recipeWords,
+	timeWords, vagueNouns, miscellaneous, cannotBeBetweenPassiveAuxiliaryAndParticiple, transitionWords ) );
+
+export default all;