axyseo 2.0.0-alpha.0.0.2 → 2.0.0-alpha.0.0.21

package/src/languageProcessing/languages/ja/helpers/internal/SentenceTokenizer.js

@@ -1,102 +0,0 @@
-import { isNaN } from "lodash";
-
-import { normalize as normalizeQuotes } from "../../../../helpers/sanitize/quotes.js";
-import SentenceTokenizer from "../../../../helpers/sentence/SentenceTokenizer";
-
-/*
- * \u2026 - ellipsis.
- * \u3002 - Japanese ideographic full stop.
- * \uFF61 - Japanese half-width ideographic full stop.
- * \uFF01 - Japanese full-width exclamation mark.
- * \u203C - Japanese double exclamation mark.
- * \uFF1F - Japanese fullwidth question mark.
- * \u2047 - Japanese double question mark.
- * \u2049 - Japanese exclamation question mark.
- * \u2048 - Japanese question exclamation mark.
- * \u2025 - Japanese two dot leader.
- */
-const sentenceDelimiters = "?!\u2026\u3002\uFF61\uFF01\u203C\uFF1F\u2047\u2049\u2048\u2049\u2025";
-
-/**
- * Class for tokenizing a (html) text into sentences.
- */
-export default class JapaneseSentenceTokenizer extends SentenceTokenizer {
-	/**
-	 * Constructor
-	 * @constructor
-	 */
-	constructor() {
-		super();
-		this.sentenceDelimiters = sentenceDelimiters;
-	}
-
-	/**
-	 * Returns whether a certain character is a number.
-	 *
-	 * @param {string} character The character to check.
-	 * @returns {boolean} Whether or not the character is a capital letter.
-	 */
-	isNumber( character ) {
-		const japaneseNumbers = [
-			// Full-width.
-			/^[\uFF10-\uFF19]+$/i,
-			// Circled digit, parenthesized digit, and digit with full stop.
-			/^[\u2460-\u249B]+$/i,
-			// Parenthesized ideograph.
-			/^[\u3220-\u3229]+$/i,
-			// Circled ideograph.
-			/^[\u3280-\u3289]+$/i,
-		];
-
-		return ( ! isNaN( parseInt( character, 10 ) ) || japaneseNumbers.some( numberRange => numberRange.test( character ) ) );
-	}
-
-	/**
-	 * Returns whether a given character is quotation mark.
-	 *
-	 * @param {string} character The character to check.
-	 * @returns {boolean} Whether or not the given character is a quotation mark.
-	 */
-	isQuotation( character ) {
-		character = normalizeQuotes( character );
-
-		const japaneseOpeningQuotationMark = /^[\u300C\u300E\u3008\u3014\u3010\uFF5B\uFF3B]+$/i;
-
-		return "'" === character || "\"" === character || japaneseOpeningQuotationMark.test( character );
-	}
-
-	/**
-	 * Checks whether a character is from Japanese language that could be sentence beginning.
-	 *
-	 * @param {string} letter The letter to check.
-	 *
-	 * @returns {boolean} Whether the letter is from Japanese language that could be sentence beginning.
-	 */
-	isLetterFromSpecificLanguage( letter ) {
-		const japaneseLetterRanges = [
-			// Hiragana.
-			/^[\u3040-\u3096]+$/i,
-			// Katakana full-width.
-			/^[\u30A1-\u30FA]+$/i,
-			/^[\u31F0-\u31FF]+$/i,
-			// Katakana half-width.
-			/^[\uFF66-\uFF9D]+$/i,
-			// Kanji.
-			/^[\u4E00-\u9FFC]+$/i,
-		];
-
-		return (
-			japaneseLetterRanges.some( ltrLetterRange => ltrLetterRange.test( letter ) )
-		);
-	}
-
-	/**
-	 * Always returns true as Japanese sentence beginning doesn't need to be preceded by a whitespace to be a valid one.
-	 *
-	 * @returns {true}  Always true.
-	 */
-	isCharacterASpace() {
-		return true;
-	}
-}
-

package/src/languageProcessing/languages/ja/helpers/internal/createWordForms.js

@@ -1,68 +0,0 @@
-import { flatten, uniq } from "lodash";
-
-/**
- * Checks if the word matches any paradigm and creates the forms from the matched paradigm(s).
- * If no matches found, returns an array with the original word.
- *
- * @param {string}  word            The word to check.
- * @param {Object}  morphologyData  The morphology data.
- *
- * @returns {Array} The array of created forms.
- */
-function createForms( word, morphologyData ) {
-	const paradigmEndingsGroups = morphologyData.paradigmGroups;
-
-	// The endings in the array are sorted from the longest to the shortest.
-	// This way, if there are multiple matches, the first element in the matched array is always the longest one.
-	let allEndings = uniq( flatten( paradigmEndingsGroups ) );
-	allEndings = allEndings.sort( ( a, b ) => b.length - a.length || a.localeCompare( b ) );
-
-	// Check if the word matches any ending(s) and save the ending(s).
-	const matchedEndings = allEndings.filter( ending => word.endsWith( ending ) );
-
-	const forms = [];
-
-	if ( matchedEndings.length === 0 ) {
-		// If there is no match found, add the original word to the forms array.
-		forms.push( word );
-	} else {
-		// Pick the longest ending.
-		const matchedEnding = matchedEndings[ 0 ];
-
-		// Extract the stem.
-		const matchedStem = word.slice( 0, -matchedEnding.length );
-
-		// Loop over each endings group in the endings group array.
-		for ( const endingsGroup of paradigmEndingsGroups ) {
-			// Check if the ending of the matched word can be found in the endings group and create all the forms with all the endings in the group.
-			if ( endingsGroup.includes( matchedEnding ) ) {
-				forms.push( endingsGroup.map( ending => matchedStem + ending ) );
-			}
-		}
-	}
-
-	// If the final character of the input word ends in る, also create an extra form where the character is removed.
-	const ruEnding = "る";
-	if ( word.endsWith( ruEnding ) ) {
-		forms.push( word.slice( 0, -ruEnding.length ) );
-	}
-
-	return uniq( flatten( forms ) );
-}
-
-/**
- * Creates forms for Japanese word.
- *
- * @param {string}  word            The word to check. Assume that the input word is a segment as outputted from `getContentWords` helper.
- * @param {Object}  morphologyData  The morphology data.
- *
- * @returns {Array} The array of created forms.
- */
-export default function( word, morphologyData ) {
-	// Check if the word is longer than 1 character.
-	if ( word.length <= 1 ) {
-		return [ word ];
-	}
-
-	return createForms( word, morphologyData );
-}

package/src/languageProcessing/languages/ja/helpers/internal/determineStem.js

@@ -1,17 +0,0 @@
-import createWordForms from "./createWordForms";
-
-/**
- * Determines the canonical stem from the word forms.
- *
- * @param {string} word             The word input.
- * @param {object} morphologyData   The morphology data file.
- *
- * @returns {string}    The shortest form from the array as the canonical stem.
- */
-export default function determineStem( word, morphologyData ) {
-	let createdWordForms = createWordForms( word, morphologyData );
-	// Sort from the shortest to the longest form.
-	createdWordForms = createdWordForms.sort( ( a, b ) => a.length - b.length || a.localeCompare( b ) );
-
-	return createdWordForms[ 0 ];
-}

package/src/languageProcessing/languages/ja/helpers/matchTextWithWord.js

@@ -1,53 +0,0 @@
-import getContentWords from "./getContentWords";
-import processExactMatchRequest from "../../../helpers/match/processExactMatchRequest";
-import { normalizeSingle } from "../../../helpers/sanitize/quotes";
-
-/**
- * Checks for word matches in a text and returns an array containing the matched word(s).
- *
- * @param {string}  text               The text to find the word to match.
- * @param {string}  wordToMatch        The word to match.
- *
- * @returns {Array} An array of the matched word(s).
- */
-export default function( text, wordToMatch ) {
-	/*
-     * Lowercase the text so that it matches the wordToMatch which is already lowercased.
-     * Note that Japanese doesn't differentiate between upper and lower case, so this is only needed in case
-     * the text contains non-Japanese characters.
-     */
-	text = text.toLowerCase();
-
-	// Check if the exact match is requested.
-	const isExactMatchRequested = processExactMatchRequest( wordToMatch );
-	if ( isExactMatchRequested.exactMatchRequested ) {
-		/*
-		 * Normalize single quotes in case they differ between the text and the word to match.
-		 * Normalizing is only needed for exact matching, because with non-exact matching single quotes are considered
-		 * word boundaries.
-		 * Quotes in wordToMatch are already normalized at an earlier point.
-		*/
-		text = normalizeSingle( text );
-
-		const keyphrase = isExactMatchRequested.keyphrase;
-		const matches = [];
-
-		// Return the index of the match. It returns -1 if there is no match.
-		let index = text.indexOf( keyphrase );
-		while ( index !== -1 ) {
-			// Push the match to the array.
-			matches.push( keyphrase );
-			// Look for the next match after the previous one and adjust the index.
-			index = text.indexOf( keyphrase, index + keyphrase.length );
-		}
-		return matches;
-	}
-
-	/*
-	 * `getContentWords` is used here to retrieve the words from the text instead of `getWords` because it has an additional step
-	 * to remove this ending -じゃ from the segmented words which means that using this method will improve matching possibility.
-	 */
-	const words = getContentWords( text );
-
-	return words.filter( word => wordToMatch === word );
-}

package/src/languageProcessing/languages/ja/helpers/matchTransitionWords.js

@@ -1,25 +0,0 @@
-import getWords from "./getWords";
-
-/**
- * Matches the sentence against transition words.
- *
- * @param {string} sentence The sentence to match against.
- * @param {Array} transitionWords The array containing transition words.
- * @returns {Array} The found transitional words.
- */
-export default function( sentence, transitionWords ) {
-	// Retrieve the words from the segmenter, join into string for easier comparison through the contains function.
-	const words = getWords( sentence ).join( "|" );
-
-	const results = [];
-
-	transitionWords.forEach( function( transitionWord ) {
-		// The transitionWords are segmented, like the sentence, so join here as well.
-		const target = transitionWord.join( "|" );
-		if ( words.includes( target ) ) {
-			results.push( transitionWord );
-		}
-	} );
-
-	return results;
-}

package/src/languageProcessing/languages/ja/helpers/memoizedSentenceTokenizer.js

@@ -1,28 +0,0 @@
-import { memoize } from "lodash";
-import SentenceTokenizer from "./internal/SentenceTokenizer";
-
-/**
- * Returns the sentences from a certain text.
- *
- * @param {string} text 					The text to retrieve sentences from..
- * @param {boolean} [trimSentences=true] 	Whether to trim whitespace from the beginning and end of the sentences or not.
- *
- * @returns {Array<string>} The list of sentences in the text.
- */
-function getSentenceTokenizer( text, trimSentences = true ) {
-	const sentenceTokenizer = new SentenceTokenizer();
-	const { tokenizer, tokens } = sentenceTokenizer.createTokenizer();
-	sentenceTokenizer.tokenize( tokenizer, text );
-
-	return ( tokens.length === 0 ? [] : sentenceTokenizer.getSentencesFromTokens( tokens, trimSentences ) );
-}
-
-/*
- * The second argument to the memoize function is a so-called resolver function.
- * It creates a cache key consisting of a combination of all arguments to a function.
- * This is needed because by default, only the first argument to a function is used as the map cache key by the memoize function.
- * This means that a function is only re-run if the value of the first argument changes.
- * We want to re-run the getSentenceTokenizer function also when only the second argument changes to prevent cache collisions.
- * @see https://lodash.com/docs/4.17.15#memoize
- */
-export default memoize( getSentenceTokenizer, ( ...args ) => JSON.stringify( args ) );

package/src/languageProcessing/languages/ja/helpers/splitIntoTokensCustom.js

@@ -1,20 +0,0 @@
-import { map } from "lodash";
-import TinySegmenter from "tiny-segmenter";
-
-/**
- * Split sentence into tokens.
- *
- * @param {string} sentenceText The sentence text to split.
- *
- * @returns {Token[]} The tokens.
- */
-function splitIntoTokensCustom( sentenceText ) {
-	// Return empty string if sentence is empty.
-	if ( sentenceText === "" ) {
-		return [];
-	}
-	// Split sentences into words that are also tokens.
-	const words = new TinySegmenter().segment( sentenceText );
-	return map( words );
-}
-export default splitIntoTokensCustom;

package/src/languageProcessing/languages/ja/helpers/wordsCharacterCount.js

@@ -1,13 +0,0 @@
-/**
- * Calculates the characters length of all the words in the array.
- *
- * @param {array} wordsArray    The array of words to calculate.
- *
- * @returns {number} The characters length of all words in the array. Returns 0 if the input is an empty array.
- */
-export default function( wordsArray ) {
-	const counts = [];
-	wordsArray.map( form => counts.push( form.length ) );
-
-	return wordsArray.length === 0 ? 0 : counts.reduce( ( a, b ) => a + b );
-}

package/src/languageProcessing/languages/nb/Researcher.js

@@ -1,45 +0,0 @@
-import {languageProcessing} from '../../../index';
-const {AbstractResearcher} = languageProcessing;
-
-// All config
-import firstWordExceptions from './config/firstWordExceptions';
-import functionWords from './config/functionWords';
-import stopWords from './config/stopWords';
-import transitionWords from './config/transitionWords';
-import twoPartTransitionWords from './config/twoPartTransitionWords';
-
-// All helpers
-import getStemmer from './helpers/getStemmer';
-import getClauses from './helpers/getClauses';
-
-/**
- * The researcher contains all the researches, helpers, data, and config.
- */
-export default class Researcher extends AbstractResearcher {
-  /**
-   * Constructor
-   * @param {Paper} paper The Paper object that is needed within the researches.
-   * @constructor
-   */
-  constructor(paper) {
-    super(paper);
-
-    // Delete Flesch Reading Ease research since Norwegian doesn't have the support for it
-    delete this.defaultResearches.getFleschReadingScore;
-
-    Object.assign(this.config, {
-      language: 'nb',
-      passiveConstructionType: 'periphrastic',
-      functionWords,
-      firstWordExceptions,
-      transitionWords,
-      twoPartTransitionWords,
-      stopWords
-    });
-
-    Object.assign(this.helpers, {
-      getStemmer,
-      getClauses
-    });
-  }
-}

package/src/languageProcessing/languages/nb/config/firstWordExceptions.js

@@ -1,12 +0,0 @@
-/**
- *  Returns an array with exceptions for the sentence beginning researcher.
- *  @returns {Array} The array filled with exceptions.
- *  */
-export default [
-	// Indefinite articles:
-	"ei", "et",
-	// Numbers 1-10:
-	"en", "ett", "to", "tre", "fire", "fem", "seks", "sju", "syv", "åtte", "ni", "ti",
-	// Demonstrative pronouns:
-	"denne", "dette", "disse", "den", "det", "de",
-];

package/src/languageProcessing/languages/nb/config/functionWords.js

@@ -1,106 +0,0 @@
-import { singleWords as transitionWords } from "./transitionWords";
-import transformWordsWithHyphens from "../../../helpers/transform/transformWordsWithHyphens";
-
-/**
- * Returns an object with function words.
- *
- * @returns {Object} The object filled with various categories of function word arrays.
- */
-// "En" is already listed among the cardinal numbers, "de", "det" and "den" among the personal pronouns.
-const articles = [ "ei", "et" ];
-
-const cardinalNumerals = [ "null", "en", "ett", "ene", "to", "tre", "fire", "fem", "seks", "syv", "åtte", "ni", "ti", "elleve", "tolv",
-	"tretten", "fjorten", "femten", "seksten", "sytten", "atten", "nitten", "tjue", "tyve", "tjueen", "enogtyve", "tretti", "tredve",
-	"førti", "førr", "femti", "seksti", "sytti", "åtti", "nitti", "hundre", "hundreogen", "etthundreogen", "tohundre", "tusen",
-	"tusenogen", "million", "millioner", "milliard", "milliarder" ];
-
-const ordinalNumerals = [ "nullte", "første", "først", "sekund", "tredje", "fjerde", "femte", "sjette", "syvende", "åttende", "niende", "tiende",
-	"ellevte", "tolvte", "trettende", "fjortende", "femtende", "sekstende", "syttende", "åttende", "nittende", "tjuende", "tjueførst", "tjueførste",
-	"trettiende", "førtiende", "femtiende", "sekstiende", "syttiende", "åttiende", "nittiende", "hundrede", "hundreogfemtiende", "to hundrede",
-	"tusende", "millionte", "millardte" ];
-
-const pronouns = [
-	// Personal pronouns.
-	"jeg", "du", "den", "det", "vi", "de", "han", "hun", "dere", "henne", "oss", "meg", "deg", "ham", "dem",
-	// Possessive pronouns.
-	"min", "din", "deres", "vår", "deres", "ditt", "mitt", "våre", "vårt", "hans", "hennes", "dens", "dets", "egen",
-	"egne", "mi", "di", "sin", "si", "sitt", "sine", "mine", "dine",
-	// Demonstrative pronouns. "De", "det" and "den" are among the personal pronouns.
-	"denne", "dette", "disse", "slik", "slikt", "slike", "sånn", "sånt", "sånne", "samme",
-	// Reciprocal pronouns.
-	"hverandre", "hvert",
-	// Relative pronouns.
-	"som" ];
-
-const interrogatives = [ "hvem", "hvordan", "hvorfor", "hvor", "hva", "hvilken", "hvilket", "hvilke" ];
-
-const quantifiers = [ "mange", "mye", "mang en", "mangt et", "hele", "mer", "ingen", "ingenting", "ikke noen", "ikke noe",
-	"alle", "all", "alt", "allting", "noen", "noe", "flere", "hver", "hvert", "annenhver", "ammethvert", "begge", "sov", "mest", "fleste",
-	"få", "fæst", "færrest", "flere", "flest" ];
-
-const reflexivePronouns = [ "seg", "selv" ];
-
-const indefinitePronouns = [ "ingenting", "annen", "annet", "andre" ];
-
-const prepositions = [ "sånn", "ved", "mot", "ned", "enn", "over", "inn", "i", "sa", "opp", "der", "fra", "din", "nei", "mellom", "di", "oppe",
-	"av", "med", "til", "å", "på", "du", "uten", "én", "under", "hos", "inne", "gjennom", "unna", "del", "nede", "til", "over", "under", "etter",
-	"kun", "blant", "for", "mellom", "blant" ];
-
-const conjunctions = [ "eller", "hvis", "ja", "et", "som", "i", "og", "både", "men", "mens", "enten", "verken", "at", "om", "da", "når", "før",
-	"idet", "etter at", "siden", "innen", "med det samme", "til", "inntil", "hver gang", "etter hvert som", "så lenge", "så lenge som", "så ofte",
-	"så ofte som", "så snart", "så snart som", "etter", "etterpå", "foran", "tidligere", "fordi", "ettersom", "derfor", "dersom", "hvis", "så fremt",
-	"så sant", "i fall", "i tilfelle", "med mindre", "uten at", "bare", "for så vidt som", "uten at", "uten å", "enda", "fordi om", "enda om",
-	"skjønt", "om enn", "hva så", "trass i at", "hvor så", "samme", "selv om", "hva enn", "til tross for at", "hvor enn", "uansett", "for at", "så",
-	"så at", "slik at", "sånn at", "for at, så", "slik som", "så som", "som om", "enn", "dess", "jo", "desto" ];
-
-const interviewVerbs = [ "tror", "fortelle", "fortell", "fortalte", "tenkte", "tenk" ];
-
-const intensifiers = [ "virkelig", "akkurat", "visst" ];
-
-const auxiliariesAndDelexicalizedVerbs = [ "ha", "har", "hadde", "gjør", "gjøre", "gjorde", "kaller", "kalte", "kalle",
-	"kalla", "virker", "virka", "virke", "virka", "går", "gikk", "gå", "leges", "lages", "legges", "lages", "består",
-	"bestod", "bestå", "bestått", "bety", "betyr" ];
-
-const generalAdjectivesAdverbs = [
-	// General adjective.
-	"helt", "andre", "litt", "lenge", "siste", "fint", "annet", "stor", "stort", "store", "neste", "lenger", "annen", "nye",
-	"alene", "flott", "gammel", "gammelt", "gamle", "klart", "liten", "langt", "gamle", "dårlig", "hyggelig", "gode", "sånt",
-	"nytt", "best", "lang", "små", "lot", "større", "vakker", "vakkert", "vakre", "ny", "bra", "bedre", "grei", "greit", "greie",
-	"høyt", "største", "størst", "slikt", "liten", "lita", "lite", "små", "mindre", "minst", "kort", "glad", "dårlig", "ille", "ond",
-	"vond", "verre", "verst", "eldre", "eldst", "lang", "lengre", "lengst", "nær", "næmerere", "nærere", "nærmest",
-	"nærest", "tung", "tyngre", "tyngst", "ung", "yngre", "yngst", "pen",
-	// General adverbs.
-	"alltid", "godt", "sammen", "tilbake", "etter", "igjen", "bare", "så", "veldig", "bedre", "samme", "far", "eneste", "enig",
-	"borte", "snart", "rundt", "beste", "bort", "vekk", "nesten", "ganske", "senere", "videre", "straks", "svært", "neste",
-	"bak", "bakre", "bakerst", "borte", "bortre", "bortest", "fremme", "fremre", "fremst", "foran", "forrest", "inne", "indre", "innerst",
-	"midt", "midtre", "midterst", "nede", "nedre", "nederst", "nord", "nordre", "nordligst", "øvre", "øverst", "sør", "søndre", "sørligst",
-	"vest", "vestre", "vestligst", "øst", "østre", "østligst", "ute", "ytre", "ytterst", "underst", "langt", "fram", "her", "der", "nok", "aldri",
-	"ut", "ned", "nede", "bort", "innom", "ingensteds", "sjelden", "sjeldnere", "sjeldnest", "raskt", "raskere", "raskest", "gjerne", "heller",
-	"helst", "dårligere", "dårligst", "vondt", "vondere", "vondest", "meget", "øverst", "enda", "neppe", "nokså", "nesten", "helt", "bitende",
-	"aller", "ganske", "aldeles", "derfra", "herfra", "utenlands", "noensteds", "oppå", "hjemme", "hit", "dit", "vekk", "fram", "fort", "hyggelig",
-	"hvorledes", "sånn", "således", "slik", "pent", "morsomt", "akkurat", "alt", "ofte", "nettopp", "bestandig", "noen gang", "noen ganger",
-	"fremdeles", "ennå", "da", "sjeldent" ];
-
-const interjections = [ "hei", "fy", "au", "hurra", "uff", "takk", "hm", "fanden", "pokker", "fillern", "åh", "isj", "hallo", "æsj" ];
-
-const recipeWords = [ "g" ];
-
-const timeWords = [ "år", "året", "går", "dag", "nå", "tid", "tiden", "morgen", "dager", "minutt", "minutter", "dagen", "uke",
-	"uker", "måneder", "stund", "timer", "time", "morges", "ettermiddag", "tidlig", "fjor", "kveld", "natt", "fogårs", "vinter",
-	"sommer", "vår", "høst" ];
-
-const vagueNouns = [ "ting", "tingene" ];
-
-const miscellaneous = [ "ok", "okay", "ja", "jo", "jaså", "nei", "ikke", "unnskyld", "beklager", "herr", "altså", "grader", "grad", "kr",
-	// Fractions.
-	"en halvdel", "en halv", "to halve", "en tredel", "tredjedel", "to tredeler", "tredjedeler", "en firedel", "fjerdedel",
-	"kvart", "en trettendedel", "en fjortendedel", "en promille", "en tusendel", "halvannen", "en og en halv" ];
-
-export const cannotBeBetweenPassiveAuxiliaryAndParticiple = transformWordsWithHyphens( auxiliariesAndDelexicalizedVerbs.concat( interviewVerbs ) );
-
-export const all = transformWordsWithHyphens( [].concat( articles, cardinalNumerals, ordinalNumerals, pronouns, interrogatives,
-	quantifiers, reflexivePronouns, indefinitePronouns, prepositions, conjunctions, interviewVerbs,
-	intensifiers, auxiliariesAndDelexicalizedVerbs, generalAdjectivesAdverbs, interjections, recipeWords,
-	timeWords, vagueNouns, miscellaneous, cannotBeBetweenPassiveAuxiliaryAndParticiple, transitionWords ) );
-
-export default all;