axyseo 2.0.0-alpha.0.0.19 → 2.0.0-alpha.0.0.2

package/src/languageProcessing/languages/he/config/passiveVoice/regularRootsNifal.js

@@ -0,0 +1,197 @@
+/**
+ * Returns a list of roots following the binyan nif'al.
+ *
+ * @returns {string[]} Hebrew roots following the binyan nif'al.
+ */
+export default function() {
+	return [
+		"אכל",
+		"אמר",
+		"לקח",
+		"נשא",
+		"נתן",
+		"קרא",
+		"שלח",
+		"שמע",
+		"אהב",
+		"אסף",
+		"כרת",
+		"כתב",
+		"מצא",
+		"פקד",
+		"שמר",
+		"שפט",
+		"אבד",
+		"אמן",
+		"גאל",
+		"דרש",
+		"הרג",
+		"חשב",
+		"טמא",
+		"יתר",
+		"לכד",
+		"ספר",
+		"עזב",
+		"קבץ",
+		"קבר",
+		"רדף",
+		"שרף",
+		"שאל",
+		"שבר",
+		"שחת",
+		"שכח",
+		"שלם",
+		"שפך",
+		"אסר",
+		"בחר",
+		"בלע",
+		"בקע",
+		"ברא",
+		"דרך",
+		"זרע",
+		"חלק",
+		"חנן",
+		"חרש",
+		"טהר",
+		"למד",
+		"מכר",
+		"משל",
+		"סגר",
+		"סתר",
+		"עזר",
+		"ערך",
+		"פרד",
+		"פרש",
+		"צפה",
+		"קנה",
+		"קרע",
+		"רחץ",
+		"רפא",
+		"שבע",
+		"שחט",
+		"שמר",
+		"תפש",
+		"תקע",
+		"אשם",
+		"בגד",
+		"בדל",
+		"בחן",
+		"גזל",
+		"גמל",
+		"גנב",
+		"הרס",
+		"זרק",
+		"חבא",
+		"חבש",
+		"חלץ",
+		"חצב",
+		"חקר",
+		"חרב",
+		"חתם",
+		"טמן",
+		"טרף",
+		"לקט",
+		"מנע",
+		"סלח",
+		"סמך",
+		"ספר",
+		"עצר",
+		"פגע",
+		"פלט",
+		"פרד",
+		"פרץ",
+		"פשט",
+		"צפן",
+		"קצר",
+		"קשר",
+		"רעש",
+		"רצח",
+		"שחק",
+		"שטף",
+		"אטם",
+		"אלם",
+		"ארג",
+		"בצע",
+		"גאל",
+		"גדר",
+		"גזז",
+		"גזר",
+		"גלח",
+		"גרע",
+		"דחה",
+		"דקר",
+		"חבל",
+		"חרש",
+		"חכה",
+		"חפר",
+		"חקק",
+		"חרץ",
+		"חשק",
+		"טבח",
+		"טבל",
+		"טבע",
+		"כבש",
+		"כלא",
+		"לחץ",
+		"מחץ",
+		"מסס",
+		"משל",
+		"נשא",
+		"סחר",
+		"סלל",
+		"סער",
+		"סקל",
+		"סתם",
+		"עטף",
+		"עכר",
+		"פרע",
+		"פרק",
+		"רמס",
+		"שאב",
+		"שאף",
+		"שבר",
+		"שכר",
+		"שלל",
+		"שלף",
+		"שקל",
+		"שקף",
+		"שתל",
+		"תכן",
+		"תמך",
+		"תעב",
+		"שפכ‎",
+		"דרכ‎",
+		"ערכ‎",
+		"סמכ‎",
+		"תמכ‎",
+		"נתנ",
+		"אמנ",
+		"חננ",
+		"בחנ",
+		"טמנ",
+		"צפנ",
+		"תכנ",
+		"שלמ",
+		"אשמ",
+		"חתמ",
+		"אטמ",
+		"אלמ",
+		"סתמ",
+		"אספ",
+		"רדפ",
+		"שרפ",
+		"טרפ",
+		"שטפ",
+		"עטפ",
+		"שאפ",
+		"שלפ",
+		"שקפ",
+		"קבצ",
+		"רחצ",
+		"חלצ",
+		"פרצ",
+		"חרצ",
+		"לחצ",
+		"מחצ",
+	];
+}

package/src/languageProcessing/languages/he/config/passiveVoice/regularRootsPual.js

@@ -0,0 +1,170 @@
+/**
+ * Returns a list of roots following the binyan pu'al.
+ *
+ * @returns {string[]} Hebrew roots following the binyan pu'al.
+ */
+export default function() {
+	return [
+		"אכל",
+		"דבר",
+		"עבר",
+		"שלח",
+		"בקש",
+		"ברך",
+		"חזק",
+		"חטא",
+		"כתב",
+		"נצל",
+		"עבר",
+		"פקד",
+		"רום",
+		"שמר",
+		"אבד",
+		"אמן",
+		"בטח",
+		"גדל",
+		"הלל",
+		"חלל",
+		"חשב",
+		"טמא",
+		"יתר",
+		"כבד",
+		"כסה",
+		"כפר",
+		"לחם",
+		"לכד",
+		"נבא",
+		"נגע",
+		"סבב",
+		"ספר",
+		"קבץ",
+		"קדש",
+		"קרב",
+		"שמח",
+		"שבר",
+		"שכן",
+		"שלם",
+		"שרת",
+		"בער",
+		"בקע",
+		"ברא",
+		"חלק",
+		"חנן",
+		"טהר",
+		"כבס",
+		"למד",
+		"מרר",
+		"מהר",
+		"מלט",
+		"נכר",
+		"נצח",
+		"פלל",
+		"פרש",
+		"צפה",
+		"קלל",
+		"רחם",
+		"רחק",
+		"רנן",
+		"רפא",
+		"שמר",
+		"בדל",
+		"בצר",
+		"בשל",
+		"גרש",
+		"דמם",
+		"זמר",
+		"חבר",
+		"מאן",
+		"נשק",
+		"נתץ",
+		"נתק",
+		"ספר",
+		"ערב",
+		"פשט",
+		"צרף",
+		"קדם",
+		"קנא",
+		"קצר",
+		"קשר",
+		"רגל",
+		"שחק",
+		"בצע",
+		"ברר",
+		"בשר",
+		"גדר",
+		"גלל",
+		"דכא",
+		"דשן",
+		"חבל",
+		"חבק",
+		"חמם",
+		"חפש",
+		"חשק",
+		"כהן",
+		"כזב",
+		"כנס",
+		"מסס",
+		"מעט",
+		"מרר",
+		"נגן",
+		"נדב",
+		"נהל",
+		"נחש",
+		"נער",
+		"נפח",
+		"נפץ",
+		"נשך",
+		"סכך",
+		"סכן",
+		"סקל",
+		"עלל",
+		"ענג",
+		"עצב",
+		"פאר",
+		"פזר",
+		"פצה",
+		"פקח",
+		"פרק",
+		"פתה",
+		"קבל",
+		"קדד",
+		"קצץ",
+		"רמה",
+		"רצץ",
+		"רקע",
+		"שבח",
+		"שבר",
+		"שנה",
+		"שקר",
+		"שקף",
+		"תכן",
+		"תעב",
+		"ברכ‎",
+		"נשכ‎",
+		"סככ‎",
+		"אמנ",
+		"שכנ",
+		"חננ",
+		"רננ",
+		"מאנ",
+		"דשנ",
+		"כהנ",
+		"נגנ",
+		"סכנ",
+		"תכנ",
+		"רומ",
+		"לחמ",
+		"שלמ",
+		"רחמ",
+		"דממ",
+		"קדמ",
+		"חממ",
+		"צרפ",
+		"שקפ",
+		"קבצ",
+		"נתצ",
+		"קצצ",
+		"רצצ",
+		"נפצ",
+	];
+}

package/src/languageProcessing/languages/he/config/prefixedFunctionWords.js

@@ -0,0 +1,2 @@
+export const PREFIXED_FUNCTION_WORDS = [ "ב", "ה", "ו", "כ", "ל", "מ", "ש" ];
+export const PREFIXED_FUNCTION_WORDS_REGEX = new RegExp( `^(${PREFIXED_FUNCTION_WORDS.join( "|" )})` );

package/src/languageProcessing/languages/he/config/sentenceLength.js

@@ -0,0 +1,3 @@
+export default {
+	recommendedLength: 15,
+};

package/src/languageProcessing/languages/he/config/transitionWords.js

@@ -0,0 +1,28 @@
+/** @module config/transitionWords */
+
+export const singleWords = [ "לכן", "משום‎", "בגלל", "מפני", "אחרי", "לפני", "חוץ", "בזכות", "כתוצאה", "הודות", "בשביל", "למרות‎",
+	"בשל‎", "ו", "או", "אבל", "אולם", "אך", "אם", "גם", "רק", "אכן", "אלא", "עדיין", "כאשר", "אז", "לכן", "כבר", "לאחר",
+	"אפילו", "אף", "כך", "כדי", "על", "עד", "בשנת", "כמו", "שני", "באופן", "במהלך", "במקום", "וכן", "בעיקר", "מאז", " בינתיים",
+	"במקום", "תחת", "מתוך", "מול", "כגון", "באמצעות", "מכן", "במשך", "ואף", "ועל", "לעתים", "בנוסף", "בעקבות",
+	"לפי", "בקרב", "כי", "ראשית", "שנית", "תחילה", "לבסוף", "הבא", "ברם", "ואילו", "להפך", "כנגד", "מנגד", "אמנם",
+	"אדרבא", "לחילופין", "בייחוד", "במיוחד", "ודאי", "ואפילו", "לו", "אילו", "לולא", "אלמלא", "אילולא", "מאחורי", "בקרבת", "כאן",
+	"שם", "כן", "למעט", "בלבד", "מלבד", "שוב", "כלומר", "דהיינו", "לאמור", "כאמור", "כידוע", "כש", "אחר-כך", "כעבור",
+	"לאחרונה", "בטרם", "עכשיו", "עתה", "מלכתחילה", "למען", "פן", "לבל", "שמא", "עקב", "לרגל", "מפאת", "בגין", "בהתאם",
+	"לפיכך", "למשל", "לדוגמה", "כדוגמת", "לסיכום", "בהתחשב", "בקיצור", "בקצרה", "חרף" ];
+export const multipleWords = [ "כתוצאה מכך", "כתוצאה מ", "בעקבות ה", "בעקבות זאת", "לעומת זאת", "אלא אם כן", "בזמן ש", "מתי ש",
+	"אף על פי ש", "אף על פי", "חוץ מ", "אחרי ש", "לפני ש", "בעוד ש", "בגלל ה", "הודות ל", "בניגוד ל", "מפני ש", "אלא ש",
+	"קודם כל", "והחשוב ביותר", "לפני כן", "לאחר מכן", "אחר כך", "שלב ראשון", "בניגוד לכך", "ראוי לציין", "יש להדגיש", "ללא ספק",
+	"קל וחומר", "אין ספק ש", "לא כל שכן", "בתנאי ש", "בדומה ל", "כשם ש", "כפי ש", "כך גם", "יחסית ל", "בהשוואה ל", "לשם כך",
+	"במקביל ל", "במידה ש", "במקום ש", "על יד", "בסמוך ל", "במרחק מה", "אל אשר", "מעבר ל", "כמו כן", "יתר על כן", "זאת ועוד",
+	"נוסף על כך", "פרט ל", "למעלה מכך", "פנים נוספות לעניין", "יתרה מכך", "אך ורק", "מעבר לכך", "זאת אומרת", "במילים אחרות",
+	"רוצה לומר", "הווה אומר", "משתמע מזאת", "הוא אשר", "ללמד ש", "פירושו של דבר", "לשון אחרת", "בהקשר זה", "דרך אגב", "כדי ש",
+	"כנזכר לעיל", "בקשר לכך", "במסגרת זו", "עד ש", "בשעה ש", "כל זמן ש", "לפי שעה", "בזמן האחרון", "עד כה", "בו בזמן", "כל אימת ש",
+	"על מנת", "לתכלית זו", "במטרה ל", "מחמת ה", "הואיל ו", "מאחר ש", "היות ש", "כיוון ש", "יען כי", "באופן ש", "בצורה ש", "כמו ש",
+	"אם כן", "אפוא", "על כן", "משום כך", "עקב כך", "בשל כך", "אי לכך", "נובע מכך", "הודות לכך", "כמו למשל", "סיכומו של דבר",
+	"מכל האמור ניתן לומר", "בסך הכול", "בכל מקרה", "בסיכום כולל", "לטווח ארוך", "על כל פנים", "אף על פי כן", "על אף ש", "על אף זאת",
+	"אף ש", "למרות זאת", "בכל אופן", "עם זאת", "אם כי", "גם אם" ];
+
+
+export const allWords = singleWords.concat( multipleWords );
+
+export default allWords;

package/src/languageProcessing/languages/he/config/twoPartTransitionWords.js

@@ -0,0 +1,8 @@
+/** @module config/twoPartTransitionWords */
+
+/**
+ * Returns an array with two-part transition words to be used by the assessments.
+ * @returns {Array}              The array filled with two-part transition words.
+ */
+export default  [ [ "או", "או" ], [ "אין", "אלא" ], [ "לא", "אלא" ], [ "מצד אחד", "מצד אחר" ], [ "מחד גיסא", "מאידך גיסא" ],
+	[ "ראשית", "שנית" ], [ "תחילה", "לבסוף" ], [ "משום ש", "מפני ש" ], [ "לכאורה", "אך" ] ];

package/src/languageProcessing/languages/he/helpers/createBasicWordForms.js

@@ -0,0 +1,33 @@
+import { stemPrefixedFunctionWords } from "../../../helpers/morphology/stemPrefixedFunctionWords.js";
+import { PREFIXED_FUNCTION_WORDS, PREFIXED_FUNCTION_WORDS_REGEX } from "../config/prefixedFunctionWords.js";
+
+/**
+ * Creates basic word forms for a given Hebrew word.
+ *
+ * @param {string} word     The word for which to create basic word forms.
+ *
+ * @returns {string[]} Prefixed and de-prefixed variations of a word.
+ */
+export function createBasicWordForms( word ) {
+	const forms = [];
+
+	/*
+	 * Add prefixes to the input word. We always do this, since some words
+	 * beginning with a prefix-like letter might be exceptions where this is the
+	 * actual first letter of the word.
+	 */
+	forms.push( ...PREFIXED_FUNCTION_WORDS.map( basicPrefix => basicPrefix + word ) );
+
+	/*
+	 * If a word starts with one of the prefixes, we strip it and create all possible
+	 * prefixed forms based on this stem.
+	 */
+	const { stem, prefix } = stemPrefixedFunctionWords( word, PREFIXED_FUNCTION_WORDS_REGEX );
+
+	if ( prefix !== "" ) {
+		forms.push( stem );
+		forms.push( ...PREFIXED_FUNCTION_WORDS.map( basicPrefix => basicPrefix + stem ) );
+	}
+
+	return forms;
+}

package/src/languageProcessing/languages/he/helpers/getStemmer.js

@@ -0,0 +1,22 @@
+import {get} from 'lodash';
+import {languageProcessing} from '../../../index';
+const {baseStemmer} = languageProcessing;
+
+import stem from './internal/stem';
+
+/**
+ * Returns the stemmer for a researcher.
+ *
+ * @param {Researcher} researcher The researcher.
+ *
+ * @returns {Function} The stemmer.
+ */
+export default function getStemmer(researcher) {
+  const morphologyData = get(researcher.getData('morphology'), 'he', false);
+
+  if (morphologyData) {
+    return word => stem(word, morphologyData);
+  }
+
+  return baseStemmer;
+}

package/src/languageProcessing/languages/he/helpers/internal/stem.js

@@ -0,0 +1,52 @@
+/**
+ *
+ * If the word starts with one of the prefixes "ב" "ה", "ו", "כ", "ל", "מ", "ש" it should be stemmed as the dictionary
+ * does not contain forms with those prefixes.
+ *
+ * @param {string} 		word		The word to check for a prefix.
+ * @param {string[]} 	prefixes	The prefixes that should be stemmed before checking the dictionary.
+ *
+ * @returns {string} Word without the prefix or the original word if no prefix was found
+ */
+const removePrefix = function( word, prefixes ) {
+	if ( prefixes.some( prefix => word.startsWith( prefix ) ) ) {
+		return word.slice( 1 );
+	}
+	return word;
+};
+
+/**
+ * Stems Hebrew words (removes possible prefixes and returns lemma if found in the dictionary).
+ *
+ * @param {string}	word			The word to stem.
+ * @param {Object}	morphologyData	The Hebrew morphology data.
+ *
+ * @returns {string} The stemmed word or the original word if no stem was found.
+ */
+export default function stem( word, morphologyData ) {
+	const dictionaryStemmer = morphologyData.dictionary;
+
+	// Check if the word exists in the dictionary stemmer. If yes, return base form of the word specified in the dictionary.
+	let stemmedWord = dictionaryStemmer[ word ];
+	if ( stemmedWord ) {
+		return stemmedWord;
+	}
+
+	// If the word was not found in the dictionary, try to remove a prefix from the word and see whether the deprefixed word is found.
+	const wordAfterRemovingPrefix = removePrefix( word, morphologyData.prefixes );
+	if ( wordAfterRemovingPrefix !== word ) {
+		stemmedWord = dictionaryStemmer[ wordAfterRemovingPrefix ];
+		if ( stemmedWord ) {
+			return stemmedWord;
+		}
+		// If a prefix was removed but the word was still not found, try removing another prefix and search in the dictionary again.
+		const wordAfterRemovingSecondPrefix = removePrefix( wordAfterRemovingPrefix, morphologyData.prefixes );
+		if ( wordAfterRemovingSecondPrefix !== wordAfterRemovingPrefix  ) {
+			stemmedWord = dictionaryStemmer[ wordAfterRemovingSecondPrefix ];
+			if ( stemmedWord ) {
+				return stemmedWord;
+			}
+		}
+	}
+	return word;
+}

package/src/languageProcessing/languages/he/helpers/isPassiveSentence.js

@@ -0,0 +1,96 @@
+import {languageProcessing} from '../../../index';
+const {getWords} = languageProcessing;
+
+import regularRootsHufal from '../config/internal/regularRootsHufal';
+import regularRootsNifal from '../config/internal/regularRootsNifal';
+import regularRootsPual from '../config/internal/regularRootsPual';
+
+/**
+ * Checks if the input word's root is in the Hebrew verb roots list.
+ *
+ * @param {string} word             The word to check.
+ * @param {string[]} verbRootsList  The Hebrew verb roots list.
+ * @param {Object[]} affixesList    The list of prefixes and suffixes.
+ *
+ * @returns {Boolean}           Returns true if the root of the input word is in the list.
+ */
+const checkHebrewVerbRootsList = function(word, verbRootsList, affixesList) {
+  return verbRootsList.some(root =>
+    affixesList.some(function(affixes) {
+      const pattern = new RegExp('^' + affixes.prefix + root + affixes.suffix + '$');
+      return pattern.test(word);
+    })
+  );
+};
+
+/**
+ * Checks the passed sentence to see if it contains Hebrew passive verb-forms.
+ *
+ * @param {string} sentence    The sentence to match against.
+ *
+ * @returns {Boolean}          Whether the sentence contains Hebrew passive voice.
+ */
+export default function isPassiveSentence(sentence) {
+  const words = getWords(sentence);
+  for (const word of words) {
+    // The list of prefixes and suffixes for nif'al.
+    const nifalAffixes = [
+      {prefix: '(נ|אי|תי|הי|יי|ני|להי)', suffix: ''},
+      {prefix: '(תי|הי)', suffix: '(י|ו|נה)'},
+      {prefix: 'נ', suffix: '(ים|ת|ות|תי|ה|נו|תם|תן|ו)'},
+      {prefix: 'יי', suffix: 'ו'}
+    ];
+
+    // Check if the root is in nif'al.
+    const nifalPassive = checkHebrewVerbRootsList(word, regularRootsNifal, nifalAffixes);
+
+    if (nifalPassive) {
+      return true;
+    }
+
+    // The list of prefixes and suffixes for pu'al.
+    const pualAffixes = [
+      {prefix: '(מ|א|ת|י|נ)', suffix: ''},
+      {prefix: 'תי', suffix: 'נה'},
+      {prefix: 'מ', suffix: '(ת|ים|ות)'},
+      {prefix: 'ת', suffix: '(י|ו|נה)'},
+      {prefix: 'י', suffix: 'ו'},
+      {prefix: '', suffix: '(תי|ת|ה|נו|תם|תן|ו)'},
+      {prefix: '', suffix: ''}
+    ];
+    const pualInfix = 'ו';
+
+    // Check if the root is in pu'al.
+    const pualPassive = regularRootsPual.some(root =>
+      pualAffixes.some(function(affixes) {
+        const pualPattern = new RegExp(
+          '^' + affixes.prefix + root[0] + pualInfix + root[1] + root[2] + affixes.suffix + '$'
+        );
+
+        return pualPattern.test(word);
+      })
+    );
+
+    if (pualPassive) {
+      return true;
+    }
+
+    // The list of prefixes and suffixes for huf'al.
+    const hufalAffixes = [
+      {prefix: '(מו|הו|או|תו|יו|נו)', suffix: ''},
+      {prefix: 'מו', suffix: '(ת|ים|ות)'},
+      {prefix: 'הו', suffix: '(תי|ת|ית|ה|נו|תם|תן|ו)'},
+      {prefix: 'תו', suffix: '(ו|נה|י)'},
+      {prefix: 'יו', suffix: 'ו'}
+    ];
+
+    // Check if the root is in huf'al.
+    const hufalPassive = checkHebrewVerbRootsList(word, regularRootsHufal, hufalAffixes);
+
+    if (hufalPassive) {
+      return true;
+    }
+  }
+
+  return false;
+}

package/src/languageProcessing/languages/he/helpers/stem.js

@@ -0,0 +1,52 @@
+/**
+ *
+ * If the word starts with one of the prefixes "ב" "ה", "ו", "כ", "ל", "מ", "ש" it should be stemmed as the dictionary
+ * does not contain forms with those prefixes.
+ *
+ * @param {string} 		word		The word to check for a prefix.
+ * @param {string[]} 	prefixes	The prefixes that should be stemmed before checking the dictionary.
+ *
+ * @returns {string} Word without the prefix or the original word if no prefix was found
+ */
+const removePrefix = function( word, prefixes ) {
+	if ( prefixes.some( prefix => word.startsWith( prefix ) ) ) {
+		return word.slice( 1 );
+	}
+	return word;
+};
+
+/**
+ * Stems Hebrew words (removes possible prefixes and returns lemma if found in the dictionary).
+ *
+ * @param {string}	word			The word to stem.
+ * @param {Object}	morphologyData	The Hebrew morphology data.
+ *
+ * @returns {string} The stemmed word or the original word if no stem was found.
+ */
+export default function stem( word, morphologyData ) {
+	const dictionaryStemmer = morphologyData.dictionary;
+
+	// Check if the word exists in the dictionary stemmer. If yes, return base form of the word specified in the dictionary.
+	let stemmedWord = dictionaryStemmer[ word ];
+	if ( stemmedWord ) {
+		return stemmedWord;
+	}
+
+	// If the word was not found in the dictionary, try to remove a prefix from the word and see whether the deprefixed word is found.
+	const wordAfterRemovingPrefix = removePrefix( word, morphologyData.prefixes );
+	if ( wordAfterRemovingPrefix !== word ) {
+		stemmedWord = dictionaryStemmer[ wordAfterRemovingPrefix ];
+		if ( stemmedWord ) {
+			return stemmedWord;
+		}
+		// If a prefix was removed but the word was still not found, try removing another prefix and search in the dictionary again.
+		const wordAfterRemovingSecondPrefix = removePrefix( wordAfterRemovingPrefix, morphologyData.prefixes );
+		if ( wordAfterRemovingSecondPrefix !== wordAfterRemovingPrefix  ) {
+			stemmedWord = dictionaryStemmer[ wordAfterRemovingSecondPrefix ];
+			if ( stemmedWord ) {
+				return stemmedWord;
+			}
+		}
+	}
+	return word;
+}