nlptoolkit-morphologicalanalysis 1.0.13 → 1.0.15

package/source/MorphologicalAnalysis/FsmMorphologicalAnalyzer.ts

@@ -15,15 +15,18 @@ import {Sentence} from "nlptoolkit-corpus/dist/Sentence";
 import {Word} from "nlptoolkit-dictionary/dist/Dictionary/Word";
 import {State} from "./State";
 import {Queue} from "nlptoolkit-datastructure/dist/Queue";
+import {FileUtils} from "nlptoolkit-util/dist/FileUtils";
 
 export class FsmMorphologicalAnalyzer {
 
     private dictionaryTrie: Trie
+    private suffixTrie: Trie
     private parsedSurfaceForms: Map<string, string> = undefined
-    private finiteStateMachine: FiniteStateMachine
+    private pronunciations: Map<string, string> = undefined
+    private readonly finiteStateMachine: FiniteStateMachine
     private static MAX_DISTANCE = 2
-    private dictionary: TxtDictionary
-    private cache: LRUCache<string, FsmParseList> = undefined
+    private readonly dictionary: TxtDictionary
+    private readonly cache: LRUCache<string, FsmParseList> = undefined
     private mostUsedPatterns: Map<string, RegExp> = new Map<string, RegExp>()
 
     /**
@@ -51,22 +54,59 @@ export class FsmMorphologicalAnalyzer {
         } else {
             this.finiteStateMachine = new FiniteStateMachine(fileName);
         }
+        this.prepareSuffixTrie();
         this.dictionaryTrie = this.dictionary.prepareTrie();
         if (cacheSize > 0){
             this.cache = new LRUCache<string, FsmParseList>(cacheSize);
         }
+        this.addPronunciations("pronunciations.txt");
     }
 
-    addParsedSurfaceForms(fileName: string){
-        this.parsedSurfaceForms = new Map<string, string>()
-        let data = fs.readFileSync(fileName, 'utf8')
+    /**
+     * Constructs and returns the reverse string of a given string.
+     * @param s String to be reversed.
+     * @return Reverse of a given string.
+     */
+    private reverseString(s: string): string{
+        let result = ""
+        for (let i = s.length - 1; i >= 0; i--){
+            result += s[i]
+        }
+        return result
+    }
+
+    /**
+     * Constructs the suffix trie from the input file suffixes.txt. suffixes.txt contains the most frequent 6000
+     * suffixes that a verb or a noun can take. The suffix trie is a trie that stores these suffixes in reverse form,
+     * which can be then used to match a given word for its possible suffix content.
+     */
+    private prepareSuffixTrie(){
+        this.suffixTrie = new Trie()
+        let data = fs.readFileSync("suffixes.txt", 'utf8')
         let lines = data.split("\n")
-        for (let line of lines) {
-            let items = line.split(" ");
-            this.parsedSurfaceForms.set(items[0], items[1])
+        for (let suffix of lines) {
+            let reverseSuffix = this.reverseString(suffix)
+            this.suffixTrie.addWord(reverseSuffix, new Word(reverseSuffix))
         }
     }
 
+    /**
+     * Reads the file for correct surface forms and their most frequent root forms, in other words, the surface forms
+     * which have at least one morphological analysis in  Turkish.
+     * @param fileName Input file containing analyzable surface forms and their root forms.
+     */
+    addParsedSurfaceForms(fileName: string){
+        this.parsedSurfaceForms = FileUtils.readHashMap(fileName)
+    }
+
+    /**
+     * Reads the file for foreign words and their pronunciations.
+     * @param fileName Input file containing foreign words and their pronunciations.
+     */
+    addPronunciations(fileName: string){
+        this.pronunciations = FileUtils.readHashMap(fileName)
+    }
+
     /**
      * The getPossibleWords method takes {@link MorphologicalParse} and {@link MetamorphicParse} as input.
      * First it determines whether the given morphologicalParse is the root verb and whether it contains a verb tag.
@@ -662,7 +702,7 @@ export class FsmMorphologicalAnalyzer {
      */
     private parseWordLength(fsmParse: Array<FsmParse>, maxLength: number): Array<FsmParse>{
         let result = new Array<FsmParse>();
-        let resultSuffixList = new Array<string>();
+        let resultTransitionList = new Array<string>();
         let parseQueue = new Queue<FsmParse>(1000)
         parseQueue.enqueueAll(fsmParse)
         while (!parseQueue.isEmpty()) {
@@ -672,11 +712,11 @@ export class FsmMorphologicalAnalyzer {
             let currentState = currentFsmParse.getFinalSuffix();
             let currentSurfaceForm = currentFsmParse.getSurfaceForm();
             if (currentState.isEndState() && currentSurfaceForm.length <= maxLength) {
-                let currentSuffixList = currentFsmParse.getSuffixList()
-                if (!resultSuffixList.includes(currentSuffixList)) {
+                let currentTransitionList = currentSurfaceForm + " " + currentFsmParse.getFsmParseTransitionList()
+                if (!resultTransitionList.includes(currentTransitionList)) {
                     result.push(currentFsmParse);
                     currentFsmParse.constructInflectionalGroups();
-                    resultSuffixList.push(currentSuffixList);
+                    resultTransitionList.push(currentTransitionList);
                 }
             }
             this.addNewParsesFromCurrentParseLength(currentFsmParse, parseQueue, maxLength, root);
@@ -694,7 +734,7 @@ export class FsmMorphologicalAnalyzer {
      */
     private parseWordSurfaceForm(fsmParse: Array<FsmParse>, surfaceForm: string): Array<FsmParse>{
         let result = new Array<FsmParse>();
-        let resultSuffixList = new Array<string>();
+        let resultTransitionList = new Array<string>();
         let parseQueue = new Queue<FsmParse>(1000)
         parseQueue.enqueueAll(fsmParse)
         while (!parseQueue.isEmpty()) {
@@ -704,11 +744,11 @@ export class FsmMorphologicalAnalyzer {
             let currentState = currentFsmParse.getFinalSuffix();
             let currentSurfaceForm = currentFsmParse.getSurfaceForm();
             if (currentState.isEndState() && currentSurfaceForm == surfaceForm) {
-                let currentSuffixList = currentFsmParse.getSuffixList()
-                if (!resultSuffixList.includes(currentSuffixList)) {
+                let currentTransitionList = currentFsmParse.getFsmParseTransitionList()
+                if (!resultTransitionList.includes(currentTransitionList)) {
                     result.push(currentFsmParse);
                     currentFsmParse.constructInflectionalGroups();
-                    resultSuffixList.push(currentSuffixList);
+                    resultTransitionList.push(currentTransitionList);
                 }
             }
             this.addNewParsesFromCurrentParseSurfaceForm(currentFsmParse, parseQueue, surfaceForm, root);
@@ -973,6 +1013,15 @@ export class FsmMorphologicalAnalyzer {
         return this.parseWordSurfaceForm(initialFsmParse, surfaceForm);
     }
 
+    /**
+     * This method uses cache idea to speed up pattern matching in Fsm. mostUsedPatterns stores the compiled forms of
+     * the previously used patterns. When Fsm tries to match a string to a pattern, first we check if it exists in
+     * mostUsedPatterns. If it exists, we directly use the compiled pattern to match the string. Otherwise, new pattern
+     * is compiled and put in the mostUsedPatterns.
+     * @param expr Pattern to check
+     * @param value String to match the pattern
+     * @return True if the string matches the pattern, false otherwise.
+     */
     private patternMatches(expr: string, value: string): boolean{
         let p = this.mostUsedPatterns.get(expr);
         if (p == undefined){
@@ -1011,6 +1060,40 @@ export class FsmMorphologicalAnalyzer {
         return this.patternMatches("^.*[0-9].*$", surfaceForm) && this.patternMatches("^.*[a-zA-ZçöğüşıÇÖĞÜŞİ].*$", surfaceForm);
     }
 
+    /**
+     * Identifies a possible new root word for a given surface form. It also adds the new root form to the dictionary
+     * for further usage. The method first searches the suffix trie for the reverse string of the surface form. This
+     * way, it can identify if the word has a suffix that is in the most frequently used suffix list. Since a word can
+     * have multiple possible suffixes, the method identifies the longest suffix and returns the substring of the
+     * surface form tht does not contain the suffix. Let say the word is 'googlelaştırdık', it will identify 'tık' as
+     * a suffix and will return 'googlelaştır' as a possible root form. Another example will be 'homelesslerimizle', it
+     * will identify 'lerimizle' as suffix and will return 'homeless' as a possible root form. If the root word ends
+     * with 'ğ', it is replacesd with 'k'. 'morfolojikliğini' will return 'morfolojikliğ' then which will be replaced
+     * with 'morfolojiklik'.
+     * @param surfaceForm Surface form for which we will identify a possible new root form.
+     * @return Possible new root form.
+     */
+    private rootOfPossiblyNewWord(surfaceForm: string): Array<TxtWord>{
+        let words = this.suffixTrie.getWordsWithPrefix(this.reverseString(surfaceForm))
+        let candidateWord = null
+        let candidateList = new Array<TxtWord>();
+        for (let word of words){
+            candidateWord = surfaceForm.substring(0, surfaceForm.length - word.getName().length)
+            let newWord
+            if (candidateWord.endsWith("ğ")){
+                candidateWord = candidateWord.substring(0, candidateWord.length - 1) + "k"
+                newWord = new TxtWord(candidateWord, "CL_ISIM")
+                newWord.addFlag("IS_SD")
+            } else {
+                newWord = new TxtWord(candidateWord, "CL_ISIM")
+                newWord.addFlag("CL_FIIL")
+            }
+            candidateList.push(newWord)
+            this.dictionaryTrie.addWord(candidateWord, newWord)
+        }
+        return candidateList
+    }
+
     /**
      * The robustMorphologicalAnalysis is used to analyse surfaceForm String. First it gets the currentParse of the surfaceForm
      * then, if the size of the currentParse is 0, and given surfaceForm is a proper noun, it adds the surfaceForm
@@ -1028,14 +1111,19 @@ export class FsmMorphologicalAnalyzer {
         if (currentParse.size() == 0) {
             let fsmParse = new Array<FsmParse>();
             if (this.isProperNoun(surfaceForm)) {
-                fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("ProperRoot")));
-            } else {
-                if (this.isCode(surfaceForm)) {
-                    fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("CodeRoot")));
-                } else {
-                    fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("NominalRoot")));
+                fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("ProperRoot")))
+            }
+            if (this.isCode(surfaceForm)) {
+                fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("CodeRoot")))
+            }
+            let newCandidateList = this.rootOfPossiblyNewWord(surfaceForm)
+            if (newCandidateList.length != 0){
+                for (let word of newCandidateList) {
+                    fsmParse.push(new FsmParse(word, this.finiteStateMachine.getState("VerbalRoot")))
+                    fsmParse.push(new FsmParse(word, this.finiteStateMachine.getState("NominalRoot")))
                 }
             }
+            fsmParse.push(new FsmParse(surfaceForm, this.finiteStateMachine.getState("NominalRoot")))
             return new FsmParseList(this.parseWordSurfaceForm(fsmParse, surfaceForm));
         } else {
             return currentParse;
@@ -1144,22 +1232,43 @@ export class FsmMorphologicalAnalyzer {
         return word == "" && count > 1;
     }
 
+    /**
+     * Checks if a given surface form matches to a percent value. It should be something like %4, %45, %4.3 or %56.786
+     * @param surfaceForm Surface form to be checked.
+     * @return True if the surface form is in percent form
+     */
     private isPercent(surfaceForm: string): boolean{
         return this.patternMatches("^%(\\d\\d|\\d)$", surfaceForm) ||
             this.patternMatches("^%(\\d\\d|\\d)\\.\\d+$", surfaceForm);
     }
 
+    /**
+     * Checks if a given surface form matches to a time form. It should be something like 3:34, 12:56 etc.
+     * @param surfaceForm Surface form to be checked.
+     * @return True if the surface form is in time form
+     */
     private isTime(surfaceForm: string): boolean{
         return this.patternMatches("^(\\d\\d|\\d):(\\d\\d|\\d):(\\d\\d|\\d)$", surfaceForm) ||
             this.patternMatches("^(\\d\\d|\\d):(\\d\\d|\\d)$", surfaceForm);
     }
 
+    /**
+     * Checks if a given surface form matches to a range form. It should be something like 123-1400 or 12:34-15:78 or
+     * 3.45-4.67.
+     * @param surfaceForm Surface form to be checked.
+     * @return True if the surface form is in range form
+     */
     private isRange(surfaceForm: string): boolean{
         return this.patternMatches("^\\d+-\\d+$", surfaceForm) ||
             this.patternMatches("^(\\d\\d|\\d):(\\d\\d|\\d)-(\\d\\d|\\d):(\\d\\d|\\d)$", surfaceForm) ||
             this.patternMatches("^(\\d\\d|\\d)\\.(\\d\\d|\\d)-(\\d\\d|\\d)\\.(\\d\\d|\\d)$", surfaceForm);
     }
 
+    /**
+     * Checks if a given surface form matches to a date form. It should be something like 3/10/2023 or 2.3.2012
+     * @param surfaceForm Surface form to be checked.
+     * @return True if the surface form is in date form
+     */
     private isDate(surfaceForm: string): boolean{
         return this.patternMatches("^(\\d\\d|\\d)/(\\d\\d|\\d)/\\d+$", surfaceForm) ||
             this.patternMatches("^(\\d\\d|\\d)\\.(\\d\\d|\\d)\\.\\d+$", surfaceForm);
@@ -1183,7 +1292,9 @@ export class FsmMorphologicalAnalyzer {
      * @return fsmParseList which holds the analysis.
      */
     morphologicalAnalysis(surfaceForm: string): FsmParseList{
-        let lowerCased = surfaceForm.toLocaleLowerCase("tr");
+        let lowerCased = surfaceForm.toLocaleLowerCase("tr")
+        let possibleRootLowerCased = "", pronunciation = ""
+        let isRootReplaced = false
         if (this.parsedSurfaceForms != undefined && this.parsedSurfaceForms.has(lowerCased) && 
             !this.isInteger(surfaceForm) && !this.isDouble(surfaceForm) && !this.isPercent(surfaceForm) && 
             !this.isTime(surfaceForm) && !this.isRange(surfaceForm) && !this.isDate(surfaceForm)){
@@ -1195,9 +1306,9 @@ export class FsmMorphologicalAnalyzer {
             return this.cache.get(surfaceForm);
         }
         if (this.patternMatches("^(\\w|Ç|Ş|İ|Ü|Ö)\\.$",surfaceForm)) {
-            this.dictionaryTrie.addWord(surfaceForm.toLocaleLowerCase("tr"), new TxtWord(surfaceForm.toLocaleLowerCase("tr"), "IS_OA"));
+            this.dictionaryTrie.addWord(lowerCased, new TxtWord(lowerCased, "IS_OA"));
         }
-        let defaultFsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+        let defaultFsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
         if (defaultFsmParse.length > 0) {
             let fsmParseList = new FsmParseList(defaultFsmParse);
             if (this.cache != undefined) {
@@ -1211,48 +1322,58 @@ export class FsmMorphologicalAnalyzer {
             if (possibleRoot != "") {
                 if (possibleRoot.includes("/") || possibleRoot.includes("\\/")) {
                     this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_KESIR"));
-                    fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                    fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                 } else {
                     if (this.isDate(possibleRoot)) {
                         this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_DATE"));
-                        fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                        fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                     } else {
                         if (this.patternMatches("^\\d+/\\d+$", possibleRoot)) {
                             this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_KESIR"));
-                            fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                            fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                         } else {
                             if (this.isPercent(possibleRoot)) {
                                 this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_PERCENT"));
-                                fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                             } else {
                                 if (this.isTime(surfaceForm)) {
                                     this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_ZAMAN"));
-                                    fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                    fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                                 } else {
                                     if (this.isRange(surfaceForm)) {
                                         this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_RANGE"));
-                                        fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                        fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                                     } else {
                                         if (this.isInteger(possibleRoot)) {
                                             this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_SAYI"));
-                                            fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                            fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                                         } else {
                                             if (this.isDouble(possibleRoot)) {
                                                 this.dictionaryTrie.addWord(possibleRoot, new TxtWord(possibleRoot, "IS_REELSAYI"));
-                                                fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                                fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                                             } else {
-                                                if (Word.isCapital(possibleRoot)) {
-                                                    let newWord = undefined;
-                                                    if (this.dictionary.getWord(possibleRoot.toLocaleLowerCase("tr")) != null) {
-                                                        (<TxtWord> this.dictionary.getWord(possibleRoot.toLocaleLowerCase("tr"))).addFlag("IS_OA");
+                                                if (Word.isCapital(possibleRoot) || "QXW".includes(possibleRoot.substring(0, 1))) {
+                                                    let newWord = undefined
+                                                    possibleRootLowerCased = possibleRoot.toLocaleLowerCase("tr");
+                                                    if (this.pronunciations.has(possibleRootLowerCased)){
+                                                        isRootReplaced = true
+                                                        pronunciation = this.pronunciations.get(possibleRootLowerCased)
+                                                        if (this.dictionary.getWord(pronunciation) != null) {
+                                                            (<TxtWord> this.dictionary.getWord(pronunciation)).addFlag("IS_OA");
+                                                        } else {
+                                                            newWord = new TxtWord(pronunciation, "IS_OA");
+                                                            this.dictionaryTrie.addWord(pronunciation, newWord);
+                                                        }
+                                                        let replacedWord = pronunciation + lowerCased.substring(possibleRootLowerCased.length);
+                                                        fsmParse = this.analysis(replacedWord, this.isProperNoun(surfaceForm));
                                                     } else {
-                                                        newWord = new TxtWord(possibleRoot.toLocaleLowerCase("tr"), "IS_OA");
-                                                        this.dictionaryTrie.addWord(possibleRoot.toLocaleLowerCase("tr"), newWord);
-                                                    }
-                                                    fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
-                                                    if (fsmParse.length == 0 && newWord != undefined) {
-                                                        newWord.addFlag("IS_KIS");
-                                                        fsmParse = this.analysis(surfaceForm.toLocaleLowerCase("tr"), this.isProperNoun(surfaceForm));
+                                                        if (this.dictionary.getWord(possibleRootLowerCased) != null) {
+                                                            (<TxtWord> this.dictionary.getWord(possibleRootLowerCased)).addFlag("IS_OA");
+                                                        } else {
+                                                            newWord = new TxtWord(possibleRootLowerCased, "IS_OA");
+                                                            this.dictionaryTrie.addWord(possibleRootLowerCased, newWord);
+                                                        }
+                                                        fsmParse = this.analysis(lowerCased, this.isProperNoun(surfaceForm));
                                                     }
                                                 }
                                             }
@@ -1265,6 +1386,11 @@ export class FsmMorphologicalAnalyzer {
                 }
             }
         }
+        if (!isRootReplaced){
+            for (let parse of fsmParse){
+                parse.restoreOriginalForm(possibleRootLowerCased, pronunciation)
+            }
+        }
         let fsmParseList = new FsmParseList(fsmParse);
         if (this.cache != undefined && fsmParseList.size() > 0) {
             this.cache.add(surfaceForm, fsmParseList);

package/source/MorphologicalAnalysis/FsmParse.ts

@@ -1,6 +1,5 @@
 import {MorphologicalParse} from "./MorphologicalParse";
 import {State} from "./State";
-import {Word} from "nlptoolkit-dictionary/dist/Dictionary/Word";
 import {TxtWord} from "nlptoolkit-dictionary/dist/Dictionary/TxtWord";
 import {InflectionalGroup} from "./InflectionalGroup";
 import {Transition} from "./Transition";
@@ -675,4 +674,20 @@ export class FsmParse extends MorphologicalParse{
     toString(): string{
         return this.getFsmParseTransitionList()
     }
+
+    /**
+     * In order to morphologically parse special proper nouns in Turkish, whose affixes obeys not the original but their
+     * pronunciations, the morphologicalAnalysis method replaces the original word with its pronunciation and do the
+     * rest. This method reverts it back, that is it restores its original form by replacing the pronunciations in the
+     * parses with the original form.
+     * @param original Original form of the proper noun.
+     * @param pronunciation Pronunciation of the proper noun.
+     */
+    restoreOriginalForm(original: string, pronunciation: string){
+        this.root = new TxtWord(original, "IS_OA");
+        this.form = original + this.form.substring(pronunciation.length);
+        for (let i = 0; i < this.formList.length; i++) {
+            this.formList[i] = original + this.formList[i].substring(pronunciation.length)
+        }
+    }
 }

package/source/MorphologicalAnalysis/InflectionalGroup.ts

@@ -28,7 +28,8 @@ export class InflectionalGroup {
         "INF2", "INF3", "BSTAG", "ESTAG", "BTTAG",
         "ETTAG", "BDTAG", "EDTAG", "INF1", "ASLONGAS",
         "DIST", "ADAMANTLY", "PERCENT", "WITHOUTBEINGABLETOHAVEDONESO", "DIM",
-        "PERS", "FRACTION", "HASHTAG", "EMAIL", "DATE", "CODE", "METRIC"];
+        "PERS", "FRACTION", "HASHTAG", "EMAIL", "DATE",
+        "CODE", "METRIC", "POL", "URGE"];
     static morphoTags = [MorphologicalTag.NOUN, MorphologicalTag.ADVERB, MorphologicalTag.ADJECTIVE,
         MorphologicalTag.VERB, MorphologicalTag.A1SG, MorphologicalTag.A2SG, MorphologicalTag.A3SG, MorphologicalTag.A1PL,
         MorphologicalTag.A2PL, MorphologicalTag.A3PL, MorphologicalTag.P1SG, MorphologicalTag.P2SG, MorphologicalTag.P3SG, MorphologicalTag.P1PL,
@@ -55,7 +56,7 @@ export class InflectionalGroup {
         MorphologicalTag.ENDOFTITLE, MorphologicalTag.BEGINNINGOFDOCUMENT, MorphologicalTag.ENDOFDOCUMENT, MorphologicalTag.INFINITIVE, MorphologicalTag.ASLONGAS,
         MorphologicalTag.DISTRIBUTIVE, MorphologicalTag.ADAMANTLY, MorphologicalTag.PERCENT, MorphologicalTag.WITHOUTBEINGABLETOHAVEDONESO, MorphologicalTag.DIMENSION,
         MorphologicalTag.PERSONALPRONOUN, MorphologicalTag.FRACTION, MorphologicalTag.HASHTAG, MorphologicalTag.EMAIL, MorphologicalTag.DATE,
-        MorphologicalTag.CODE, MorphologicalTag.METRIC];
+        MorphologicalTag.CODE, MorphologicalTag.METRIC, MorphologicalTag.POLITE, MorphologicalTag.URGE];
 
     /**
      * The getMorphologicalTag method takes a String tag as an input and if the input matches with one of the elements of