@delali/narsil 0.1.0

package/dist/languages/italian.mjs

@@ -0,0 +1,713 @@
+// src/languages/italian.ts
+var VOWELS = "aeiou\xE0\xE8\xE9\xEC\xF2\xF3\xF9";
+var RE_QU = /qu/g;
+function isVowel(ch) {
+  return VOWELS.includes(ch);
+}
+function findRegions(word) {
+  const length = word.length;
+  let rv = length;
+  if (length > 3) {
+    if (!isVowel(word[1])) {
+      for (let i = 2; i < length; i++) {
+        if (isVowel(word[i])) {
+          rv = i + 1;
+          break;
+        }
+      }
+    } else if (isVowel(word[0]) && isVowel(word[1])) {
+      for (let i = 2; i < length; i++) {
+        if (!isVowel(word[i])) {
+          rv = i + 1;
+          break;
+        }
+      }
+    } else {
+      rv = 3;
+    }
+  }
+  let r1 = length;
+  for (let i = 1; i < length; i++) {
+    if (!isVowel(word[i]) && isVowel(word[i - 1])) {
+      r1 = i + 1;
+      break;
+    }
+  }
+  let r2 = length;
+  for (let i = r1 + 1; i < length; i++) {
+    if (!isVowel(word[i]) && isVowel(word[i - 1])) {
+      r2 = i + 1;
+      break;
+    }
+  }
+  return { rv, r1, r2 };
+}
+var RE_ACCENT_A = /á/g;
+var RE_ACCENT_E = /é/g;
+var RE_ACCENT_I = /í/g;
+var RE_ACCENT_O = /ó/g;
+var RE_ACCENT_U = /ú/g;
+function replaceAccentedVowels(word) {
+  return word.replace(RE_ACCENT_A, "\xE0").replace(RE_ACCENT_E, "\xE8").replace(RE_ACCENT_I, "\xEC").replace(RE_ACCENT_O, "\xF2").replace(RE_ACCENT_U, "\xF9");
+}
+function markVowelPairsAsUppercase(word) {
+  let result = "";
+  for (let i = 0; i < word.length; i++) {
+    const ch = word[i];
+    if (i > 0 && i < word.length - 1 && isVowel(word[i - 1]) && isVowel(word[i + 1])) {
+      if (ch === "i") {
+        result += "I";
+        continue;
+      }
+      if (ch === "u") {
+        result += "U";
+        continue;
+      }
+    }
+    result += ch;
+  }
+  return result;
+}
+var RE_UPPERCASE_I = /I/g;
+var RE_UPPERCASE_U = /U/g;
+function restoreMarkedVowels(word) {
+  return word.replace(RE_UPPERCASE_I, "i").replace(RE_UPPERCASE_U, "u");
+}
+function endsWith(word, suffix) {
+  return word.endsWith(suffix);
+}
+function suffixInRegion(word, suffix, regionStart) {
+  return word.length - suffix.length >= regionStart;
+}
+function removeSuffix(word, suffix) {
+  return word.slice(0, word.length - suffix.length);
+}
+var STANDARD_SUFFIXES = [
+  { suffix: "atrice", group: "A" },
+  { suffix: "atrici", group: "A" },
+  { suffix: "azione", group: "B" },
+  { suffix: "azioni", group: "B" },
+  { suffix: "uzione", group: "D" },
+  { suffix: "uzioni", group: "D" },
+  { suffix: "usione", group: "D" },
+  { suffix: "usioni", group: "D" },
+  { suffix: "amento", group: "F" },
+  { suffix: "amenti", group: "F" },
+  { suffix: "imento", group: "F" },
+  { suffix: "imenti", group: "F" },
+  { suffix: "amente", group: "G" },
+  { suffix: "atore", group: "B" },
+  { suffix: "atori", group: "B" },
+  { suffix: "mente", group: "A" },
+  { suffix: "logia", group: "C" },
+  { suffix: "logie", group: "C" },
+  { suffix: "abile", group: "A" },
+  { suffix: "abili", group: "A" },
+  { suffix: "ibile", group: "A" },
+  { suffix: "ibili", group: "A" },
+  { suffix: "ista", group: "A" },
+  { suffix: "iste", group: "A" },
+  { suffix: "isti", group: "A" },
+  { suffix: "ist\xE0", group: "A" },
+  { suffix: "ist\xE8", group: "A" },
+  { suffix: "ist\xEC", group: "A" },
+  { suffix: "anza", group: "A" },
+  { suffix: "anze", group: "A" },
+  { suffix: "enza", group: "E" },
+  { suffix: "enze", group: "E" },
+  { suffix: "iche", group: "A" },
+  { suffix: "ichi", group: "A" },
+  { suffix: "ismo", group: "A" },
+  { suffix: "ismi", group: "A" },
+  { suffix: "ante", group: "A" },
+  { suffix: "anti", group: "A" },
+  { suffix: "ica", group: "A" },
+  { suffix: "ice", group: "A" },
+  { suffix: "ico", group: "A" },
+  { suffix: "ici", group: "A" },
+  { suffix: "osa", group: "A" },
+  { suffix: "ose", group: "A" },
+  { suffix: "oso", group: "A" },
+  { suffix: "osi", group: "A" },
+  { suffix: "iva", group: "I" },
+  { suffix: "ive", group: "I" },
+  { suffix: "ivo", group: "I" },
+  { suffix: "ivi", group: "I" },
+  { suffix: "it\xE0", group: "H" }
+];
+function findLongestSuffix(word, suffixes) {
+  let best = null;
+  for (const entry of suffixes) {
+    if (endsWith(word, entry.suffix)) {
+      if (best === null || entry.suffix.length > best.suffix.length) {
+        best = entry;
+      }
+    }
+  }
+  return best;
+}
+function removeStandardSuffix(word, r1, r2, rv) {
+  const match = findLongestSuffix(word, STANDARD_SUFFIXES);
+  if (match === null) return { word, changed: false };
+  const { suffix, group } = match;
+  if (group === "A") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    return { word: removeSuffix(word, suffix), changed: true };
+  }
+  if (group === "B") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    word = removeSuffix(word, suffix);
+    if (endsWith(word, "ic") && suffixInRegion(word, "ic", r2)) {
+      word = removeSuffix(word, "ic");
+    }
+    return { word, changed: true };
+  }
+  if (group === "C") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    return { word: `${removeSuffix(word, suffix)}log`, changed: true };
+  }
+  if (group === "D") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    return { word: `${removeSuffix(word, suffix)}u`, changed: true };
+  }
+  if (group === "E") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    return { word: `${removeSuffix(word, suffix)}ente`, changed: true };
+  }
+  if (group === "F") {
+    if (!suffixInRegion(word, suffix, rv)) return { word, changed: false };
+    return { word: removeSuffix(word, suffix), changed: true };
+  }
+  if (group === "G") {
+    if (!suffixInRegion(word, suffix, r1)) return { word, changed: false };
+    word = removeSuffix(word, suffix);
+    if (endsWith(word, "iv") && suffixInRegion(word, "iv", r2)) {
+      word = removeSuffix(word, "iv");
+      if (endsWith(word, "at") && suffixInRegion(word, "at", r2)) {
+        word = removeSuffix(word, "at");
+      }
+    } else if (endsWith(word, "os") && suffixInRegion(word, "os", r2)) {
+      word = removeSuffix(word, "os");
+    } else if (endsWith(word, "ic") && suffixInRegion(word, "ic", r2)) {
+      word = removeSuffix(word, "ic");
+    } else if (endsWith(word, "abil") && suffixInRegion(word, "abil", r2)) {
+      word = removeSuffix(word, "abil");
+    }
+    return { word, changed: true };
+  }
+  if (group === "H") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    word = removeSuffix(word, suffix);
+    if (endsWith(word, "abil") && suffixInRegion(word, "abil", r2)) {
+      word = removeSuffix(word, "abil");
+    } else if (endsWith(word, "ic") && suffixInRegion(word, "ic", r2)) {
+      word = removeSuffix(word, "ic");
+    } else if (endsWith(word, "iv") && suffixInRegion(word, "iv", r2)) {
+      word = removeSuffix(word, "iv");
+    }
+    return { word, changed: true };
+  }
+  if (group === "I") {
+    if (!suffixInRegion(word, suffix, r2)) return { word, changed: false };
+    word = removeSuffix(word, suffix);
+    if (endsWith(word, "at") && suffixInRegion(word, "at", r2)) {
+      word = removeSuffix(word, "at");
+      if (endsWith(word, "ic") && suffixInRegion(word, "ic", r2)) {
+        word = removeSuffix(word, "ic");
+      }
+    }
+    return { word, changed: true };
+  }
+  return { word, changed: false };
+}
+var VERB_SUFFIXES = [
+  "erebbero",
+  "irebbero",
+  "erebbe",
+  "irebbe",
+  "assero",
+  "essero",
+  "issero",
+  "assimo",
+  "eremmo",
+  "iremmo",
+  "eranno",
+  "iranno",
+  "ereste",
+  "ireste",
+  "iscano",
+  "iscono",
+  "eremo",
+  "iremo",
+  "avamo",
+  "evamo",
+  "ivamo",
+  "avate",
+  "evate",
+  "ivate",
+  "erete",
+  "irete",
+  "arono",
+  "erono",
+  "irono",
+  "avano",
+  "evano",
+  "ivano",
+  "eresti",
+  "iresti",
+  "isco",
+  "isca",
+  "isce",
+  "isci",
+  "ando",
+  "endo",
+  "Yamo",
+  "iamo",
+  "ammo",
+  "emmo",
+  "immo",
+  "asse",
+  "aste",
+  "este",
+  "iste",
+  "erai",
+  "irai",
+  "erei",
+  "irei",
+  "ava",
+  "eva",
+  "iva",
+  "ata",
+  "ita",
+  "uta",
+  "ate",
+  "ete",
+  "ite",
+  "ute",
+  "ati",
+  "iti",
+  "uti",
+  "ato",
+  "ito",
+  "uto",
+  "avo",
+  "evo",
+  "ivo",
+  "ano",
+  "ono",
+  "are",
+  "ere",
+  "ire",
+  "assi",
+  "avi",
+  "evi",
+  "ivi",
+  "endi",
+  "ar",
+  "ir",
+  "er\xE0",
+  "ir\xE0",
+  "er\xF2",
+  "ir\xF2"
+];
+var ATTACHED_PRONOUN_SUFFIXES = [
+  "gliela",
+  "gliele",
+  "glieli",
+  "glielo",
+  "gliene",
+  "cela",
+  "cele",
+  "celi",
+  "celo",
+  "cene",
+  "mela",
+  "mele",
+  "meli",
+  "melo",
+  "mene",
+  "sene",
+  "tela",
+  "tele",
+  "teli",
+  "telo",
+  "tene",
+  "vela",
+  "vele",
+  "veli",
+  "velo",
+  "vene",
+  "gli",
+  "la",
+  "le",
+  "li",
+  "lo",
+  "ci",
+  "mi",
+  "ne",
+  "si",
+  "ti",
+  "vi"
+];
+var ATTACHED_PRONOUN_PRECEDING = ["ando", "endo", "ar", "er", "ir"];
+function removeAttachedPronouns(word, rv) {
+  let longestPronoun = null;
+  for (const pronoun of ATTACHED_PRONOUN_SUFFIXES) {
+    if (endsWith(word, pronoun) && suffixInRegion(word, pronoun, rv)) {
+      if (longestPronoun === null || pronoun.length > longestPronoun.length) {
+        longestPronoun = pronoun;
+      }
+    }
+  }
+  if (longestPronoun === null) return word;
+  const stemmed = removeSuffix(word, longestPronoun);
+  for (const preceding of ATTACHED_PRONOUN_PRECEDING) {
+    if (endsWith(stemmed, preceding)) {
+      if (preceding === "ar" || preceding === "er" || preceding === "ir") {
+        return `${removeSuffix(stemmed, preceding) + preceding}e`;
+      }
+      return stemmed;
+    }
+  }
+  return word;
+}
+function removeVerbSuffix(word, rv) {
+  let longestSuffix = null;
+  for (const suffix of VERB_SUFFIXES) {
+    if (endsWith(word, suffix)) {
+      if (longestSuffix === null || suffix.length > longestSuffix.length) {
+        longestSuffix = suffix;
+      }
+    }
+  }
+  if (longestSuffix !== null && suffixInRegion(word, longestSuffix, rv)) {
+    return { word: removeSuffix(word, longestSuffix), changed: true };
+  }
+  return { word, changed: false };
+}
+function removeResidualSuffix(word, rv) {
+  const lastChar = word[word.length - 1];
+  if (lastChar === "a" || lastChar === "e" || lastChar === "i" || lastChar === "o" || lastChar === "\xE0" || lastChar === "\xE8" || lastChar === "\xEC" || lastChar === "\xF2") {
+    if (suffixInRegion(word, lastChar, rv)) {
+      word = word.slice(0, -1);
+      if (word.length > 0 && word[word.length - 1] === "i" && suffixInRegion(word, "i", rv)) {
+        word = word.slice(0, -1);
+      }
+    }
+  }
+  if (endsWith(word, "h") && suffixInRegion(word, "h", rv)) {
+    const twoBack = word.length >= 2 ? word[word.length - 2] : "";
+    if (twoBack === "c" || twoBack === "g") {
+      word = word.slice(0, -1);
+    }
+  }
+  return word;
+}
+function stem(word) {
+  if (word.length < 3) return word;
+  word = word.toLowerCase();
+  word = replaceAccentedVowels(word);
+  word = word.replace(RE_QU, "qU");
+  word = markVowelPairsAsUppercase(word);
+  const { rv, r1, r2 } = findRegions(word);
+  word = removeAttachedPronouns(word, rv);
+  const standardResult = removeStandardSuffix(word, r1, r2, rv);
+  if (standardResult.changed) {
+    word = standardResult.word;
+  } else {
+    const verbResult = removeVerbSuffix(word, rv);
+    word = verbResult.word;
+  }
+  word = removeResidualSuffix(word, rv);
+  word = restoreMarkedVowels(word);
+  return word;
+}
+var stopWords = /* @__PURE__ */ new Set([
+  "ad",
+  "al",
+  "allo",
+  "ai",
+  "agli",
+  "all",
+  "agl",
+  "alla",
+  "alle",
+  "con",
+  "col",
+  "coi",
+  "da",
+  "dal",
+  "dallo",
+  "dai",
+  "dagli",
+  "dall",
+  "dagl",
+  "dalla",
+  "dalle",
+  "di",
+  "del",
+  "dello",
+  "dei",
+  "degli",
+  "dell",
+  "degl",
+  "della",
+  "delle",
+  "in",
+  "nel",
+  "nello",
+  "nei",
+  "negli",
+  "nell",
+  "negl",
+  "nella",
+  "nelle",
+  "su",
+  "sul",
+  "sullo",
+  "sui",
+  "sugli",
+  "sull",
+  "sugl",
+  "sulla",
+  "sulle",
+  "per",
+  "tra",
+  "contro",
+  "io",
+  "tu",
+  "lui",
+  "lei",
+  "noi",
+  "voi",
+  "loro",
+  "mio",
+  "mia",
+  "miei",
+  "mie",
+  "tuo",
+  "tua",
+  "tuoi",
+  "tue",
+  "suo",
+  "sua",
+  "suoi",
+  "sue",
+  "nostro",
+  "nostra",
+  "nostri",
+  "nostre",
+  "vostro",
+  "vostra",
+  "vostri",
+  "vostre",
+  "mi",
+  "ti",
+  "ci",
+  "vi",
+  "lo",
+  "la",
+  "li",
+  "le",
+  "gli",
+  "ne",
+  "il",
+  "un",
+  "uno",
+  "una",
+  "ma",
+  "ed",
+  "se",
+  "perch\xE9",
+  "anche",
+  "come",
+  "dov",
+  "dove",
+  "che",
+  "chi",
+  "cui",
+  "non",
+  "pi\xF9",
+  "quale",
+  "quanto",
+  "quanti",
+  "quanta",
+  "quante",
+  "quello",
+  "quelli",
+  "quella",
+  "quelle",
+  "questo",
+  "questi",
+  "questa",
+  "queste",
+  "si",
+  "tutto",
+  "tutti",
+  "a",
+  "c",
+  "e",
+  "i",
+  "l",
+  "o",
+  "ho",
+  "hai",
+  "ha",
+  "abbiamo",
+  "avete",
+  "hanno",
+  "abbia",
+  "abbiate",
+  "abbiano",
+  "avr\xF2",
+  "avrai",
+  "avr\xE0",
+  "avremo",
+  "avrete",
+  "avranno",
+  "avrei",
+  "avresti",
+  "avrebbe",
+  "avremmo",
+  "avreste",
+  "avrebbero",
+  "avevo",
+  "avevi",
+  "aveva",
+  "avevamo",
+  "avevate",
+  "avevano",
+  "ebbi",
+  "avesti",
+  "ebbe",
+  "avemmo",
+  "aveste",
+  "ebbero",
+  "avessi",
+  "avesse",
+  "avessimo",
+  "avessero",
+  "avendo",
+  "avuto",
+  "avuta",
+  "avuti",
+  "avute",
+  "sono",
+  "sei",
+  "\xE8",
+  "siamo",
+  "siete",
+  "sia",
+  "siate",
+  "siano",
+  "sar\xF2",
+  "sarai",
+  "sar\xE0",
+  "saremo",
+  "sarete",
+  "saranno",
+  "sarei",
+  "saresti",
+  "sarebbe",
+  "saremmo",
+  "sareste",
+  "sarebbero",
+  "ero",
+  "eri",
+  "era",
+  "eravamo",
+  "eravate",
+  "erano",
+  "fui",
+  "fosti",
+  "fu",
+  "fummo",
+  "foste",
+  "furono",
+  "fossi",
+  "fosse",
+  "fossimo",
+  "fossero",
+  "essendo",
+  "faccio",
+  "fai",
+  "facciamo",
+  "fanno",
+  "faccia",
+  "facciate",
+  "facciano",
+  "far\xF2",
+  "farai",
+  "far\xE0",
+  "faremo",
+  "farete",
+  "faranno",
+  "farei",
+  "faresti",
+  "farebbe",
+  "faremmo",
+  "fareste",
+  "farebbero",
+  "facevo",
+  "facevi",
+  "faceva",
+  "facevamo",
+  "facevate",
+  "facevano",
+  "feci",
+  "facesti",
+  "fece",
+  "facemmo",
+  "faceste",
+  "fecero",
+  "facessi",
+  "facesse",
+  "facessimo",
+  "facessero",
+  "facendo",
+  "sto",
+  "stai",
+  "sta",
+  "stiamo",
+  "stanno",
+  "stia",
+  "stiate",
+  "stiano",
+  "star\xF2",
+  "starai",
+  "star\xE0",
+  "staremo",
+  "starete",
+  "staranno",
+  "starei",
+  "staresti",
+  "starebbe",
+  "staremmo",
+  "stareste",
+  "starebbero",
+  "stavo",
+  "stavi",
+  "stava",
+  "stavamo",
+  "stavate",
+  "stavano",
+  "stetti",
+  "stesti",
+  "stette",
+  "stemmo",
+  "steste",
+  "stettero",
+  "stessi",
+  "stesse",
+  "stessimo",
+  "stessero",
+  "stando"
+]);
+var italian = {
+  name: "italian",
+  stemmer: stem,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9\u00e0\u00e8\u00e9\u00ec\u00f2\u00f3\u00f9'-]+/gi }
+};
+
+export { italian };

package/dist/languages/japanese.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-ClUGLuoC.js';
+
+declare const japanese: LanguageModule;
+
+export { japanese };