@delali/narsil 0.1.0

package/dist/languages/hungarian.mjs

@@ -0,0 +1,600 @@
+// src/languages/hungarian.ts
+var VOWELS = "a\xE1e\xE9i\xEDo\xF3\xF6\u0151u\xFA\xFC\u0171";
+function isVowel(ch) {
+  return VOWELS.includes(ch);
+}
+var DOUBLE_CONSONANTS = [
+  ["ccs", "cs"],
+  ["ggy", "gy"],
+  ["lly", "ly"],
+  ["nny", "ny"],
+  ["ssz", "sz"],
+  ["tty", "ty"],
+  ["zzs", "zs"],
+  ["bb", "b"],
+  ["cc", "c"],
+  ["dd", "d"],
+  ["ff", "f"],
+  ["gg", "g"],
+  ["jj", "j"],
+  ["kk", "k"],
+  ["ll", "l"],
+  ["mm", "m"],
+  ["nn", "n"],
+  ["pp", "p"],
+  ["rr", "r"],
+  ["ss", "s"],
+  ["tt", "t"],
+  ["vv", "v"],
+  ["zz", "z"]
+];
+var DIGRAPH_STARTS = ["dzs", "cs", "dz", "gy", "ly", "ny", "sz", "ty", "zs"];
+var STEP1_SUFFIXES = ["al", "el"];
+var STEP2_SUFFIXES = [
+  "k\xE9ppen",
+  "onk\xE9nt",
+  "enk\xE9nt",
+  "ank\xE9nt",
+  "k\xE9nt",
+  "k\xE9pp",
+  "ban",
+  "ben",
+  "nak",
+  "nek",
+  "val",
+  "vel",
+  "b\xF3l",
+  "r\xF3l",
+  "t\xF3l",
+  "b\xF6l",
+  "r\u0151l",
+  "t\u0151l",
+  "n\xE1l",
+  "n\xE9l",
+  "hoz",
+  "hez",
+  "h\xF6z",
+  "v\xE1",
+  "v\xE9",
+  "ba",
+  "be",
+  "ra",
+  "re",
+  "an",
+  "en",
+  "on",
+  "\xF6n",
+  "\xE9rt",
+  "kor",
+  "ig",
+  "at",
+  "et",
+  "ot",
+  "\xF6t",
+  "ul",
+  "\xFCl",
+  "n",
+  "t"
+];
+var STEP3_SUFFIXES = ["\xE1nk\xE9nt", "\xE1n", "\xE9n"];
+var STEP4_SUFFIXES = ["\xE1stul", "\xE9st\xFCl", "astul", "est\xFCl", "stul", "st\xFCl"];
+var STEP5_SUFFIXES = ["\xE1", "\xE9"];
+var STEP6_SUFFIXES = [
+  "\xE9k\xE9i",
+  "\xE1\xE9i",
+  "\xE9\xE9i",
+  "\xE1k\xE9",
+  "\xE9k\xE9",
+  "ok\xE9\xE9",
+  "\xF6k\xE9\xE9",
+  "ak\xE9\xE9",
+  "ek\xE9\xE9",
+  "ok\xE9",
+  "\xF6k\xE9",
+  "ak\xE9",
+  "ek\xE9",
+  "\xE9\xE9",
+  "\xE9i",
+  "k\xE9",
+  "\xE9"
+];
+var STEP7_SUFFIXES = [
+  "\xE1juk",
+  "\xE9j\xFCk",
+  "juk",
+  "j\xFCk",
+  "\xE1nk",
+  "\xE9nk",
+  "\xFCnk",
+  "unk",
+  "\xE1m",
+  "\xE9m",
+  "\xE1d",
+  "\xE9d",
+  "uk",
+  "\xFCk",
+  "ja",
+  "je",
+  "am",
+  "em",
+  "om",
+  "ad",
+  "ed",
+  "od",
+  "\xF6d",
+  "\xE1",
+  "\xE9",
+  "a",
+  "e",
+  "o",
+  "nk",
+  "m",
+  "d"
+];
+var STEP8_SUFFIXES = [
+  "jaitok",
+  "jeitek",
+  "\xE1itok",
+  "\xE9itek",
+  "jaink",
+  "jeink",
+  "\xE1ink",
+  "\xE9ink",
+  "jaim",
+  "jeim",
+  "jaid",
+  "jeid",
+  "jaik",
+  "jeik",
+  "\xE1im",
+  "\xE9im",
+  "\xE1id",
+  "\xE9id",
+  "\xE1ik",
+  "\xE9ik",
+  "jai",
+  "jei",
+  "\xE1i",
+  "\xE9i",
+  "aink",
+  "eink",
+  "aim",
+  "eim",
+  "aid",
+  "eid",
+  "aik",
+  "eik",
+  "ai",
+  "ei",
+  "itek",
+  "ink",
+  "im",
+  "id",
+  "ik",
+  "i"
+];
+var STEP9_SUFFIXES = ["\xE1k", "\xE9k", "\xF6k", "ok", "ek", "ak", "k"];
+function computeR1(word) {
+  if (word.length === 0) return 0;
+  let i = 0;
+  if (isVowel(word[0])) {
+    i = 1;
+    while (i < word.length && isVowel(word[i])) i++;
+    if (i >= word.length) return word.length;
+    let advanced = false;
+    for (const dg of DIGRAPH_STARTS) {
+      if (word.startsWith(dg, i)) {
+        i += dg.length;
+        advanced = true;
+        break;
+      }
+    }
+    if (!advanced) i++;
+  } else {
+    let advanced = false;
+    for (const dg of DIGRAPH_STARTS) {
+      if (word.startsWith(dg, 0)) {
+        i = dg.length;
+        advanced = true;
+        break;
+      }
+    }
+    if (!advanced) i = 1;
+    while (i < word.length && !isVowel(word[i])) i++;
+    if (i >= word.length) return word.length;
+    i++;
+  }
+  if (i <= word.length) return i;
+  return word.length;
+}
+function endsWithDouble(word) {
+  return DOUBLE_CONSONANTS.some(([dbl]) => word.endsWith(dbl));
+}
+function undoubleConsonant(word) {
+  for (const [dbl, single] of DOUBLE_CONSONANTS) {
+    if (word.endsWith(dbl)) {
+      return word.slice(0, word.length - dbl.length) + single;
+    }
+  }
+  return word;
+}
+function inR1(word, r1, suffixLen) {
+  return word.length - suffixLen >= r1;
+}
+function trySuffix(word, suffixes) {
+  for (const s of suffixes) {
+    if (word.endsWith(s)) return s;
+  }
+  return null;
+}
+function step1Instrumental(word, r1) {
+  const matched = trySuffix(word, STEP1_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (endsWithDouble(stripped)) {
+      return undoubleConsonant(stripped);
+    }
+  }
+  return word;
+}
+function step2CaseEndings(word, r1) {
+  const matched = trySuffix(word, STEP2_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (stripped.endsWith("\xE1")) {
+      return `${stripped.slice(0, -1)}a`;
+    }
+    if (stripped.endsWith("\xE9")) {
+      return `${stripped.slice(0, -1)}e`;
+    }
+    return stripped;
+  }
+  return word;
+}
+function step3Emotive(word, r1) {
+  const matched = trySuffix(word, STEP3_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE9n") return `${stripped}e`;
+    return `${stripped}a`;
+  }
+  return word;
+}
+function step4Comitative(word, r1) {
+  const matched = trySuffix(word, STEP4_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE1stul") return `${stripped}a`;
+    if (matched === "\xE9st\xFCl") return `${stripped}e`;
+    return stripped;
+  }
+  return word;
+}
+function step5LongVowel(word, r1) {
+  const matched = trySuffix(word, STEP5_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (endsWithDouble(stripped)) {
+      return undoubleConsonant(stripped);
+    }
+  }
+  return word;
+}
+function step6Possessive(word, r1) {
+  const matched = trySuffix(word, STEP6_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE1k\xE9" || matched === "\xE1\xE9i") return `${stripped}a`;
+    if (matched === "\xE9k\xE9" || matched === "\xE9\xE9i" || matched === "\xE9\xE9") return `${stripped}e`;
+    return stripped;
+  }
+  return word;
+}
+function step7SingularPossessive(word, r1) {
+  const matched = trySuffix(word, STEP7_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE1juk" || matched === "\xE1nk" || matched === "\xE1m" || matched === "\xE1d" || matched === "\xE1") {
+      return `${stripped}a`;
+    }
+    if (matched === "\xE9j\xFCk" || matched === "\xE9nk" || matched === "\xE9m" || matched === "\xE9d" || matched === "\xE9") {
+      return `${stripped}e`;
+    }
+    return stripped;
+  }
+  return word;
+}
+function step8PluralPossessive(word, r1) {
+  const matched = trySuffix(word, STEP8_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE1itok" || matched === "\xE1ink" || matched === "\xE1im" || matched === "\xE1id" || matched === "\xE1ik" || matched === "\xE1i") {
+      return `${stripped}a`;
+    }
+    if (matched === "\xE9itek" || matched === "\xE9ink" || matched === "\xE9im" || matched === "\xE9id" || matched === "\xE9ik" || matched === "\xE9i") {
+      return `${stripped}e`;
+    }
+    return stripped;
+  }
+  return word;
+}
+function step9Plural(word, r1) {
+  const matched = trySuffix(word, STEP9_SUFFIXES);
+  if (matched && inR1(word, r1, matched.length)) {
+    const stripped = word.slice(0, -matched.length);
+    if (matched === "\xE1k") return `${stripped}a`;
+    if (matched === "\xE9k") return `${stripped}e`;
+    return stripped;
+  }
+  return word;
+}
+function stem(word) {
+  if (word.length < 2) return word;
+  word = word.toLowerCase();
+  const r1 = computeR1(word);
+  word = step1Instrumental(word, r1);
+  word = step2CaseEndings(word, r1);
+  word = step3Emotive(word, r1);
+  word = step4Comitative(word, r1);
+  word = step5LongVowel(word, r1);
+  word = step6Possessive(word, r1);
+  word = step7SingularPossessive(word, r1);
+  word = step8PluralPossessive(word, r1);
+  word = step9Plural(word, r1);
+  return word;
+}
+var stopWords = /* @__PURE__ */ new Set([
+  "a",
+  "abba",
+  "abban",
+  "abb\xF3l",
+  "addig",
+  "ahhoz",
+  "ahogy",
+  "ahol",
+  "aki",
+  "akik",
+  "akkor",
+  "ak\xE1r",
+  "alapj\xE1n",
+  "alatt",
+  "alatta",
+  "al\xE1",
+  "amely",
+  "amelyek",
+  "amelyet",
+  "amelyik",
+  "amelynek",
+  "ami",
+  "amikor",
+  "amit",
+  "am\xEDg",
+  "annak",
+  "ann\xE1l",
+  "arra",
+  "arr\xF3l",
+  "att\xF3l",
+  "az",
+  "azok",
+  "azokat",
+  "azon",
+  "azonban",
+  "azt",
+  "azt\xE1n",
+  "azut\xE1n",
+  "azzal",
+  "az\xE9rt",
+  "be",
+  "bel\xFCl",
+  "benne",
+  "b\xE1r",
+  "csak",
+  "de",
+  "e",
+  "eddig",
+  "egy",
+  "egyed\xFCl",
+  "egyes",
+  "egyetlen",
+  "egyik",
+  "egym\xE1s",
+  "egyre",
+  "egy\xE9b",
+  "egy\xFCtt",
+  "eg\xE9sz",
+  "ehhez",
+  "ekkor",
+  "el",
+  "ellen",
+  "el\xE9g",
+  "el\u0151",
+  "el\u0151bb",
+  "el\u0151sz\xF6r",
+  "el\u0151tt",
+  "el\u0151z\u0151",
+  "engem",
+  "ennek",
+  "erre",
+  "ez",
+  "ezek",
+  "ezen",
+  "ezt",
+  "ezut\xE1n",
+  "ezzel",
+  "ez\xE9rt",
+  "fel",
+  "fel\xE9",
+  "ha",
+  "hanem",
+  "hiszen",
+  "hogy",
+  "hogyan",
+  "hol",
+  "holnap",
+  "honnan",
+  "hova",
+  "hozz\xE1",
+  "ide",
+  "igen",
+  "ill",
+  "illetve",
+  "ilyen",
+  "imm\xE1r",
+  "ink\xE1bb",
+  "is",
+  "ism\xE9t",
+  "itt",
+  "jobban",
+  "j\xF3l",
+  "kell",
+  "kellett",
+  "kereszt\xFCl",
+  "kett\u0151",
+  "kev\xE9s",
+  "ki",
+  "kit",
+  "k\xF6zben",
+  "k\xF6zel",
+  "k\xF6z\xE9",
+  "k\xF6z\xF6tt",
+  "k\xF6z\xFCl",
+  "k\xFCl\xF6nben",
+  "k\xFCl\xF6nb\xF6z\u0151",
+  "le",
+  "legal\xE1bb",
+  "legyen",
+  "lehet",
+  "lenne",
+  "lenni",
+  "lesz",
+  "lett",
+  "ma",
+  "maga",
+  "mag\xE1t",
+  "mai",
+  "majd",
+  "meg",
+  "megint",
+  "mellett",
+  "mell\xE9",
+  "mely",
+  "melyek",
+  "melyik",
+  "mennyi",
+  "mert",
+  "mi",
+  "miatt",
+  "mikor",
+  "milyen",
+  "mind",
+  "minden",
+  "mindenki",
+  "mindent",
+  "mindig",
+  "minek",
+  "minket",
+  "mint",
+  "mintha",
+  "mit",
+  "mivel",
+  "mi\xE9rt",
+  "mondta",
+  "most",
+  "m\xE1r",
+  "m\xE1s",
+  "m\xE1sik",
+  "m\xE1sodik",
+  "m\xE9g",
+  "m\xE9gis",
+  "m\xEDg",
+  "m\xFAlva",
+  "nagy",
+  "nagyon",
+  "ne",
+  "neked",
+  "nekem",
+  "neki",
+  "nekik",
+  "nek\xFCnk",
+  "nem",
+  "nincs",
+  "n\xE1lunk",
+  "n\xE9ha",
+  "n\xE9h\xE1ny",
+  "n\xE9lk\xFCl",
+  "oda",
+  "olyan",
+  "onnan",
+  "ott",
+  "pedig",
+  "persze",
+  "rajta",
+  "r\xF3la",
+  "saj\xE1t",
+  "se",
+  "sem",
+  "semmi",
+  "senki",
+  "soha",
+  "sok",
+  "sokat",
+  "sokkal",
+  "sor\xE1n",
+  "szemben",
+  "szerint",
+  "szinte",
+  "sz\xE1m\xE1ra",
+  "s\u0151t",
+  "tal\xE1n",
+  "te",
+  "teh\xE1t",
+  "ti",
+  "tov\xE1bb",
+  "tov\xE1bbi",
+  "tov\xE1bb\xE1",
+  "t\xE9ged",
+  "t\u0151le",
+  "ugyanis",
+  "ugye",
+  "ut\xE1n",
+  "ut\xE1na",
+  "vagy",
+  "vagyis",
+  "vagyok",
+  "vagyunk",
+  "vajon",
+  "valaki",
+  "valami",
+  "valamint",
+  "val\xF3",
+  "van",
+  "vannak",
+  "vele",
+  "velem",
+  "vel\xFCnk",
+  "vissza",
+  "viszont",
+  "volna",
+  "volt",
+  "voltak",
+  "v\xE9gre",
+  "v\xE9g\xFCl",
+  "\xE1ltal",
+  "\xE1ltal\xE1ban",
+  "\xE1t",
+  "\xE9n",
+  "\xE9s",
+  "\xEDgy",
+  "\u0151",
+  "\u0151k",
+  "\u0151ket",
+  "\u0151t"
+]);
+var hungarian = {
+  name: "hungarian",
+  stemmer: stem,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9áéíóöőúüű]+/gi }
+};
+
+export { hungarian };

package/dist/languages/igbo.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-ClUGLuoC.js';
+
+declare const igbo: LanguageModule;
+
+export { igbo };

package/dist/languages/igbo.mjs

@@ -0,0 +1,96 @@
+// src/languages/igbo.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "na",
+  "onye",
+  "gi",
+  "g\u1ECB",
+  "ka",
+  "ya",
+  "m",
+  "ndi",
+  "nd\u1ECB",
+  "a",
+  "o",
+  "\u1ECD",
+  "nke",
+  "di",
+  "d\u1ECB",
+  "ihe",
+  "i",
+  "\u1ECB",
+  "no",
+  "bu",
+  "b\u1EE5",
+  "anyi",
+  "any\u1ECB",
+  "ga",
+  "e",
+  "ma",
+  "ha",
+  "ike",
+  "ebe",
+  "kwa",
+  "eme",
+  "isi",
+  "maka",
+  "si",
+  "okwu",
+  "ji",
+  "otu",
+  "mgbe",
+  "oge",
+  "onwe",
+  "mana",
+  "eji",
+  "unu",
+  "aka",
+  "obi",
+  "anya",
+  "ala",
+  "nwa",
+  "site",
+  "n",
+  "ghi",
+  "gh\u1ECB",
+  "nwere",
+  "mere",
+  "kwu",
+  "gwa",
+  "nye",
+  "we",
+  "kwara",
+  "kwuru",
+  "gara",
+  "aga",
+  "nime",
+  "elu",
+  "okpuru",
+  "etiti",
+  "banyere",
+  "tupu",
+  "nile",
+  "ozo",
+  "\u1ECDz\u1ECD",
+  "ufodu",
+  "\u1EE5f\u1ECDd\u1EE5",
+  "obula",
+  "\u1ECDb\u1EE5la",
+  "ncha",
+  "nani",
+  "nan\u1ECB",
+  "puru",
+  "ugbu",
+  "taa"
+]);
+var igbo = {
+  name: "igbo",
+  stemmer: null,
+  stopWords,
+  tokenizer: {
+    splitPattern: /[^a-zA-ZịụọṅĩỊỤỌṄĨ0-9\p{M}]+/giu,
+    normalizeDiacritics: false,
+    minTokenLength: 1
+  }
+};
+
+export { igbo };

package/dist/languages/indonesian.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-ClUGLuoC.js';
+
+declare const indonesian: LanguageModule;
+
+export { indonesian };