@delali/narsil 0.1.0

package/dist/languages/finnish.mjs

@@ -0,0 +1,536 @@
+// src/languages/finnish.ts
+var VOWELS = "aeiouy\xE4\xF6";
+var LONG_VOWELS = ["aa", "ee", "ii", "oo", "uu", "\xE4\xE4", "\xF6\xF6"];
+var CONSONANT_PATTERN = /[a-zäö]/;
+var PARTICLES = ["kaan", "k\xE4\xE4n", "han", "h\xE4n", "kin", "ko", "k\xF6", "pa", "p\xE4"];
+var BACK_VOWEL_ENDINGS = ["lla", "na", "ssa", "ta", "lta", "sta"];
+var FRONT_VOWEL_ENDINGS = ["ll\xE4", "n\xE4", "ss\xE4", "t\xE4", "lt\xE4", "st\xE4"];
+var NEUTRAL_ENDINGS = ["lle", "ine"];
+var H_SUFFIXES = [
+  { suffix: "han", requiredPreceding: "a" },
+  { suffix: "hen", requiredPreceding: "e" },
+  { suffix: "hin", requiredPreceding: "i" },
+  { suffix: "hon", requiredPreceding: "o" },
+  { suffix: "hun", requiredPreceding: "u" },
+  { suffix: "hyn", requiredPreceding: "y" },
+  { suffix: "h\xE4n", requiredPreceding: "\xE4" },
+  { suffix: "h\xF6n", requiredPreceding: "\xF6" }
+];
+var SIMPLE_SUFFIXES = ["ssa", "ss\xE4", "sta", "st\xE4", "lla", "ll\xE4", "lta", "lt\xE4", "lle", "ine", "ksi"];
+var STEP4_SUFFIXES = [
+  { text: "impa", checkPo: false },
+  { text: "imp\xE4", checkPo: false },
+  { text: "impi", checkPo: true },
+  { text: "imma", checkPo: false },
+  { text: "imm\xE4", checkPo: false },
+  { text: "immi", checkPo: true },
+  { text: "eja", checkPo: false },
+  { text: "ej\xE4", checkPo: false },
+  { text: "mpa", checkPo: false },
+  { text: "mp\xE4", checkPo: false },
+  { text: "mpi", checkPo: true },
+  { text: "mma", checkPo: false },
+  { text: "mm\xE4", checkPo: false },
+  { text: "mmi", checkPo: true }
+];
+function isVowel(ch) {
+  return VOWELS.includes(ch);
+}
+function isConsonant(ch) {
+  return ch.length === 1 && !isVowel(ch) && CONSONANT_PATTERN.test(ch);
+}
+function endsWithLongVowel(word) {
+  if (word.length < 2) return false;
+  const tail = word.slice(-2);
+  return LONG_VOWELS.includes(tail);
+}
+function computeRegions(word) {
+  let r1 = word.length;
+  let r2 = word.length;
+  let foundVowel = false;
+  for (let i = 0; i < word.length; i++) {
+    if (foundVowel && !isVowel(word[i])) {
+      r1 = i + 1;
+      break;
+    }
+    if (isVowel(word[i])) {
+      foundVowel = true;
+    }
+  }
+  foundVowel = false;
+  for (let i = r1; i < word.length; i++) {
+    if (foundVowel && !isVowel(word[i])) {
+      r2 = i + 1;
+      break;
+    }
+    if (isVowel(word[i])) {
+      foundVowel = true;
+    }
+  }
+  return { r1, r2 };
+}
+function suffixInR1(word, suffix, r1) {
+  return word.endsWith(suffix) && word.length - suffix.length >= r1;
+}
+function suffixInR2(word, suffix, r2) {
+  return word.endsWith(suffix) && word.length - suffix.length >= r2;
+}
+function step1(word, r1, r2) {
+  for (const p of PARTICLES) {
+    if (suffixInR1(word, p, r1)) {
+      const charBefore = word[word.length - p.length - 1];
+      if (charBefore && (isVowel(charBefore) || charBefore === "n")) {
+        return word.slice(0, -p.length);
+      }
+    }
+  }
+  if (suffixInR2(word, "sti", r2)) {
+    return word.slice(0, -3);
+  }
+  return word;
+}
+function step2(word, r1) {
+  if (suffixInR1(word, "nsa", r1)) return word.slice(0, -3);
+  if (suffixInR1(word, "ns\xE4", r1)) return word.slice(0, -3);
+  if (suffixInR1(word, "mme", r1)) return word.slice(0, -3);
+  if (suffixInR1(word, "nne", r1)) return word.slice(0, -3);
+  if (suffixInR1(word, "si", r1)) {
+    const base = word.slice(0, -2);
+    if (base.endsWith("k")) return base;
+    if (base.endsWith("kse")) return `${base.slice(0, -3)}ksi`;
+    return base;
+  }
+  if (suffixInR1(word, "ni", r1)) {
+    const base = word.slice(0, -2);
+    if (base.endsWith("kse")) return `${base.slice(0, -3)}ksi`;
+    return base;
+  }
+  if (suffixInR1(word, "an", r1)) {
+    const base = word.slice(0, -2);
+    for (const ending of BACK_VOWEL_ENDINGS) {
+      if (base.endsWith(ending)) return base;
+    }
+  }
+  if (suffixInR1(word, "\xE4n", r1)) {
+    const base = word.slice(0, -2);
+    for (const ending of FRONT_VOWEL_ENDINGS) {
+      if (base.endsWith(ending)) return base;
+    }
+  }
+  if (suffixInR1(word, "en", r1)) {
+    const base = word.slice(0, -2);
+    for (const ending of NEUTRAL_ENDINGS) {
+      if (base.endsWith(ending)) return base;
+    }
+  }
+  return word;
+}
+function step3(word, r1) {
+  for (const { suffix, requiredPreceding } of H_SUFFIXES) {
+    if (suffixInR1(word, suffix, r1)) {
+      const base = word.slice(0, -suffix.length);
+      if (base.length > 0 && base[base.length - 1] === requiredPreceding) {
+        return { word: base, caseRemoved: true };
+      }
+    }
+  }
+  if (suffixInR1(word, "siin", r1)) {
+    const base = word.slice(0, -4);
+    if (base.length >= 2 && isConsonant(base[base.length - 1])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "seen", r1)) {
+    const base = word.slice(0, -4);
+    if (endsWithLongVowel(base)) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "den", r1)) {
+    const base = word.slice(0, -3);
+    if (base.length >= 2 && isConsonant(base[base.length - 1])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "tten", r1)) {
+    const base = word.slice(0, -4);
+    if (base.length >= 2 && isConsonant(base[base.length - 1])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  for (const suffix of SIMPLE_SUFFIXES) {
+    if (suffixInR1(word, suffix, r1)) {
+      return { word: word.slice(0, -suffix.length), caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "na", r1)) return { word: word.slice(0, -2), caseRemoved: true };
+  if (suffixInR1(word, "n\xE4", r1)) return { word: word.slice(0, -2), caseRemoved: true };
+  if (suffixInR1(word, "tta", r1)) {
+    const base = word.slice(0, -3);
+    if (base.length > 0 && isVowel(base[base.length - 1])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "tt\xE4", r1)) {
+    const base = word.slice(0, -3);
+    if (base.length > 0 && isVowel(base[base.length - 1])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "ta", r1)) return { word: word.slice(0, -2), caseRemoved: true };
+  if (suffixInR1(word, "t\xE4", r1)) return { word: word.slice(0, -2), caseRemoved: true };
+  if (suffixInR1(word, "n", r1)) {
+    const base = word.slice(0, -1);
+    if (endsWithLongVowel(base)) {
+      return { word: base.slice(0, -1), caseRemoved: true };
+    }
+    if (base.endsWith("ie")) {
+      return { word: base, caseRemoved: true };
+    }
+    if (base.length > 0 && isVowel(base[base.length - 1]) && isConsonant(base[base.length - 2])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "a", r1)) {
+    const base = word.slice(0, -1);
+    if (base.length >= 2 && isVowel(base[base.length - 1]) && isConsonant(base[base.length - 2])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  if (suffixInR1(word, "\xE4", r1)) {
+    const base = word.slice(0, -1);
+    if (base.length >= 2 && isVowel(base[base.length - 1]) && isConsonant(base[base.length - 2])) {
+      return { word: base, caseRemoved: true };
+    }
+  }
+  return { word, caseRemoved: false };
+}
+function step4(word, r2) {
+  for (const { text, checkPo } of STEP4_SUFFIXES) {
+    if (suffixInR2(word, text, r2)) {
+      if (checkPo) {
+        const base = word.slice(0, -text.length);
+        if (base.endsWith("po")) continue;
+      }
+      return word.slice(0, -text.length);
+    }
+  }
+  return word;
+}
+function step5(word, r1, caseRemoved) {
+  if (!caseRemoved) return word;
+  if (suffixInR1(word, "j", r1)) {
+    const base = word.slice(0, -1);
+    if (base.endsWith("o") || base.endsWith("u")) {
+      return base;
+    }
+  }
+  if (suffixInR1(word, "o", r1)) {
+    const base = word.slice(0, -1);
+    if (base.endsWith("j")) {
+      return base;
+    }
+  }
+  if (suffixInR1(word, "i", r1)) {
+    return word.slice(0, -1);
+  }
+  return word;
+}
+function step6(word, r1, r2, caseRemoved) {
+  if (caseRemoved) return word;
+  if (suffixInR1(word, "t", r1)) {
+    const base = word.slice(0, -1);
+    if (base.length > 0 && isVowel(base[base.length - 1])) {
+      word = base;
+      if (suffixInR2(word, "imma", r2)) {
+        return word.slice(0, -4);
+      }
+      if (suffixInR2(word, "mma", r2)) {
+        const beforeMma = word.slice(0, -3);
+        if (!beforeMma.endsWith("po")) {
+          return beforeMma;
+        }
+      }
+    }
+  }
+  return word;
+}
+function step7(word, r1) {
+  if (word.length >= 2 && word.length - 1 >= r1 && endsWithLongVowel(word)) {
+    word = word.slice(0, -1);
+  }
+  let trimmed = word;
+  while (trimmed.length > r1) {
+    const last = trimmed[trimmed.length - 1];
+    if (!isConsonant(last)) break;
+    trimmed = trimmed.slice(0, -1);
+  }
+  if (trimmed.length > 0 && isVowel(trimmed[trimmed.length - 1])) {
+    const afterVowel = word.slice(trimmed.length);
+    if (afterVowel.length >= 2) {
+      const first = afterVowel[0];
+      const second = afterVowel[1];
+      if (isConsonant(first) && first === second) {
+        word = word.slice(0, trimmed.length) + first + afterVowel.slice(2);
+      }
+    }
+  }
+  return word;
+}
+function stem(word) {
+  if (word.length < 3) return word;
+  word = word.toLowerCase();
+  const { r1, r2 } = computeRegions(word);
+  word = step1(word, r1, r2);
+  word = step2(word, r1);
+  const result = step3(word, r1);
+  word = result.word;
+  const caseRemoved = result.caseRemoved;
+  word = step4(word, r2);
+  word = step5(word, r1, caseRemoved);
+  word = step6(word, r1, r2, caseRemoved);
+  word = step7(word, r1);
+  return word;
+}
+var stopWords = /* @__PURE__ */ new Set([
+  "olla",
+  "olen",
+  "olet",
+  "on",
+  "olemme",
+  "olette",
+  "ovat",
+  "ole",
+  "oli",
+  "olisi",
+  "olisit",
+  "olisin",
+  "olisimme",
+  "olisitte",
+  "olisivat",
+  "olit",
+  "olin",
+  "olimme",
+  "olitte",
+  "olivat",
+  "ollut",
+  "olleet",
+  "en",
+  "et",
+  "ei",
+  "emme",
+  "ette",
+  "eiv\xE4t",
+  "min\xE4",
+  "minun",
+  "minut",
+  "minua",
+  "minussa",
+  "minusta",
+  "minuun",
+  "minulla",
+  "minulta",
+  "minulle",
+  "sin\xE4",
+  "sinun",
+  "sinut",
+  "sinua",
+  "sinussa",
+  "sinusta",
+  "sinuun",
+  "sinulla",
+  "sinulta",
+  "sinulle",
+  "h\xE4n",
+  "h\xE4nen",
+  "h\xE4net",
+  "h\xE4nt\xE4",
+  "h\xE4ness\xE4",
+  "h\xE4nest\xE4",
+  "h\xE4neen",
+  "h\xE4nell\xE4",
+  "h\xE4nelt\xE4",
+  "h\xE4nelle",
+  "me",
+  "meid\xE4n",
+  "meid\xE4t",
+  "meit\xE4",
+  "meiss\xE4",
+  "meist\xE4",
+  "meihin",
+  "meill\xE4",
+  "meilt\xE4",
+  "meille",
+  "te",
+  "teid\xE4n",
+  "teid\xE4t",
+  "teit\xE4",
+  "teiss\xE4",
+  "teist\xE4",
+  "teihin",
+  "teill\xE4",
+  "teilt\xE4",
+  "teille",
+  "he",
+  "heid\xE4n",
+  "heid\xE4t",
+  "heit\xE4",
+  "heiss\xE4",
+  "heist\xE4",
+  "heihin",
+  "heill\xE4",
+  "heilt\xE4",
+  "heille",
+  "t\xE4m\xE4",
+  "t\xE4m\xE4n",
+  "t\xE4t\xE4",
+  "t\xE4ss\xE4",
+  "t\xE4st\xE4",
+  "t\xE4h\xE4n",
+  "t\xE4ll\xE4",
+  "t\xE4lt\xE4",
+  "t\xE4lle",
+  "t\xE4n\xE4",
+  "t\xE4ksi",
+  "tuo",
+  "tuon",
+  "tuota",
+  "tuossa",
+  "tuosta",
+  "tuohon",
+  "tuolla",
+  "tuolta",
+  "tuolle",
+  "tuona",
+  "tuoksi",
+  "se",
+  "sen",
+  "sit\xE4",
+  "siin\xE4",
+  "siit\xE4",
+  "siihen",
+  "sill\xE4",
+  "silt\xE4",
+  "sille",
+  "sin\xE4",
+  "siksi",
+  "n\xE4m\xE4",
+  "n\xE4iden",
+  "n\xE4it\xE4",
+  "n\xE4iss\xE4",
+  "n\xE4ist\xE4",
+  "n\xE4ihin",
+  "n\xE4ill\xE4",
+  "n\xE4ilt\xE4",
+  "n\xE4ille",
+  "n\xE4in\xE4",
+  "n\xE4iksi",
+  "nuo",
+  "noiden",
+  "noita",
+  "noissa",
+  "noista",
+  "noihin",
+  "noilla",
+  "noilta",
+  "noille",
+  "noina",
+  "noiksi",
+  "ne",
+  "niiden",
+  "niit\xE4",
+  "niiss\xE4",
+  "niist\xE4",
+  "niihin",
+  "niill\xE4",
+  "niilt\xE4",
+  "niille",
+  "niin\xE4",
+  "niiksi",
+  "kuka",
+  "kenen",
+  "kenet",
+  "ket\xE4",
+  "keness\xE4",
+  "kenest\xE4",
+  "keneen",
+  "kenell\xE4",
+  "kenelt\xE4",
+  "kenelle",
+  "kenen\xE4",
+  "keneksi",
+  "ketk\xE4",
+  "keiden",
+  "keit\xE4",
+  "keiss\xE4",
+  "keist\xE4",
+  "keihin",
+  "keill\xE4",
+  "keilt\xE4",
+  "keille",
+  "kein\xE4",
+  "keiksi",
+  "mik\xE4",
+  "mink\xE4",
+  "mit\xE4",
+  "miss\xE4",
+  "mist\xE4",
+  "mihin",
+  "mill\xE4",
+  "milt\xE4",
+  "mille",
+  "min\xE4",
+  "miksi",
+  "mitk\xE4",
+  "joka",
+  "jonka",
+  "jota",
+  "jossa",
+  "josta",
+  "johon",
+  "jolla",
+  "jolta",
+  "jolle",
+  "jona",
+  "joksi",
+  "jotka",
+  "joiden",
+  "joita",
+  "joissa",
+  "joista",
+  "joihin",
+  "joilla",
+  "joilta",
+  "joille",
+  "joina",
+  "joiksi",
+  "ett\xE4",
+  "ja",
+  "jos",
+  "koska",
+  "kuin",
+  "mutta",
+  "niin",
+  "sek\xE4",
+  "sill\xE4",
+  "tai",
+  "vaan",
+  "vai",
+  "vaikka",
+  "kanssa",
+  "mukaan",
+  "noin",
+  "poikki",
+  "yli",
+  "kun",
+  "nyt",
+  "itse"
+]);
+var finnish = {
+  name: "finnish",
+  stemmer: stem,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9äö]+/gi }
+};
+
+export { finnish };

package/dist/languages/french.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-ClUGLuoC.js';
+
+declare const french: LanguageModule;
+
+export { french };