@delali/narsil 0.1.14 → 0.2.0

package/dist/languages/slovenian.d.ts

@@ -1,4 +1,4 @@
-import { L as LanguageModule } from '../language-ONQ3iVDf.js';
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
 
 declare const slovenian: LanguageModule;
 

package/dist/languages/slovenian.mjs

@@ -1,3 +1,6 @@
+import { removeMarks } from '../chunk-KH3YGMVT.mjs';
+import { withNormalisedSpellings } from '../chunk-GHJWC2E5.mjs';
+
 // src/languages/slovenian.ts
 var NOUN_SUFFIXES = [
   "ovanje",
@@ -512,11 +515,17 @@ var stopWords = /* @__PURE__ */ new Set([
   "\u0161tiri",
   "\u017Ee"
 ]);
+var TONAL_MARKS = /[\u0300\u0301\u0302]/g;
+function normalize(token) {
+  return removeMarks(token, TONAL_MARKS);
+}
 var slovenian = {
   name: "slovenian",
+  revision: "1",
   stemmer: stem,
-  stopWords,
-  tokenizer: { splitPattern: /[^a-z0-9čšž]+/gi }
+  stopWords: withNormalisedSpellings(stopWords, normalize),
+  normalizer: normalize,
+  tokenizer: { splitPattern: /[^a-z0-9čšžáéóú]+/gi }
 };
 
 export { slovenian };

package/dist/languages/sorani.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
+
+declare const sorani: LanguageModule;
+
+export { sorani };

package/dist/languages/sorani.mjs

@@ -0,0 +1,156 @@
+import { withNormalisedSpellings } from '../chunk-GHJWC2E5.mjs';
+
+// src/languages/sorani.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "\u0626\u06D5\u0645",
+  "\u0626\u06D5\u0648",
+  "\u0626\u06D5\u0648\u0627\u0646",
+  "\u0626\u06D5\u0648\u06D5\u06CC",
+  "\u0626\u06CE\u0645\u06D5",
+  "\u0626\u06CE\u0648\u06D5",
+  "\u0628\u06D5",
+  "\u0628\u06D5\u0628\u06CE",
+  "\u0628\u06D5\u067E\u06CE\u06CC",
+  "\u0628\u06D5\u062F\u06D5\u0645",
+  "\u0628\u06D5\u0631\u062F\u06D5\u0645",
+  "\u0628\u06D5\u0631\u0644\u06D5",
+  "\u0628\u06D5\u0631\u06D5\u0648\u06D5",
+  "\u0628\u06D5\u0631\u06D5\u0648\u06CC",
+  "\u0628\u06D5\u0644\u0627\u06CC",
+  "\u0628\u06C6",
+  "\u0628\u06CE",
+  "\u0628\u06CE\u062C\u06AF\u06D5",
+  "\u067E\u0627\u0634",
+  "\u067E\u06CE",
+  "\u067E\u06CE\u0634",
+  "\u062A\u06C6",
+  "\u062A\u06CE",
+  "\u062C\u06AF\u06D5",
+  "\u0686\u06D5\u0646\u062F",
+  "\u062F\u06D5",
+  "\u062F\u06D5\u06A9\u0627\u062A",
+  "\u062F\u06D5\u06AF\u06D5\u06B5",
+  "\u062F\u0648\u0627\u06CC",
+  "\u062F\u0648\u0648",
+  "\u0633\u06D5\u0631",
+  "\u06A9\u0631\u062F",
+  "\u06A9\u06D5",
+  "\u0644\u06D5",
+  "\u0644\u06D5\u0628\u0627\u0628\u06D5\u062A",
+  "\u0644\u06D5\u0628\u0627\u062A\u06CC",
+  "\u0644\u06D5\u0628\u0627\u0631\u06D5\u06CC",
+  "\u0644\u06D5\u0628\u0631\u06CE\u062A\u06CC",
+  "\u0644\u06D5\u0628\u0646",
+  "\u0644\u06D5\u0628\u06D5\u0631",
+  "\u0644\u06D5\u0628\u06D5\u06CC\u0646\u06CC",
+  "\u0644\u06D5\u067E\u06CE\u0646\u0627\u0648\u06CC",
+  "\u0644\u06D5\u062F\u06D5\u0645",
+  "\u0644\u06D5\u0631\u06D5\u0648\u06CC",
+  "\u0644\u06D5\u0631\u06CE",
+  "\u0644\u06D5\u0631\u06CE\u06AF\u0627",
+  "\u0644\u06D5\u0698\u06CE\u0631",
+  "\u0644\u06D5\u0633\u06D5\u0631",
+  "\u0644\u06D5\u06AF\u06D5\u06B5",
+  "\u0644\u06D5\u0644\u0627\u06CC\u06D5\u0646",
+  "\u0644\u06D5\u0646\u0627\u0648",
+  "\u0644\u06D5\u0646\u06CE\u0648",
+  "\u0644\u06D5\u0648",
+  "\u0644\u06CE",
+  "\u0645\u0646",
+  "\u0646\u0627\u0648",
+  "\u0646\u06CE\u0648\u0627\u0646",
+  "\u0647\u06D5\u0631",
+  "\u0647\u06D5\u0631\u0648\u06D5\u0647\u0627",
+  "\u06BE\u06D5\u0631",
+  "\u06BE\u06D5\u0631\u0648\u06D5\u06BE\u0627",
+  "\u0648",
+  "\u0648\u06D5\u06A9",
+  "\u06CC"
+]);
+var YEH = "\u064A";
+var DOTLESS_YEH = "\u0649";
+var FARSI_YEH = "\u06CC";
+var KAF = "\u0643";
+var KEHEH = "\u06A9";
+var HEH = "\u0647";
+var AE = "\u06D5";
+var ZWNJ = "\u200C";
+var HEH_DOACHASHMEE = "\u06BE";
+var TEH_MARBUTA = "\u0629";
+var REH = "\u0631";
+var RREH = "\u0695";
+var RREH_ABOVE = "\u0692";
+var TATWEEL = "\u0640";
+var FATHATAN = "\u064B";
+var DAMMATAN = "\u064C";
+var KASRATAN = "\u064D";
+var FATHA = "\u064E";
+var DAMMA = "\u064F";
+var KASRA = "\u0650";
+var SHADDA = "\u0651";
+var SUKUN = "\u0652";
+var FORMAT_CHARACTER = /\p{Cf}/u;
+var REWRITTEN = /[\u0629\u0631\u0640\u064b-\u0652\u0649\u064a\u0643\u0647\u0692\u06be]|\p{Cf}/u;
+function normalize(token) {
+  if (!REWRITTEN.test(token)) return token;
+  const chars = Array.from(token);
+  for (let i = 0; i < chars.length; i++) {
+    switch (chars[i]) {
+      case YEH:
+      case DOTLESS_YEH:
+        chars[i] = FARSI_YEH;
+        break;
+      case KAF:
+        chars[i] = KEHEH;
+        break;
+      case ZWNJ:
+        if (i > 0 && chars[i - 1] === HEH) chars[i - 1] = AE;
+        chars.splice(i, 1);
+        i--;
+        break;
+      case HEH:
+        if (i === chars.length - 1) chars[i] = AE;
+        break;
+      case TEH_MARBUTA:
+        chars[i] = AE;
+        break;
+      case HEH_DOACHASHMEE:
+        chars[i] = HEH;
+        break;
+      case REH:
+        if (i === 0) chars[i] = RREH;
+        break;
+      case RREH_ABOVE:
+        chars[i] = RREH;
+        break;
+      case TATWEEL:
+      case FATHATAN:
+      case DAMMATAN:
+      case KASRATAN:
+      case FATHA:
+      case DAMMA:
+      case KASRA:
+      case SHADDA:
+      case SUKUN:
+        chars.splice(i, 1);
+        i--;
+        break;
+      default:
+        if (FORMAT_CHARACTER.test(chars[i])) {
+          chars.splice(i, 1);
+          i--;
+        }
+    }
+  }
+  return chars.join("");
+}
+var sorani = {
+  name: "sorani",
+  revision: "1",
+  stemmer: null,
+  stopWords: withNormalisedSpellings(stopWords, normalize),
+  normalizer: normalize,
+  tokenizer: { splitPattern: /[^\u0621-\u065f\u0660-\u0669\u066e-\u06d5\u06f0-\u06f9\u200ca-z0-9]+/gi }
+};
+
+export { sorani };

package/dist/languages/spanish.d.ts

@@ -1,4 +1,4 @@
-import { L as LanguageModule } from '../language-ONQ3iVDf.js';
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
 
 declare const spanish: LanguageModule;