@lokascript/framework 2.8.0 → 2.9.0

package/dist/core/index.js

@@ -125,6 +125,9 @@ function isQuote(char) {
 function isDigit(char) {
   return /\d/.test(char);
 }
+function stripOptionalDiacritics(word) {
+  return word.replace(/[ً-ْٰ]/g, "");
+}
 function isAsciiLetter(char) {
   return /[a-zA-Z]/.test(char);
 }
@@ -1080,7 +1083,7 @@ var _BaseTokenizer = class _BaseTokenizer {
    * @returns Word without diacritics
    */
   removeDiacritics(word) {
-    return word.replace(/[\u064B-\u0652\u0670]/g, "");
+    return stripOptionalDiacritics(word);
   }
   /**
    * Try to match a keyword from profile at the current position.
@@ -1171,24 +1174,40 @@ var _BaseTokenizer = class _BaseTokenizer {
     });
   }
   /**
-   * Look up a keyword by native word (case-insensitive).
+   * Look up a keyword by native word (case-insensitive, diacritic-insensitive).
    * O(1) lookup using the keyword map.
    *
+   * The map is INDEXED both with and without diacritics (see
+   * `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
+   * that: it lets a surface form carrying harakat the profile does not happen to
+   * spell still find its entry. Only consulted after the exact lookup misses, so
+   * every previously-matching word resolves byte-identically.
+   *
+   * Half-implementing this — indexing stripped but querying exact — is what made
+   * diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
+   * `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
+   * exists to prevent exactly that handed the word on, and the single-char `ب`
+   * bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
+   *
    * @param native - Native word to look up
    * @returns KeywordEntry if found, undefined otherwise
    */
   lookupKeyword(native) {
-    return this.profileKeywordMap.get(native.toLowerCase());
+    const exact = this.profileKeywordMap.get(native.toLowerCase());
+    if (exact) return exact;
+    const stripped = this.removeDiacritics(native);
+    if (stripped === native) return void 0;
+    return this.profileKeywordMap.get(stripped.toLowerCase());
   }
   /**
-   * Check if a word is a known keyword (case-insensitive).
-   * O(1) lookup using the keyword map.
+   * Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
+   * O(1) lookup using the keyword map. See {@link lookupKeyword}.
    *
    * @param native - Native word to check
    * @returns true if the word is a keyword
    */
   isKeyword(native) {
-    return this.profileKeywordMap.has(native.toLowerCase());
+    return this.lookupKeyword(native) !== void 0;
   }
   /**
    * Set the morphological normalizer for this tokenizer.
@@ -2984,6 +3003,7 @@ export {
   noChange,
   normalized,
   patternMatcher,
+  stripOptionalDiacritics,
   validateValueType,
   withDefaultExtractors
 };