@lokascript/semantic 1.0.0

package/src/tokenizers/morphology/arabic-normalizer.ts

@@ -0,0 +1,242 @@
+/**
+ * Arabic Morphological Normalizer
+ *
+ * Arabic uses a complex root-pattern morphology system where most words
+ * are derived from triliteral (3-consonant) roots. This normalizer focuses
+ * on prefix/suffix stripping rather than full root extraction.
+ *
+ * Key features:
+ * - Definite article prefix: ال (al-)
+ * - Conjunction/preposition prefixes: و (wa-), ف (fa-), ب (bi-), ل (li-), ك (ka-)
+ * - Verb prefixes (present tense markers): ي (ya-), ت (ta-), ن (na-), أ (a-)
+ * - Plural/gender suffixes: ون (ūn), ين (īn), ات (āt), ة (a)
+ * - Pronoun suffixes: ها (hā), هم (hum), etc.
+ * - Diacritics handling: Words with and without diacritics should match
+ *
+ * Examples:
+ *   والتبديل → تبديل → بدّل (and the changing → changing → change!)
+ *   يبدّل → بدّل (he changes → change!)
+ *   المستخدمين → مستخدم (the users → user)
+ */
+
+import type { MorphologicalNormalizer, NormalizationResult, PrefixRule } from './types';
+import { noChange, normalized } from './types';
+
+/**
+ * Check if a character is Arabic.
+ */
+function isArabic(char: string): boolean {
+  const code = char.charCodeAt(0);
+  return (
+    (code >= 0x0600 && code <= 0x06ff) || // Arabic
+    (code >= 0x0750 && code <= 0x077f) || // Arabic Supplement
+    (code >= 0x08a0 && code <= 0x08ff) || // Arabic Extended-A
+    (code >= 0xfb50 && code <= 0xfdff) || // Arabic Presentation Forms-A
+    (code >= 0xfe70 && code <= 0xfeff)
+  ); // Arabic Presentation Forms-B
+}
+
+/**
+ * Check if a word contains Arabic characters.
+ */
+function containsArabic(word: string): boolean {
+  for (const char of word) {
+    if (isArabic(char)) return true;
+  }
+  return false;
+}
+
+/**
+ * Remove Arabic diacritics (tashkeel) from a word.
+ * This helps match words regardless of vocalization marks.
+ */
+function removeDiacritics(word: string): string {
+  // Arabic diacritics: fatha, kasra, damma, sukun, shadda, etc.
+  return word.replace(/[\u064B-\u0652\u0670]/g, '');
+}
+
+/**
+ * Prefix rules for Arabic, ordered by priority.
+ * Combined prefixes should be checked first.
+ */
+const COMBINED_PREFIXES: readonly PrefixRule[] = [
+  // Conjunction + article combinations (4 chars)
+  { pattern: 'وال', confidencePenalty: 0.15, prefixType: 'conjunction' }, // wa + al
+  { pattern: 'فال', confidencePenalty: 0.15, prefixType: 'conjunction' }, // fa + al
+  { pattern: 'بال', confidencePenalty: 0.15, prefixType: 'preposition' }, // bi + al
+  { pattern: 'كال', confidencePenalty: 0.15, prefixType: 'preposition' }, // ka + al
+  { pattern: 'لل', confidencePenalty: 0.12, prefixType: 'preposition' }, // li + al (assimilation)
+];
+
+/**
+ * Single prefix rules.
+ * Note: Single-character prefixes require minimum 3-char remaining stem
+ * to avoid over-stripping words where the character is part of the root.
+ */
+const SINGLE_PREFIXES: readonly PrefixRule[] = [
+  // Definite article (2 chars) - can leave 2-char stem
+  { pattern: 'ال', confidencePenalty: 0.08, prefixType: 'article', minRemaining: 2 },
+
+  // Conjunctions and prepositions (1 char) - need longer stem to be safe
+  { pattern: 'و', confidencePenalty: 0.08, prefixType: 'conjunction', minRemaining: 3 }, // wa- (and)
+  { pattern: 'ف', confidencePenalty: 0.08, prefixType: 'conjunction', minRemaining: 3 }, // fa- (then/so)
+  { pattern: 'ب', confidencePenalty: 0.1, prefixType: 'preposition', minRemaining: 3 }, // bi- (with/by)
+  { pattern: 'ل', confidencePenalty: 0.1, prefixType: 'preposition', minRemaining: 3 }, // li- (to/for)
+  { pattern: 'ك', confidencePenalty: 0.1, prefixType: 'preposition', minRemaining: 3 }, // ka- (like/as)
+];
+
+/**
+ * Verb prefixes (present tense markers).
+ * These are more tentative as they change verb meaning.
+ * Require minimum 3-char remaining to avoid over-stripping.
+ */
+const VERB_PREFIXES: readonly PrefixRule[] = [
+  { pattern: 'ي', confidencePenalty: 0.12, prefixType: 'verb-marker', minRemaining: 3 }, // ya- (he/it)
+  { pattern: 'ت', confidencePenalty: 0.12, prefixType: 'verb-marker', minRemaining: 3 }, // ta- (she/you)
+  { pattern: 'ن', confidencePenalty: 0.12, prefixType: 'verb-marker', minRemaining: 3 }, // na- (we)
+  { pattern: 'أ', confidencePenalty: 0.12, prefixType: 'verb-marker', minRemaining: 3 }, // a- (I)
+  { pattern: 'ا', confidencePenalty: 0.12, prefixType: 'verb-marker', minRemaining: 3 }, // a- without hamza
+];
+
+/**
+ * Suffix rules for Arabic.
+ */
+const SUFFIXES: readonly { pattern: string; confidencePenalty: number; type: string }[] = [
+  // Plural forms
+  { pattern: 'ون', confidencePenalty: 0.1, type: 'masculine-plural' },
+  { pattern: 'ين', confidencePenalty: 0.1, type: 'masculine-plural-accusative' },
+  { pattern: 'ات', confidencePenalty: 0.1, type: 'feminine-plural' },
+  // Dual forms
+  { pattern: 'ان', confidencePenalty: 0.1, type: 'dual-nominative' },
+  { pattern: 'ين', confidencePenalty: 0.1, type: 'dual-accusative' },
+  // Pronoun suffixes
+  { pattern: 'ها', confidencePenalty: 0.1, type: 'pronoun-her' },
+  { pattern: 'هم', confidencePenalty: 0.1, type: 'pronoun-them' },
+  { pattern: 'هن', confidencePenalty: 0.1, type: 'pronoun-them-f' },
+  { pattern: 'نا', confidencePenalty: 0.1, type: 'pronoun-us' },
+  { pattern: 'كم', confidencePenalty: 0.1, type: 'pronoun-you-pl' },
+  { pattern: 'ك', confidencePenalty: 0.08, type: 'pronoun-you' },
+  { pattern: 'ه', confidencePenalty: 0.08, type: 'pronoun-him' },
+  { pattern: 'ي', confidencePenalty: 0.08, type: 'pronoun-me' },
+  // Feminine marker
+  { pattern: 'ة', confidencePenalty: 0.08, type: 'feminine' },
+];
+
+/**
+ * Arabic morphological normalizer.
+ */
+export class ArabicMorphologicalNormalizer implements MorphologicalNormalizer {
+  readonly language = 'ar';
+
+  /**
+   * Check if a word might be an Arabic word that can be normalized.
+   */
+  isNormalizable(word: string): boolean {
+    if (!containsArabic(word)) return false;
+    // Arabic words are typically at least 2 characters
+    if (word.length < 2) return false;
+    return true;
+  }
+
+  /**
+   * Normalize an Arabic word by stripping prefixes and suffixes.
+   */
+  normalize(word: string): NormalizationResult {
+    // Remove diacritics for consistent matching
+    let stem = removeDiacritics(word);
+    let confidence = 1.0;
+    const removedPrefixes: string[] = [];
+    const removedSuffixes: string[] = [];
+
+    // Try combined prefixes first (longest match)
+    for (const rule of COMBINED_PREFIXES) {
+      if (stem.startsWith(rule.pattern)) {
+        const remaining = stem.slice(rule.pattern.length);
+        // Must leave a meaningful stem (at least 2 characters)
+        if (remaining.length >= 2) {
+          stem = remaining;
+          confidence -= rule.confidencePenalty;
+          removedPrefixes.push(rule.pattern);
+          break; // Only one combined prefix
+        }
+      }
+    }
+
+    // Try single prefixes (if no combined prefix was found)
+    if (removedPrefixes.length === 0) {
+      for (const rule of SINGLE_PREFIXES) {
+        if (stem.startsWith(rule.pattern)) {
+          const remaining = stem.slice(rule.pattern.length);
+          const minLen = rule.minRemaining ?? 2;
+          if (remaining.length >= minLen) {
+            stem = remaining;
+            confidence -= rule.confidencePenalty;
+            removedPrefixes.push(rule.pattern);
+            break; // Only one prefix at a time for now
+          }
+        }
+      }
+    }
+
+    // Try verb prefixes ONLY for words that look like verbs (not nouns)
+    // Skip if the word has noun-pattern suffixes or pronoun suffixes
+    // This prevents stripping ت from تغييرات (changes) or تغييرها (her change)
+    const looksLikeNoun =
+      stem.endsWith('ات') ||
+      stem.endsWith('ة') ||
+      stem.endsWith('ون') ||
+      stem.endsWith('ين') ||
+      stem.endsWith('ها') ||
+      stem.endsWith('هم') ||
+      stem.endsWith('هن') ||
+      stem.endsWith('نا') ||
+      stem.endsWith('كم');
+    if (
+      !looksLikeNoun &&
+      (removedPrefixes.length === 0 || removedPrefixes[0] === 'و' || removedPrefixes[0] === 'ف')
+    ) {
+      for (const rule of VERB_PREFIXES) {
+        if (stem.startsWith(rule.pattern)) {
+          const remaining = stem.slice(rule.pattern.length);
+          const minLen = rule.minRemaining ?? 3;
+          if (remaining.length >= minLen) {
+            stem = remaining;
+            confidence -= rule.confidencePenalty;
+            removedPrefixes.push(rule.pattern);
+            break;
+          }
+        }
+      }
+    }
+
+    // Try suffixes (can apply multiple passes)
+    for (const rule of SUFFIXES) {
+      if (stem.endsWith(rule.pattern)) {
+        const remaining = stem.slice(0, -rule.pattern.length);
+        // Must leave a meaningful stem
+        if (remaining.length >= 2) {
+          stem = remaining;
+          confidence -= rule.confidencePenalty;
+          removedSuffixes.push(rule.pattern);
+          // Don't break - some suffixes can be stacked
+        }
+      }
+    }
+
+    // Ensure confidence stays reasonable
+    confidence = Math.max(0.5, confidence);
+
+    // If nothing was stripped, return unchanged
+    if (removedPrefixes.length === 0 && removedSuffixes.length === 0) {
+      return noChange(word);
+    }
+
+    return normalized(stem, confidence, {
+      removedPrefixes,
+      removedSuffixes,
+    });
+  }
+}
+
+// Export singleton instance
+export const arabicMorphologicalNormalizer = new ArabicMorphologicalNormalizer();

package/src/tokenizers/morphology/french-normalizer.ts

@@ -0,0 +1,268 @@
+/**
+ * French Morphological Normalizer
+ *
+ * Reduces French verb conjugations to their infinitive forms.
+ * French has three verb conjugation groups:
+ * - 1st group: -er verbs (parler, montrer, afficher)
+ * - 2nd group: -ir verbs with -iss- forms (finir, choisir)
+ * - 3rd group: irregular -ir, -re, -oir verbs (partir, prendre, voir)
+ *
+ * Key features:
+ * - Reflexive verb handling: se montrer → montrer
+ * - Regular conjugation patterns for all three groups
+ * - Past participle (-é, -i, -u) and present participle (-ant) forms
+ *
+ * Examples:
+ *   affiche → afficher (3rd person present)
+ *   montrant → montrer (present participle)
+ *   caché → cacher (past participle)
+ *   finissons → finir (1st person plural present)
+ */
+
+import type { MorphologicalNormalizer, NormalizationResult, ConjugationType } from './types';
+import { noChange, normalized } from './types';
+
+/**
+ * Check if a word looks like a French verb.
+ */
+function looksLikeFrenchVerb(word: string): boolean {
+  const lower = word.toLowerCase();
+  // Check for infinitive endings
+  if (lower.endsWith('er') || lower.endsWith('ir') || lower.endsWith('re')) return true;
+  // Check for common conjugation endings
+  if (lower.endsWith('ant')) return true; // present participle
+  if (lower.endsWith('é') || lower.endsWith('i') || lower.endsWith('u')) return true; // past participles
+  // Check for French-specific characters
+  if (/[àâäéèêëïîôùûüÿçœæ]/i.test(word)) return true;
+  return false;
+}
+
+/**
+ * Reflexive pronouns that attach to verbs in imperative form.
+ */
+const REFLEXIVE_SUFFIXES = ['toi', 'vous', 'nous'];
+
+/**
+ * -ER verb conjugation endings (1st group - largest group).
+ */
+const ER_ENDINGS: readonly {
+  ending: string;
+  stem: string;
+  confidence: number;
+  type: ConjugationType;
+}[] = [
+  // Present participle
+  { ending: 'ant', stem: 'er', confidence: 0.88, type: 'gerund' },
+  // Past participle
+  { ending: 'é', stem: 'er', confidence: 0.88, type: 'participle' },
+  { ending: 'ée', stem: 'er', confidence: 0.88, type: 'participle' },
+  { ending: 'és', stem: 'er', confidence: 0.88, type: 'participle' },
+  { ending: 'ées', stem: 'er', confidence: 0.88, type: 'participle' },
+  // Present indicative
+  { ending: 'e', stem: 'er', confidence: 0.75, type: 'present' }, // je/il/elle
+  { ending: 'es', stem: 'er', confidence: 0.78, type: 'present' }, // tu
+  { ending: 'ons', stem: 'er', confidence: 0.85, type: 'present' }, // nous
+  { ending: 'ez', stem: 'er', confidence: 0.85, type: 'present' }, // vous
+  { ending: 'ent', stem: 'er', confidence: 0.82, type: 'present' }, // ils/elles
+  // Imperfect
+  { ending: 'ais', stem: 'er', confidence: 0.82, type: 'past' }, // je/tu
+  { ending: 'ait', stem: 'er', confidence: 0.82, type: 'past' }, // il/elle
+  { ending: 'ions', stem: 'er', confidence: 0.85, type: 'past' }, // nous
+  { ending: 'iez', stem: 'er', confidence: 0.85, type: 'past' }, // vous
+  { ending: 'aient', stem: 'er', confidence: 0.85, type: 'past' }, // ils/elles
+  // Simple past (passé simple)
+  { ending: 'ai', stem: 'er', confidence: 0.8, type: 'past' }, // je
+  { ending: 'as', stem: 'er', confidence: 0.78, type: 'past' }, // tu
+  { ending: 'a', stem: 'er', confidence: 0.75, type: 'past' }, // il/elle
+  { ending: 'âmes', stem: 'er', confidence: 0.88, type: 'past' }, // nous
+  { ending: 'âtes', stem: 'er', confidence: 0.88, type: 'past' }, // vous
+  { ending: 'èrent', stem: 'er', confidence: 0.88, type: 'past' }, // ils/elles
+  // Future
+  { ending: 'erai', stem: 'er', confidence: 0.85, type: 'future' }, // je
+  { ending: 'eras', stem: 'er', confidence: 0.85, type: 'future' }, // tu
+  { ending: 'era', stem: 'er', confidence: 0.82, type: 'future' }, // il/elle
+  { ending: 'erons', stem: 'er', confidence: 0.88, type: 'future' }, // nous
+  { ending: 'erez', stem: 'er', confidence: 0.88, type: 'future' }, // vous
+  { ending: 'eront', stem: 'er', confidence: 0.88, type: 'future' }, // ils/elles
+  // Conditional
+  { ending: 'erais', stem: 'er', confidence: 0.85, type: 'conditional' }, // je/tu
+  { ending: 'erait', stem: 'er', confidence: 0.85, type: 'conditional' }, // il/elle
+  { ending: 'erions', stem: 'er', confidence: 0.88, type: 'conditional' }, // nous
+  { ending: 'eriez', stem: 'er', confidence: 0.88, type: 'conditional' }, // vous
+  { ending: 'eraient', stem: 'er', confidence: 0.88, type: 'conditional' }, // ils/elles
+  // Subjunctive
+  { ending: 'ions', stem: 'er', confidence: 0.8, type: 'subjunctive' }, // nous
+  { ending: 'iez', stem: 'er', confidence: 0.8, type: 'subjunctive' }, // vous
+  // Imperative
+  { ending: 'ons', stem: 'er', confidence: 0.82, type: 'imperative' }, // nous
+  { ending: 'ez', stem: 'er', confidence: 0.82, type: 'imperative' }, // vous
+  // Infinitive
+  { ending: 'er', stem: 'er', confidence: 0.92, type: 'dictionary' },
+];
+
+/**
+ * -IR verb conjugation endings (2nd group - verbs with -iss- forms).
+ * Examples: finir → finissons, choisir → choisissons
+ */
+const IR_ENDINGS: readonly {
+  ending: string;
+  stem: string;
+  confidence: number;
+  type: ConjugationType;
+}[] = [
+  // Present participle
+  { ending: 'issant', stem: 'ir', confidence: 0.88, type: 'gerund' },
+  // Past participle
+  { ending: 'i', stem: 'ir', confidence: 0.8, type: 'participle' },
+  { ending: 'ie', stem: 'ir', confidence: 0.82, type: 'participle' },
+  { ending: 'is', stem: 'ir', confidence: 0.78, type: 'participle' },
+  { ending: 'ies', stem: 'ir', confidence: 0.82, type: 'participle' },
+  // Present indicative with -iss-
+  { ending: 'is', stem: 'ir', confidence: 0.78, type: 'present' }, // je/tu
+  { ending: 'it', stem: 'ir', confidence: 0.78, type: 'present' }, // il/elle
+  { ending: 'issons', stem: 'ir', confidence: 0.88, type: 'present' }, // nous
+  { ending: 'issez', stem: 'ir', confidence: 0.88, type: 'present' }, // vous
+  { ending: 'issent', stem: 'ir', confidence: 0.88, type: 'present' }, // ils/elles
+  // Imperfect
+  { ending: 'issais', stem: 'ir', confidence: 0.85, type: 'past' }, // je/tu
+  { ending: 'issait', stem: 'ir', confidence: 0.85, type: 'past' }, // il/elle
+  { ending: 'issions', stem: 'ir', confidence: 0.88, type: 'past' }, // nous
+  { ending: 'issiez', stem: 'ir', confidence: 0.88, type: 'past' }, // vous
+  { ending: 'issaient', stem: 'ir', confidence: 0.88, type: 'past' }, // ils/elles
+  // Future
+  { ending: 'irai', stem: 'ir', confidence: 0.85, type: 'future' }, // je
+  { ending: 'iras', stem: 'ir', confidence: 0.85, type: 'future' }, // tu
+  { ending: 'ira', stem: 'ir', confidence: 0.82, type: 'future' }, // il/elle
+  { ending: 'irons', stem: 'ir', confidence: 0.88, type: 'future' }, // nous
+  { ending: 'irez', stem: 'ir', confidence: 0.88, type: 'future' }, // vous
+  { ending: 'iront', stem: 'ir', confidence: 0.88, type: 'future' }, // ils/elles
+  // Infinitive
+  { ending: 'ir', stem: 'ir', confidence: 0.9, type: 'dictionary' },
+];
+
+/**
+ * -RE verb conjugation endings (3rd group).
+ * Examples: prendre, vendre, attendre
+ */
+const RE_ENDINGS: readonly {
+  ending: string;
+  stem: string;
+  confidence: number;
+  type: ConjugationType;
+}[] = [
+  // Present participle
+  { ending: 'ant', stem: 're', confidence: 0.82, type: 'gerund' },
+  // Past participle (common patterns)
+  { ending: 'u', stem: 're', confidence: 0.8, type: 'participle' },
+  { ending: 'ue', stem: 're', confidence: 0.82, type: 'participle' },
+  { ending: 'us', stem: 're', confidence: 0.82, type: 'participle' },
+  { ending: 'ues', stem: 're', confidence: 0.82, type: 'participle' },
+  // Present indicative
+  { ending: 's', stem: 're', confidence: 0.72, type: 'present' }, // je/tu
+  { ending: 'd', stem: 're', confidence: 0.75, type: 'present' }, // il/elle (prend, vend)
+  { ending: 'ons', stem: 're', confidence: 0.82, type: 'present' }, // nous
+  { ending: 'ez', stem: 're', confidence: 0.82, type: 'present' }, // vous
+  { ending: 'ent', stem: 're', confidence: 0.8, type: 'present' }, // ils/elles
+  // Infinitive
+  { ending: 're', stem: 're', confidence: 0.9, type: 'dictionary' },
+];
+
+/**
+ * All endings combined, sorted by length (longest first).
+ */
+const ALL_ENDINGS = [...ER_ENDINGS, ...IR_ENDINGS, ...RE_ENDINGS].sort(
+  (a, b) => b.ending.length - a.ending.length
+);
+
+/**
+ * French morphological normalizer.
+ */
+export class FrenchMorphologicalNormalizer implements MorphologicalNormalizer {
+  readonly language = 'fr';
+
+  /**
+   * Check if a word might be a French verb that can be normalized.
+   */
+  isNormalizable(word: string): boolean {
+    if (word.length < 3) return false;
+    return looksLikeFrenchVerb(word);
+  }
+
+  /**
+   * Normalize a French word to its infinitive form.
+   */
+  normalize(word: string): NormalizationResult {
+    const lower = word.toLowerCase();
+
+    // Check if this is already an infinitive (no change needed)
+    if (lower.endsWith('er') || lower.endsWith('ir') || lower.endsWith('re')) {
+      // Simple infinitive, return as-is
+      if (lower.length >= 4) {
+        return noChange(word);
+      }
+    }
+
+    // Try reflexive verb normalization first (for imperative forms like "montrez-vous")
+    const reflexiveResult = this.tryReflexiveNormalization(lower);
+    if (reflexiveResult) return reflexiveResult;
+
+    // Try standard conjugation normalization
+    const conjugationResult = this.tryConjugationNormalization(lower);
+    if (conjugationResult) return conjugationResult;
+
+    // No normalization needed
+    return noChange(word);
+  }
+
+  /**
+   * Try to normalize a reflexive verb (imperative forms with attached pronouns).
+   * Examples: montrez-vous → montrer, lève-toi → lever
+   */
+  private tryReflexiveNormalization(word: string): NormalizationResult | null {
+    // Check for hyphenated reflexive forms (e.g., "montrez-vous")
+    for (const suffix of REFLEXIVE_SUFFIXES) {
+      const hyphenatedSuffix = '-' + suffix;
+      if (word.endsWith(hyphenatedSuffix)) {
+        const withoutReflexive = word.slice(0, -hyphenatedSuffix.length);
+
+        // Try to normalize the remaining part
+        const innerResult = this.tryConjugationNormalization(withoutReflexive);
+        if (innerResult && innerResult.stem !== withoutReflexive) {
+          return normalized(innerResult.stem, innerResult.confidence * 0.95, {
+            removedSuffixes: [hyphenatedSuffix, ...(innerResult.metadata?.removedSuffixes || [])],
+            conjugationType: 'reflexive',
+          });
+        }
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Try to normalize a conjugated verb to its infinitive.
+   */
+  private tryConjugationNormalization(word: string): NormalizationResult | null {
+    for (const rule of ALL_ENDINGS) {
+      if (word.endsWith(rule.ending)) {
+        const stemBase = word.slice(0, -rule.ending.length);
+
+        // Must have a meaningful stem (at least 2 characters)
+        if (stemBase.length < 2) continue;
+
+        // Reconstruct infinitive
+        const infinitive = stemBase + rule.stem;
+
+        return normalized(infinitive, rule.confidence, {
+          removedSuffixes: [rule.ending],
+          conjugationType: rule.type,
+        });
+      }
+    }
+
+    return null;
+  }
+}
+
+// Export singleton instance
+export const frenchMorphologicalNormalizer = new FrenchMorphologicalNormalizer();