@lokascript/framework 2.0.0 → 2.2.0

package/dist/core/tokenization/index.js

@@ -1312,7 +1312,140 @@ function normalized(stem, confidence, metadata) {
   }
   return { stem, confidence };
 }
+
+// src/core/tokenization/morphology/base-normalizer.ts
+var BaseMorphologicalNormalizer = class {
+  constructor(config) {
+    this.language = config.language;
+    this.config = {
+      minWordLength: 3,
+      minStemLength: 2,
+      ...config
+    };
+  }
+  /**
+   * Standard normalization pipeline. Override for custom behavior.
+   */
+  normalize(word) {
+    const lower = word.toLowerCase();
+    if (this.isAlreadyNormalized(lower)) {
+      return noChange(word);
+    }
+    if (this.config.reflexiveSuffixes) {
+      const reflexive = this.tryReflexiveNormalization(lower);
+      if (reflexive) return reflexive;
+    }
+    if (this.config.endings) {
+      const conjugation = this.tryConjugationEndings(lower);
+      if (conjugation) return conjugation;
+    }
+    if (this.config.suffixRules) {
+      const suffix = this.trySuffixRules(lower);
+      if (suffix) return suffix;
+    }
+    if (this.config.prefixRules) {
+      const prefix = this.tryPrefixRules(lower);
+      if (prefix) return prefix;
+    }
+    return noChange(word);
+  }
+  /**
+   * Check if word is already in dictionary form (e.g., ends in -ar/-er/-ir).
+   * Override for language-specific checks.
+   */
+  isAlreadyNormalized(word) {
+    if (this.config.infinitiveEndings) {
+      return this.config.infinitiveEndings.some((e) => word.endsWith(e));
+    }
+    return false;
+  }
+  /**
+   * Try to strip reflexive suffixes and normalize the remainder.
+   * Common in Romance languages (Spanish, Portuguese, French).
+   */
+  tryReflexiveNormalization(word) {
+    const suffixes = this.config.reflexiveSuffixes;
+    if (!suffixes) return null;
+    for (const suffix of suffixes) {
+      if (!word.endsWith(suffix)) continue;
+      const remainder = word.slice(0, -suffix.length);
+      if (this.isAlreadyNormalized(remainder)) {
+        return normalized(remainder, 0.88, {
+          removedSuffixes: [suffix],
+          conjugationType: "reflexive"
+        });
+      }
+      const inner = this.tryConjugationEndings(remainder) || this.trySuffixRules(remainder);
+      if (inner && inner.stem !== remainder) {
+        return normalized(inner.stem, inner.confidence * 0.95, {
+          removedSuffixes: [suffix, ...inner.metadata?.removedSuffixes || []],
+          conjugationType: "reflexive"
+        });
+      }
+    }
+    return null;
+  }
+  /**
+   * Try conjugation endings (verb class endings like -ar/-er/-ir patterns).
+   * Endings must be pre-sorted longest-first.
+   */
+  tryConjugationEndings(word) {
+    const endings = this.config.endings;
+    if (!endings) return null;
+    const minStem = this.config.minStemLength ?? 2;
+    for (const rule of endings) {
+      if (!word.endsWith(rule.ending)) continue;
+      const stemBase = word.slice(0, -rule.ending.length);
+      if (stemBase.length < minStem) continue;
+      const infinitive = stemBase + rule.stem;
+      return normalized(infinitive, rule.confidence, {
+        removedSuffixes: [rule.ending],
+        conjugationType: rule.type
+      });
+    }
+    return null;
+  }
+  /**
+   * Try SuffixRule-style normalization.
+   * Rules must be pre-sorted longest-first.
+   */
+  trySuffixRules(word) {
+    const rules = this.config.suffixRules;
+    if (!rules) return null;
+    const defaultMinStem = this.config.minStemLength ?? 2;
+    for (const rule of rules) {
+      if (!word.endsWith(rule.pattern)) continue;
+      const stem = word.slice(0, -rule.pattern.length);
+      const minStem = rule.minStemLength ?? defaultMinStem;
+      if (stem.length < minStem) continue;
+      const result = stem + (rule.replacement || "");
+      return normalized(result, rule.confidence, {
+        removedSuffixes: [rule.pattern],
+        ...rule.conjugationType && { conjugationType: rule.conjugationType }
+      });
+    }
+    return null;
+  }
+  /**
+   * Try PrefixRule-style normalization.
+   */
+  tryPrefixRules(word) {
+    const rules = this.config.prefixRules;
+    if (!rules) return null;
+    for (const rule of rules) {
+      if (!word.startsWith(rule.pattern)) continue;
+      const remainder = word.slice(rule.pattern.length);
+      const minRemaining = rule.minRemaining ?? this.config.minStemLength ?? 2;
+      if (remainder.length < minRemaining) continue;
+      return normalized(remainder, 1 - rule.confidencePenalty, {
+        removedPrefixes: [rule.pattern]
+      });
+    }
+    return null;
+  }
+};
 export {
+  BaseMorphologicalNormalizer,
   BaseTokenizer,
   DEFAULT_OPERATORS,
   DEFAULT_PUNCTUATION,