@lokascript/semantic 1.2.1 → 2.0.0

package/dist/tokenizers/hindi.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hindi.d.ts","sourceRoot":"","sources":["../../src/tokenizers/hindi.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AAoEhB,qBAAa,cAAe,SAAQ,aAAa;IAC/C,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAwKpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAoBvC;;OAEG;IACH,OAAO,CAAC,aAAa;CAgCtB;AAMD,eAAO,MAAM,cAAc,gBAAuB,CAAC"}

package/dist/tokenizers/index.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Tokenizer Registry
+ *
+ * Provides a unified interface for tokenization.
+ * Delegates to the central registry for language lookups.
+ *
+ * For tree-shaking, import specific tokenizers directly:
+ *   import { englishTokenizer } from './tokenizers/english';
+ *
+ * To register languages, import the language modules:
+ *   import '@lokascript/semantic/languages/en';
+ */
+import type { LanguageTokenizer, TokenStream } from '../types';
+/**
+ * Get a tokenizer for the specified language.
+ * Returns undefined if language is not registered.
+ */
+export declare function getTokenizer(language: string): LanguageTokenizer | undefined;
+/**
+ * Tokenize input in the specified language.
+ * @throws Error if language is not registered
+ */
+export declare function tokenize(input: string, language: string): TokenStream;
+/**
+ * Get all supported languages.
+ * Returns only languages that have been registered.
+ */
+export declare function getSupportedLanguages(): string[];
+/**
+ * Check if a language is supported.
+ */
+export declare function isLanguageSupported(language: string): boolean;
+/**
+ * Register a custom tokenizer.
+ * Note: For full language support, use registerLanguage() from registry instead.
+ */
+export declare function registerTokenizer(tokenizer: LanguageTokenizer): void;
+export { englishTokenizer } from './english';
+export { japaneseTokenizer } from './japanese';
+export { koreanTokenizer } from './korean';
+export { arabicTokenizer } from './arabic';
+export { spanishTokenizer } from './spanish';
+export { turkishTokenizer } from './turkish';
+export { chineseTokenizer } from './chinese';
+export { portugueseTokenizer } from './portuguese';
+export { frenchTokenizer } from './french';
+export { germanTokenizer } from './german';
+export { indonesianTokenizer } from './indonesian';
+export { quechuaTokenizer } from './quechua';
+export { swahiliTokenizer } from './swahili';
+export { italianTokenizer } from './italian';
+export { vietnameseTokenizer } from './vietnamese';
+export { polishTokenizer } from './polish';
+export { russianTokenizer } from './russian';
+export { ukrainianTokenizer } from './ukrainian';
+export { hebrewTokenizer } from './he';
+export { hindiTokenizer } from './hindi';
+export { bengaliTokenizer } from './bengali';
+export { thaiTokenizer } from './thai';
+export { malayTokenizer } from './ms';
+export { tagalogTokenizer } from './tl';
+export { BaseTokenizer, TokenStreamImpl, createToken, createPosition, extractCssSelector, extractStringLiteral, extractNumber, isUrlStart, extractUrl, isWhitespace, isSelectorStart, isQuote, isDigit, isAsciiLetter, isAsciiIdentifierChar, createUnicodeRangeClassifier, combineClassifiers, createLatinCharClassifiers, } from './base';
+export type { KeywordEntry, TokenizerProfile, TimeUnitMapping, UnicodeRange, LatinCharClassifiers, CreateTokenOptions, } from './base';
+//# sourceMappingURL=index.d.ts.map

package/dist/tokenizers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tokenizers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAa/D;;;GAGG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAE5E;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,CAErE;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,EAAE,CAEhD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE7D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,GAAG,IAAI,CAEpE;AAMD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,MAAM,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC,OAAO,EAAE,cAAc,EAAE,MAAM,MAAM,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AAExC,OAAO,EACL,aAAa,EACb,eAAe,EACf,WAAW,EACX,cAAc,EACd,kBAAkB,EAClB,oBAAoB,EACpB,aAAa,EACb,UAAU,EACV,UAAU,EAEV,YAAY,EACZ,eAAe,EACf,OAAO,EACP,OAAO,EACP,aAAa,EACb,qBAAqB,EAErB,4BAA4B,EAC5B,kBAAkB,EAClB,0BAA0B,GAC3B,MAAM,QAAQ,CAAC;AAGhB,YAAY,EACV,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,GACnB,MAAM,QAAQ,CAAC"}

package/dist/tokenizers/indonesian.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Indonesian Tokenizer
+ *
+ * Tokenizes Indonesian hyperscript input.
+ * Indonesian characteristics:
+ * - SVO word order
+ * - Space-separated words
+ * - Prepositions
+ * - Agglutinative prefixes/suffixes (me-, ber-, di-, -kan, -i)
+ * - No grammatical gender or conjugation
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class IndonesianTokenizer extends BaseTokenizer {
+    readonly language = "id";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    private extractNumber;
+}
+export declare const indonesianTokenizer: IndonesianTokenizer;
+//# sourceMappingURL=indonesian.d.ts.map

package/dist/tokenizers/indonesian.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indonesian.d.ts","sourceRoot":"","sources":["../../src/tokenizers/indonesian.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAWd,MAAM,QAAQ,CAAC;AAwGhB,qBAAa,mBAAoB,SAAQ,aAAa;IACpD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IAyBnB,OAAO,CAAC,aAAa;CA2CtB;AAED,eAAO,MAAM,mBAAmB,qBAA4B,CAAC"}

package/dist/tokenizers/italian.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Italian Tokenizer
+ *
+ * Tokenizes Italian hyperscript input.
+ * Italian is very similar to Spanish:
+ * - Uses space-separated words
+ * - Has similar preposition structure (SVO)
+ * - Uses accent marks that need proper handling
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class ItalianTokenizer extends BaseTokenizer {
+    readonly language = "it";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Try to match multi-word phrases that function as single units.
+     * Multi-word phrases are included in profileKeywords and sorted longest-first,
+     * so they'll be matched before their constituent words.
+     */
+    private tryMultiWordPhrase;
+    /**
+     * Extract an Italian word.
+     *
+     * Uses morphological normalization to handle:
+     * - Reflexive verbs (mostrarsi → mostrare)
+     * - Verb conjugations (alternando → alternare)
+     */
+    private extractItalianWord;
+    /**
+     * Extract a number, including Italian time unit suffixes.
+     */
+    private extractItalianNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const italianTokenizer: ItalianTokenizer;
+//# sourceMappingURL=italian.d.ts.map

package/dist/tokenizers/italian.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"italian.d.ts","sourceRoot":"","sources":["../../src/tokenizers/italian.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AA0NhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAUpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA6GpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAqBvC;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IA6B1B;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAM7B;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/japanese.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Japanese Tokenizer
+ *
+ * Tokenizes Japanese hyperscript input.
+ * Japanese is challenging because:
+ * - No spaces between words
+ * - Particles (助詞) mark grammatical roles
+ * - Mixed scripts (hiragana, katakana, kanji, romaji)
+ * - CSS selectors are embedded ASCII
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class JapaneseTokenizer extends BaseTokenizer {
+    readonly language = "ja";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Extract a Japanese word (sequence of kanji/kana).
+     * Stops at particles, ASCII, or whitespace.
+     *
+     * Uses morphological normalization to handle verb conjugations:
+     * 1. First checks if the exact word is in the keyword map
+     * 2. If not found, tries to strip conjugation suffixes and check again
+     */
+    private extractJapaneseWord;
+    /**
+     * Extract an ASCII word (for mixed Japanese/English content).
+     */
+    private extractAsciiWord;
+    /**
+     * Extract a number, including Japanese time unit suffixes.
+     * Japanese time units attach directly without whitespace.
+     */
+    private extractJapaneseNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const japaneseTokenizer: JapaneseTokenizer;
+//# sourceMappingURL=japanese.d.ts.map

package/dist/tokenizers/japanese.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"japanese.d.ts","sourceRoot":"","sources":["../../src/tokenizers/japanese.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAcd,MAAM,QAAQ,CAAC;AA0KhB,qBAAa,iBAAkB,SAAQ,aAAa;IAClD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAUpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAoJpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IA+C3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;CAM9B;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,mBAA0B,CAAC"}

package/dist/tokenizers/korean.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Korean Tokenizer
+ *
+ * Tokenizes Korean hyperscript input.
+ * Korean is an agglutinative language with:
+ * - Hangul syllable blocks (가-힣)
+ * - Particles (조사) mark grammatical roles
+ * - 하다 verbs (noun + 하다)
+ * - CSS selectors are embedded ASCII
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class KoreanTokenizer extends BaseTokenizer {
+    readonly language = "ko";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Extract a Korean word (sequence of Hangul).
+     * Prioritizes known keywords, then uses particle-based word boundaries.
+     *
+     * Uses morphological normalization to handle verb conjugations.
+     */
+    private extractKoreanWord;
+    /**
+     * Extract an ASCII word (for mixed Korean/English content).
+     */
+    private extractAsciiWord;
+    /**
+     * Extract a number, including Korean time unit suffixes.
+     * Korean time units attach directly without whitespace.
+     */
+    private extractKoreanNumber;
+    /**
+     * Try to split a temporal event suffix from a word token.
+     * This handles compact forms like 클릭할때 → 클릭 + 할때
+     *
+     * @returns Split tokens if a suffix is found, null otherwise
+     */
+    private trySplitTemporalSuffix;
+}
+/**
+ * Singleton instance.
+ */
+export declare const koreanTokenizer: KoreanTokenizer;
+//# sourceMappingURL=korean.d.ts.map

package/dist/tokenizers/korean.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"korean.d.ts","sourceRoot":"","sources":["../../src/tokenizers/korean.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAcd,MAAM,QAAQ,CAAC;AAkThB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAUpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAqJpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAyHzB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAO3B;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;CAsC/B;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/tokenizers/morphology/arabic-normalizer.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Arabic Morphological Normalizer
+ *
+ * Arabic uses a complex root-pattern morphology system where most words
+ * are derived from triliteral (3-consonant) roots. This normalizer focuses
+ * on prefix/suffix stripping rather than full root extraction.
+ *
+ * Key features:
+ * - Definite article prefix: ال (al-)
+ * - Conjunction/preposition prefixes: و (wa-), ف (fa-), ب (bi-), ل (li-), ك (ka-)
+ * - Verb prefixes (present tense markers): ي (ya-), ت (ta-), ن (na-), أ (a-)
+ * - Plural/gender suffixes: ون (ūn), ين (īn), ات (āt), ة (a)
+ * - Pronoun suffixes: ها (hā), هم (hum), etc.
+ * - Diacritics handling: Words with and without diacritics should match
+ *
+ * Examples:
+ *   والتبديل → تبديل → بدّل (and the changing → changing → change!)
+ *   يبدّل → بدّل (he changes → change!)
+ *   المستخدمين → مستخدم (the users → user)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Arabic morphological normalizer.
+ */
+export declare class ArabicMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "ar";
+    /**
+     * Check if a word might be an Arabic word that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize an Arabic word by stripping prefixes and suffixes.
+     */
+    normalize(word: string): NormalizationResult;
+}
+export declare const arabicMorphologicalNormalizer: ArabicMorphologicalNormalizer;
+//# sourceMappingURL=arabic-normalizer.d.ts.map

package/dist/tokenizers/morphology/arabic-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arabic-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/arabic-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAc,MAAM,SAAS,CAAC;AAuGxF;;GAEG;AACH,qBAAa,6BAA8B,YAAW,uBAAuB;IAC3E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAOrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;CA+F7C;AAGD,eAAO,MAAM,6BAA6B,+BAAsC,CAAC"}

package/dist/tokenizers/morphology/french-normalizer.d.ts

@@ -0,0 +1,46 @@
+/**
+ * French Morphological Normalizer
+ *
+ * Reduces French verb conjugations to their infinitive forms.
+ * French has three verb conjugation groups:
+ * - 1st group: -er verbs (parler, montrer, afficher)
+ * - 2nd group: -ir verbs with -iss- forms (finir, choisir)
+ * - 3rd group: irregular -ir, -re, -oir verbs (partir, prendre, voir)
+ *
+ * Key features:
+ * - Reflexive verb handling: se montrer → montrer
+ * - Regular conjugation patterns for all three groups
+ * - Past participle (-é, -i, -u) and present participle (-ant) forms
+ *
+ * Examples:
+ *   affiche → afficher (3rd person present)
+ *   montrant → montrer (present participle)
+ *   caché → cacher (past participle)
+ *   finissons → finir (1st person plural present)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * French morphological normalizer.
+ */
+export declare class FrenchMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "fr";
+    /**
+     * Check if a word might be a French verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a French word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a reflexive verb (imperative forms with attached pronouns).
+     * Examples: montrez-vous → montrer, lève-toi → lever
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Try to normalize a conjugated verb to its infinitive.
+     */
+    private tryConjugationNormalization;
+}
+export declare const frenchMorphologicalNormalizer: FrenchMorphologicalNormalizer;
+//# sourceMappingURL=french-normalizer.d.ts.map

package/dist/tokenizers/morphology/french-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"french-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/french-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAmB,MAAM,SAAS,CAAC;AA2J7F;;GAEG;AACH,qBAAa,6BAA8B,YAAW,uBAAuB;IAC3E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAuB5C;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAqBjC;;OAEG;IACH,OAAO,CAAC,2BAA2B;CAoBpC;AAGD,eAAO,MAAM,6BAA6B,+BAAsC,CAAC"}

package/dist/tokenizers/morphology/german-normalizer.d.ts

@@ -0,0 +1,55 @@
+/**
+ * German Morphological Normalizer
+ *
+ * Reduces German verb conjugations to their infinitive forms.
+ * German verbs have:
+ * - Weak verbs (regular): machen → machte (past)
+ * - Strong verbs (stem changes): fahren → fuhr (past)
+ * - Mixed verbs: kennen → kannte
+ * - Separable prefixes: an-, auf-, aus-, ein-, mit-, vor-, zu-
+ *
+ * Key features:
+ * - Handles common conjugation endings
+ * - Recognizes past participle ge- prefix
+ * - Handles separable prefix verbs
+ *
+ * Examples:
+ *   zeigt → zeigen (3rd person present)
+ *   gemacht → machen (past participle)
+ *   anzeigen → anzeigen (separable prefix verb)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * German morphological normalizer.
+ */
+export declare class GermanMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "de";
+    /**
+     * Check if a word might be a German verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a German word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a past participle.
+     * German past participles often have ge- prefix and -t or -en suffix.
+     *
+     * Examples:
+     *   gemacht → machen (weak verb)
+     *   gegangen → gehen (strong verb)
+     *   angemacht → anmachen (separable prefix)
+     */
+    private tryParticipleNormalization;
+    /**
+     * Try to normalize a simple ge-...-t or ge-...-en participle.
+     */
+    private trySimpleParticipleNormalization;
+    /**
+     * Try to normalize a conjugated verb to its infinitive.
+     */
+    private tryConjugationNormalization;
+}
+export declare const germanMorphologicalNormalizer: GermanMorphologicalNormalizer;
+//# sourceMappingURL=german-normalizer.d.ts.map

package/dist/tokenizers/morphology/german-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"german-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/german-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAmB,MAAM,SAAS,CAAC;AAuG7F;;GAEG;AACH,qBAAa,6BAA8B,YAAW,uBAAuB;IAC3E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAuB5C;;;;;;;;OAQG;IACH,OAAO,CAAC,0BAA0B;IA2BlC;;OAEG;IACH,OAAO,CAAC,gCAAgC;IA0BxC;;OAEG;IACH,OAAO,CAAC,2BAA2B;CAoBpC;AAGD,eAAO,MAAM,6BAA6B,+BAAsC,CAAC"}

package/dist/tokenizers/morphology/index.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Morphological Normalizers
+ *
+ * Re-exports all morphological normalizer types and implementations.
+ */
+export type { NormalizationResult, NormalizationMetadata, ConjugationType, MorphologicalNormalizer, SuffixRule, PrefixRule, } from './types';
+export { noChange, normalized } from './types';
+export { JapaneseMorphologicalNormalizer, japaneseMorphologicalNormalizer, } from './japanese-normalizer';
+export { KoreanMorphologicalNormalizer, koreanMorphologicalNormalizer } from './korean-normalizer';
+export { SpanishMorphologicalNormalizer, spanishMorphologicalNormalizer, } from './spanish-normalizer';
+export { ArabicMorphologicalNormalizer, arabicMorphologicalNormalizer } from './arabic-normalizer';
+export { TurkishMorphologicalNormalizer, turkishMorphologicalNormalizer, } from './turkish-normalizer';
+export { PortugueseMorphologicalNormalizer, portugueseMorphologicalNormalizer, } from './portuguese-normalizer';
+export { FrenchMorphologicalNormalizer, frenchMorphologicalNormalizer } from './french-normalizer';
+export { GermanMorphologicalNormalizer, germanMorphologicalNormalizer } from './german-normalizer';
+export { ItalianMorphologicalNormalizer, italianMorphologicalNormalizer, } from './italian-normalizer';
+export { PolishMorphologicalNormalizer, polishMorphologicalNormalizer } from './polish-normalizer';
+export { RussianMorphologicalNormalizer, russianMorphologicalNormalizer, } from './russian-normalizer';
+export { UkrainianMorphologicalNormalizer, ukrainianMorphologicalNormalizer, } from './ukrainian-normalizer';
+//# sourceMappingURL=index.d.ts.map

package/dist/tokenizers/morphology/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EACV,mBAAmB,EACnB,qBAAqB,EACrB,eAAe,EACf,uBAAuB,EACvB,UAAU,EACV,UAAU,GACX,MAAM,SAAS,CAAC;AAEjB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG/C,OAAO,EACL,+BAA+B,EAC/B,+BAA+B,GAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEnG,OAAO,EACL,8BAA8B,EAC9B,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEnG,OAAO,EACL,8BAA8B,EAC9B,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,iCAAiC,EACjC,iCAAiC,GAClC,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEnG,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEnG,OAAO,EACL,8BAA8B,EAC9B,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,6BAA6B,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAEnG,OAAO,EACL,8BAA8B,EAC9B,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,gCAAgC,EAChC,gCAAgC,GACjC,MAAM,wBAAwB,CAAC"}

package/dist/tokenizers/morphology/italian-normalizer.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Italian Morphological Normalizer
+ *
+ * Reduces Italian verb conjugations to their infinitive forms.
+ * Italian has three verb conjugation classes (-are, -ere, -ire) and
+ * supports reflexive verbs (verbs with -si suffix).
+ *
+ * Key features:
+ * - Reflexive verb handling: mostrarsi → mostrare, nascondersi → nascondere
+ * - Regular conjugation patterns for -are, -ere, -ire verbs
+ * - Handles common irregular verbs
+ *
+ * Examples:
+ *   mostrarsi → mostrare (reflexive infinitive)
+ *   alternando → alternare (gerund)
+ *   nascosto → nascondere (past participle)
+ *   mostra → mostrare (3rd person present)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Italian morphological normalizer.
+ */
+export declare class ItalianMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "it";
+    /**
+     * Check if a word might be an Italian verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize an Italian word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a reflexive verb.
+     * Reflexive verbs end with -si, -mi, -ti, -ci, -vi attached to infinitive.
+     *
+     * In Italian, reflexive infinitives drop the final -e before attaching the pronoun:
+     *   mostrare + si → mostrarsi (not mostraresi)
+     *   nascondere + si → nascondersi
+     *
+     * Examples:
+     *   mostrarsi → mostrare
+     *   nascondersi → nascondere
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Try to normalize a conjugated verb to its infinitive.
+     */
+    private tryConjugationNormalization;
+}
+export declare const italianMorphologicalNormalizer: ItalianMorphologicalNormalizer;
+//# sourceMappingURL=italian-normalizer.d.ts.map

package/dist/tokenizers/morphology/italian-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"italian-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/italian-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAmB,MAAM,SAAS,CAAC;AAkL7F;;GAEG;AACH,qBAAa,8BAA+B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IA4B5C;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,yBAAyB;IAgDjC;;OAEG;IACH,OAAO,CAAC,2BAA2B;CAoBpC;AAGD,eAAO,MAAM,8BAA8B,gCAAuC,CAAC"}

package/dist/tokenizers/morphology/japanese-normalizer.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Japanese Morphological Normalizer
+ *
+ * Reduces Japanese verb conjugations to their stem forms.
+ * Japanese verbs conjugate by modifying their endings:
+ *
+ * Base: 切り替え (kiri-kae) - "toggle"
+ * て-form: 切り替えて (kiri-kaete) - "toggle and..."
+ * た-form: 切り替えた (kiri-kaeta) - "toggled" (past)
+ * ます-form: 切り替えます (kiri-kaemasu) - polite present
+ * ている: 切り替えている (kiri-kaeteiru) - "is toggling" (progressive)
+ * ない: 切り替えない (kiri-kaenai) - "don't toggle" (negative)
+ *
+ * This normalizer strips these suffixes to find the stem,
+ * which can then be matched against keyword dictionaries.
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Japanese morphological normalizer.
+ */
+export declare class JapaneseMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "ja";
+    /**
+     * Check if a word might be a Japanese verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Japanese word to its stem form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a する verb.
+     */
+    private trySuruNormalization;
+    /**
+     * Normalize compound conjugations (multi-layer suffixes).
+     * These are combinations like ていなかった (was not doing), でいない (is not doing).
+     * Handles cases that single-suffix rules miss.
+     */
+    private normalizeCompound;
+}
+export declare const japaneseMorphologicalNormalizer: JapaneseMorphologicalNormalizer;
+//# sourceMappingURL=japanese-normalizer.d.ts.map

package/dist/tokenizers/morphology/japanese-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"japanese-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/japanese-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,mBAAmB,EAGpB,MAAM,SAAS,CAAC;AAkIjB;;GAEG;AACH,qBAAa,+BAAgC,YAAW,uBAAuB;IAC7E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAYrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAoC5C;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAmB5B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;CA6C1B;AAGD,eAAO,MAAM,+BAA+B,iCAAwC,CAAC"}

package/dist/tokenizers/morphology/korean-normalizer.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Korean Morphological Normalizer
+ *
+ * Reduces Korean verb conjugations to their stem forms.
+ * Korean verbs conjugate by modifying their endings:
+ *
+ * Base: 토글 (togeul) - "toggle" (loanword)
+ * 다 ending: 토글하다 (togeul-hada) - "to toggle" (dictionary form)
+ * 요 ending: 토글해요 (togeul-haeyo) - polite present
+ * 니다 ending: 토글합니다 (togeul-hamnida) - formal present
+ * 세요 ending: 토글하세요 (togeul-haseyo) - honorific request
+ * 았/었 past: 토글했어 (togeul-haesseo) - informal past
+ *
+ * Korean also has vowel harmony affecting suffix forms.
+ *
+ * This normalizer strips these suffixes to find the stem,
+ * which can then be matched against keyword dictionaries.
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Korean morphological normalizer.
+ */
+export declare class KoreanMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "ko";
+    /**
+     * Check if a word might be a Korean verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Korean word to its stem form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a 하다 verb.
+     * 하다 verbs are formed by noun + 하다, very common in Korean.
+     */
+    private tryHadaNormalization;
+    /**
+     * Normalize compound conjugations (multi-layer suffixes).
+     * Korean has complex compound forms that combine multiple grammatical elements.
+     */
+    private normalizeCompound;
+}
+export declare const koreanMorphologicalNormalizer: KoreanMorphologicalNormalizer;
+//# sourceMappingURL=korean-normalizer.d.ts.map

package/dist/tokenizers/morphology/korean-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"korean-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/korean-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,mBAAmB,EAGpB,MAAM,SAAS,CAAC;AA8MjB;;GAEG;AACH,qBAAa,6BAA8B,YAAW,uBAAuB;IAC3E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAUrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAmC5C;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAmB5B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;CA8G1B;AAGD,eAAO,MAAM,6BAA6B,+BAAsC,CAAC"}

package/dist/tokenizers/morphology/polish-normalizer.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Polish Morphological Normalizer
+ *
+ * Normalizes Polish verb forms to their base/infinitive form.
+ *
+ * Polish verb conjugation is complex with:
+ * - Three main conjugation classes (determined by infinitive ending)
+ * - Person/number agreement (6 forms per tense)
+ * - Aspect pairs (perfective/imperfective)
+ *
+ * For software UI, Polish uses IMPERATIVE form (unlike most languages):
+ * - zapisz (save), otwórz (open), usuń (delete)
+ *
+ * This normalizer focuses on recognizing imperative forms and
+ * mapping them back to their base form for keyword matching.
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Polish Morphological Normalizer
+ *
+ * Key patterns:
+ * - Imperative suffixes: -aj, -ij, -uj (2nd person singular)
+ * - Infinitive endings: -ać, -eć, -ić, -yć, -ąć
+ * - Present tense endings: -am, -em, -ę, -asz, -esz, -isz, -ysz
+ */
+export declare class PolishMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "pl";
+    /**
+     * Check if a word appears to be a Polish verb form that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Polish verb to its base/infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Check if word is already in infinitive form.
+     */
+    private isInfinitive;
+    /**
+     * Try to normalize imperative form to infinitive.
+     *
+     * Polish imperative (2nd person singular) patterns:
+     * - pisać → pisz (write)
+     * - czytać → czytaj (read)
+     * - robić → rób (do)
+     * - mówić → mów (speak)
+     * - uczyć → ucz (teach)
+     */
+    private tryImperativeNormalization;
+    /**
+     * Try to normalize present tense form to infinitive.
+     */
+    private tryPresentTenseNormalization;
+    /**
+     * Try to normalize past tense form to infinitive.
+     */
+    private tryPastTenseNormalization;
+}
+export declare const polishMorphologicalNormalizer: PolishMorphologicalNormalizer;
+//# sourceMappingURL=polish-normalizer.d.ts.map

package/dist/tokenizers/morphology/polish-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"polish-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/polish-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAmE5E;;;;;;;GAOG;AACH,qBAAa,6BAA8B,YAAW,uBAAuB;IAC3E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAMrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAwB5C;;OAEG;IACH,OAAO,CAAC,YAAY;IAKpB;;;;;;;;;OASG;IACH,OAAO,CAAC,0BAA0B;IAsClC;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAoCpC;;OAEG;IACH,OAAO,CAAC,yBAAyB;CA+ClC;AAGD,eAAO,MAAM,6BAA6B,+BAAsC,CAAC"}