@lokascript/semantic 1.2.1 → 2.0.0

package/dist/tokenizers/morphology/portuguese-normalizer.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Portuguese Morphological Normalizer
+ *
+ * Reduces Portuguese verb conjugations to their infinitive forms.
+ * Portuguese has three verb conjugation classes (-ar, -er, -ir) and
+ * supports reflexive verbs (verbs with -se suffix).
+ *
+ * Key features:
+ * - Reflexive verb handling: mostrar-se → mostrar, esconder-se → esconder
+ * - Regular conjugation patterns for -ar, -er, -ir verbs
+ * - Handles common irregular verbs
+ * - Brazilian Portuguese variants
+ *
+ * Examples:
+ *   mostrar-se → mostrar (reflexive infinitive)
+ *   alternando → alternar (gerund)
+ *   escondido → esconder (past participle)
+ *   mostra → mostrar (3rd person present)
+ *   clicou → clicar (3rd person preterite)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Portuguese morphological normalizer.
+ */
+export declare class PortugueseMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "pt";
+    /**
+     * Check if a word might be a Portuguese verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Portuguese word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a reflexive verb.
+     * Portuguese reflexive verbs use hyphenated pronouns: mostrar-se, esconder-me
+     *
+     * Examples:
+     *   mostrar-se → mostrar
+     *   esconder-se → esconder
+     *   exibir-se → exibir
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Try to normalize a conjugated verb to its infinitive.
+     */
+    private tryConjugationNormalization;
+}
+export declare const portugueseMorphologicalNormalizer: PortugueseMorphologicalNormalizer;
+//# sourceMappingURL=portuguese-normalizer.d.ts.map

package/dist/tokenizers/morphology/portuguese-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"portuguese-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/portuguese-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAmB,MAAM,SAAS,CAAC;AAmL7F;;GAEG;AACH,qBAAa,iCAAkC,YAAW,uBAAuB;IAC/E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAwB5C;;;;;;;;OAQG;IACH,OAAO,CAAC,yBAAyB;IAiCjC;;OAEG;IACH,OAAO,CAAC,2BAA2B;CAoBpC;AAGD,eAAO,MAAM,iCAAiC,mCAA0C,CAAC"}

package/dist/tokenizers/morphology/russian-normalizer.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Russian Morphological Normalizer
+ *
+ * Reduces Russian verb conjugations to their infinitive forms.
+ * Russian is a fusional language with rich verb morphology:
+ * - Two aspects: perfective (single completed action) and imperfective (ongoing/repeated)
+ * - Three tenses: present, past, future
+ * - Person/number agreement in present tense (6 forms)
+ * - Gender/number agreement in past tense (4 forms)
+ * - Imperative mood (2 forms: singular and plural)
+ * - Reflexive verbs (suffix -ся/-сь)
+ *
+ * For software UI, Russian uses INFINITIVE form (industry standard):
+ * - переключить (toggle), добавить (add), удалить (remove)
+ *
+ * Examples:
+ *   переключи → переключить (imperative → infinitive)
+ *   добавил → добавить (past → infinitive)
+ *   удаляет → удалять (present → infinitive, imperfective)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Russian morphological normalizer.
+ */
+export declare class RussianMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "ru";
+    /**
+     * Check if a word might be a Russian verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Russian word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize reflexive verbs (-ся/-сь).
+     * Strip the reflexive suffix, normalize the base, then append -ться.
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Normalize a non-reflexive verb form.
+     */
+    private normalizeNonReflexive;
+    /**
+     * Try imperative → infinitive via lookup table only.
+     */
+    private tryImperativeLookup;
+    /**
+     * Try generic imperative patterns (lower confidence fallback).
+     */
+    private tryGenericImperativeNormalization;
+    /**
+     * Try past tense → infinitive.
+     *
+     * Russian past tense endings:
+     * - masculine: -л (делал → делать)
+     * - feminine: -ла (делала → делать)
+     * - neuter: -ло (делало → делать)
+     * - plural: -ли (делали → делать)
+     */
+    private tryPastTenseNormalization;
+    /**
+     * Try present tense → infinitive.
+     *
+     * Russian present tense endings (imperfective verbs):
+     * 1st conj: -ю/-у, -ешь, -ет, -ем, -ете, -ют/-ут
+     * 2nd conj: -ю/-у, -ишь, -ит, -им, -ите, -ят/-ат
+     */
+    private tryPresentTenseNormalization;
+}
+export declare const russianMorphologicalNormalizer: RussianMorphologicalNormalizer;
+//# sourceMappingURL=russian-normalizer.d.ts.map

package/dist/tokenizers/morphology/russian-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"russian-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/russian-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAkE5E;;GAEG;AACH,qBAAa,8BAA+B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAkC5C;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAyBjC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAyB7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;OAEG;IACH,OAAO,CAAC,iCAAiC;IAqCzC;;;;;;;;OAQG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;;;;;OAMG;IACH,OAAO,CAAC,4BAA4B;CAsErC;AAGD,eAAO,MAAM,8BAA8B,gCAAuC,CAAC"}

package/dist/tokenizers/morphology/spanish-normalizer.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Spanish Morphological Normalizer
+ *
+ * Reduces Spanish verb conjugations to their infinitive forms.
+ * Spanish has three verb conjugation classes (-ar, -er, -ir) and
+ * supports reflexive verbs (verbs with -se suffix).
+ *
+ * Key features:
+ * - Reflexive verb handling: mostrarse → mostrar, ocultarse → ocultar
+ * - Regular conjugation patterns for -ar, -er, -ir verbs
+ * - Handles common irregular verbs
+ *
+ * Examples:
+ *   mostrarse → mostrar (reflexive infinitive)
+ *   alternando → alternar (gerund)
+ *   escondido → esconder (past participle)
+ *   muestra → mostrar (3rd person present)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Spanish morphological normalizer.
+ */
+export declare class SpanishMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "es";
+    /**
+     * Check if a word might be a Spanish verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Spanish word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize a reflexive verb.
+     * Reflexive verbs end with -se, -me, -te, -nos, -os attached to infinitive.
+     *
+     * Examples:
+     *   mostrarse → mostrar
+     *   ocultarse → ocultar
+     *   esconderse → esconder
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Try to normalize a conjugated verb to its infinitive.
+     */
+    private tryConjugationNormalization;
+}
+export declare const spanishMorphologicalNormalizer: SpanishMorphologicalNormalizer;
+//# sourceMappingURL=spanish-normalizer.d.ts.map

package/dist/tokenizers/morphology/spanish-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spanish-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/spanish-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAmB,MAAM,SAAS,CAAC;AAkM7F;;GAEG;AACH,qBAAa,8BAA+B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IA4B5C;;;;;;;;OAQG;IACH,OAAO,CAAC,yBAAyB;IAiCjC;;OAEG;IACH,OAAO,CAAC,2BAA2B;CAoBpC;AAGD,eAAO,MAAM,8BAA8B,gCAAuC,CAAC"}

package/dist/tokenizers/morphology/turkish-normalizer.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Turkish Morphological Normalizer
+ *
+ * Turkish is a highly agglutinative language with strict vowel harmony.
+ * Suffixes attach in sequence and their vowels change based on the last
+ * vowel of the stem (front/back, rounded/unrounded).
+ *
+ * Vowel Harmony Rules:
+ * - Back vowels (a, ı, o, u) take back vowel suffixes
+ * - Front vowels (e, i, ö, ü) take front vowel suffixes
+ *
+ * Common verb suffixes:
+ * - Infinitive: -mak/-mek (değiştirmek = to change)
+ * - Present continuous: -iyor/-ıyor/-üyor/-uyor (değiştiriyor = is changing)
+ * - Past: -di/-dı/-dü/-du (değiştirdi = changed)
+ * - Reported past: -miş/-mış/-müş/-muş (değiştirmiş = apparently changed)
+ * - Future: -ecek/-acak (değiştirecek = will change)
+ * - Negation: -me/-ma before tense (değiştirmiyor = is not changing)
+ * - Passive: -il/-ıl/-ül/-ul (değiştirildi = was changed)
+ * - Causative: -tir/-tır/-tür/-tur (değiştirtmek = to make change)
+ *
+ * Person suffixes (after tense):
+ * - 1sg: -im/-ım/-üm/-um or -m (yapıyorum = I am doing)
+ * - 2sg: -sin/-sın/-sün/-sun (yapıyorsun = you are doing)
+ * - 3sg: (no suffix) (yapıyor = he/she is doing)
+ * - 1pl: -iz/-ız/-üz/-uz (yapıyoruz = we are doing)
+ * - 2pl: -siniz/-sınız/-sünüz/-sunuz (yapıyorsunuz = you all are doing)
+ * - 3pl: -ler/-lar (yapıyorlar = they are doing)
+ *
+ * Examples:
+ *   değiştiriyorum → değiştir (I am changing)
+ *   değiştirmek → değiştir (to change)
+ *   gösterdi → göster (showed)
+ *   gizleniyor → gizle (is being hidden)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Turkish morphological normalizer.
+ */
+export declare class TurkishMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "tr";
+    /**
+     * Check if a word might be a Turkish verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Turkish word to its stem form.
+     */
+    normalize(word: string): NormalizationResult;
+}
+export declare const turkishMorphologicalNormalizer: TurkishMorphologicalNormalizer;
+//# sourceMappingURL=turkish-normalizer.d.ts.map

package/dist/tokenizers/morphology/turkish-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"turkish-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/turkish-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EACV,uBAAuB,EACvB,mBAAmB,EAGpB,MAAM,SAAS,CAAC;AAgTjB;;GAEG;AACH,qBAAa,8BAA+B,YAAW,uBAAuB;IAC5E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAUrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;CAyC7C;AAGD,eAAO,MAAM,8BAA8B,gCAAuC,CAAC"}

package/dist/tokenizers/morphology/types.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Morphological Normalizer Types
+ *
+ * Defines interfaces for language-specific morphological analysis.
+ * Normalizers reduce conjugated/inflected forms to canonical stems
+ * that can be matched against keyword dictionaries.
+ */
+/**
+ * Result of morphological normalization.
+ */
+export interface NormalizationResult {
+    /** The extracted stem/root form */
+    readonly stem: string;
+    /** Confidence in the normalization (0.0-1.0) */
+    readonly confidence: number;
+    /** Optional metadata about the transformation */
+    readonly metadata?: NormalizationMetadata;
+}
+/**
+ * Metadata about morphological transformations applied.
+ */
+export interface NormalizationMetadata {
+    /** Prefixes that were removed */
+    readonly removedPrefixes?: readonly string[];
+    /** Suffixes that were removed */
+    readonly removedSuffixes?: readonly string[];
+    /** Type of conjugation detected */
+    readonly conjugationType?: ConjugationType;
+    /** Original form classification */
+    readonly originalForm?: string;
+    /** Applied transformation rules (for debugging) */
+    readonly appliedRules?: readonly string[];
+}
+/**
+ * Types of verb conjugation/inflection.
+ */
+export type ConjugationType = 'present' | 'past' | 'future' | 'progressive' | 'perfect' | 'imperative' | 'subjunctive' | 'conditional' | 'passive' | 'causative' | 'polite' | 'humble' | 'honorific' | 'negative' | 'potential' | 'volitional' | 'conditional-tara' | 'conditional-to' | 'conditional-ba' | 'connective' | 'conditional-myeon' | 'temporal-ttae' | 'causal-nikka' | 'honorific-conditional' | 'honorific-temporal' | 'honorific-causal' | 'honorific-past' | 'honorific-polite' | 'sequential-after' | 'sequential-before' | 'immediate' | 'obligation' | 'reflexive' | 'reflexive-imperative' | 'gerund' | 'participle' | 'conditional-idha' | 'temporal-indama' | 'temporal-hina' | 'temporal-lamma' | 'past-verb' | 'conditional-se' | 'temporal-ince' | 'temporal-dikce' | 'aorist' | 'optative' | 'necessitative' | 'request' | 'casual-request' | 'contracted' | 'contracted-past' | 'compound' | 'te-form' | 'dictionary';
+/**
+ * Interface for language-specific morphological normalizers.
+ *
+ * Normalizers attempt to reduce inflected word forms to their
+ * canonical stems. This enables matching conjugated verbs against
+ * keyword dictionaries that only contain base forms.
+ *
+ * Example (Japanese):
+ *   切り替えた (past) → { stem: '切り替え', confidence: 0.85 }
+ *   切り替えます (polite) → { stem: '切り替え', confidence: 0.85 }
+ *
+ * Example (Spanish):
+ *   mostrarse (reflexive infinitive) → { stem: 'mostrar', confidence: 0.85 }
+ *   alternando (gerund) → { stem: 'alternar', confidence: 0.85 }
+ */
+export interface MorphologicalNormalizer {
+    /** Language code this normalizer handles */
+    readonly language: string;
+    /**
+     * Normalize a word to its canonical stem form.
+     *
+     * @param word - The word to normalize
+     * @returns Normalization result with stem and confidence
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Check if a word appears to be a verb form that can be normalized.
+     * Optional optimization to skip normalization for non-verb tokens.
+     *
+     * @param word - The word to check
+     * @returns true if the word might be a normalizable verb form
+     */
+    isNormalizable?(word: string): boolean;
+}
+/**
+ * Configuration for suffix-based normalization rules.
+ * Used by agglutinative languages (Japanese, Korean, Turkish).
+ */
+export interface SuffixRule {
+    /** The suffix pattern to match */
+    readonly pattern: string;
+    /** Confidence when this suffix is stripped */
+    readonly confidence: number;
+    /** What to replace the suffix with (empty string for simple removal) */
+    readonly replacement?: string;
+    /** Conjugation type this suffix indicates */
+    readonly conjugationType?: ConjugationType;
+    /** Minimum stem length after stripping (to avoid over-stripping) */
+    readonly minStemLength?: number;
+}
+/**
+ * Configuration for prefix-based normalization rules.
+ * Used primarily by Arabic for article/conjunction prefixes.
+ */
+export interface PrefixRule {
+    /** The prefix pattern to match */
+    readonly pattern: string;
+    /** Confidence penalty when this prefix is stripped */
+    readonly confidencePenalty: number;
+    /** What the prefix indicates (for metadata) */
+    readonly prefixType?: 'article' | 'conjunction' | 'preposition' | 'verb-marker';
+    /** Minimum remaining characters after stripping (to avoid over-stripping) */
+    readonly minRemaining?: number;
+}
+/**
+ * Helper to create a "no change" normalization result.
+ */
+export declare function noChange(word: string): NormalizationResult;
+/**
+ * Helper to create a normalization result with metadata.
+ */
+export declare function normalized(stem: string, confidence: number, metadata?: NormalizationMetadata): NormalizationResult;
+//# sourceMappingURL=types.d.ts.map

package/dist/tokenizers/morphology/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,gDAAgD;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,iDAAiD;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,qBAAqB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,iCAAiC;IACjC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C,iCAAiC;IACjC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C,mCAAmC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;IAE3C,mCAAmC;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAE/B,mDAAmD;IACnD,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAEvB,SAAS,GACT,MAAM,GACN,QAAQ,GACR,aAAa,GACb,SAAS,GAET,YAAY,GACZ,aAAa,GACb,aAAa,GAEb,SAAS,GACT,WAAW,GAEX,QAAQ,GACR,QAAQ,GACR,WAAW,GAEX,UAAU,GACV,WAAW,GACX,YAAY,GAEZ,kBAAkB,GAClB,gBAAgB,GAChB,gBAAgB,GAEhB,YAAY,GACZ,mBAAmB,GACnB,eAAe,GACf,cAAc,GAEd,uBAAuB,GACvB,oBAAoB,GACpB,kBAAkB,GAClB,gBAAgB,GAChB,kBAAkB,GAElB,kBAAkB,GAClB,mBAAmB,GACnB,WAAW,GACX,YAAY,GAEZ,WAAW,GACX,sBAAsB,GACtB,QAAQ,GACR,YAAY,GAEZ,kBAAkB,GAClB,iBAAiB,GACjB,eAAe,GACf,gBAAgB,GAChB,WAAW,GAEX,gBAAgB,GAChB,eAAe,GACf,gBAAgB,GAChB,QAAQ,GACR,UAAU,GACV,eAAe,GAEf,SAAS,GACT,gBAAgB,GAChB,YAAY,GACZ,iBAAiB,GAEjB,UAAU,GACV,SAAS,GACT,YAAY,CAAC;AAEjB;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,uBAAuB;IACtC,4CAA4C;IAC5C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAE7C;;;;;;OAMG;IACH,cAAc,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,8CAA8C;IAC9C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B,6CAA6C;IAC7C,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;IAE3C,oEAAoE;IACpE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,sDAAsD;IACtD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,+CAA+C;IAC/C,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,GAAG,aAAa,GAAG,aAAa,GAAG,aAAa,CAAC;IAEhF,6EAA6E;IAC7E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAE1D;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,qBAAqB,GAC/B,mBAAmB,CAKrB"}

package/dist/tokenizers/morphology/ukrainian-normalizer.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Ukrainian Morphological Normalizer
+ *
+ * Reduces Ukrainian verb conjugations to their infinitive forms.
+ * Ukrainian is a fusional language closely related to Russian with rich verb morphology:
+ * - Two aspects: perfective and imperfective
+ * - Three tenses: present, past, future
+ * - Person/number agreement in present tense (6 forms)
+ * - Gender/number agreement in past tense (4 forms)
+ * - Imperative mood (2 forms: singular and plural)
+ * - Reflexive verbs (suffix -ся/-сь)
+ *
+ * Key differences from Russian:
+ * - Infinitive ends in -ти (not -ть): робити, ходити
+ * - Past masculine ends in -в (not -л): робив, ходив
+ * - Unique vowels: і, ї, є, ґ
+ *
+ * For software UI, Ukrainian uses INFINITIVE form (industry standard):
+ * - перемкнути (toggle), додати (add), видалити (remove)
+ *
+ * Examples:
+ *   перемкни → перемкнути (imperative → infinitive)
+ *   додав → додати (past → infinitive)
+ *   видаляє → видаляти (present → infinitive)
+ */
+import type { MorphologicalNormalizer, NormalizationResult } from './types';
+/**
+ * Ukrainian morphological normalizer.
+ */
+export declare class UkrainianMorphologicalNormalizer implements MorphologicalNormalizer {
+    readonly language = "uk";
+    /**
+     * Check if a word might be a Ukrainian verb that can be normalized.
+     */
+    isNormalizable(word: string): boolean;
+    /**
+     * Normalize a Ukrainian word to its infinitive form.
+     */
+    normalize(word: string): NormalizationResult;
+    /**
+     * Try to normalize reflexive verbs (-ся/-сь).
+     */
+    private tryReflexiveNormalization;
+    /**
+     * Normalize a non-reflexive verb form.
+     */
+    private normalizeNonReflexive;
+    /**
+     * Try imperative → infinitive via lookup table only.
+     */
+    private tryImperativeLookup;
+    /**
+     * Try generic imperative patterns (lower confidence fallback).
+     */
+    private tryGenericImperativeNormalization;
+    /**
+     * Try past tense → infinitive.
+     *
+     * Ukrainian past tense endings (differs from Russian):
+     * - masculine: -в (робив → робити)
+     * - feminine: -ла (робила → робити)
+     * - neuter: -ло (робило → робити)
+     * - plural: -ли (робили → робити)
+     */
+    private tryPastTenseNormalization;
+    /**
+     * Try present tense → infinitive.
+     *
+     * Ukrainian present tense endings:
+     * 1st conj: -ю/-у, -еш/-єш, -е/-є, -емо/-ємо, -ете/-єте, -ють/-уть
+     * 2nd conj: -ю/-у, -иш, -ить, -имо, -ите, -ять/-ать
+     */
+    private tryPresentTenseNormalization;
+}
+export declare const ukrainianMorphologicalNormalizer: UkrainianMorphologicalNormalizer;
+//# sourceMappingURL=ukrainian-normalizer.d.ts.map

package/dist/tokenizers/morphology/ukrainian-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ukrainian-normalizer.d.ts","sourceRoot":"","sources":["../../../src/tokenizers/morphology/ukrainian-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAwD5E;;GAEG;AACH,qBAAa,gCAAiC,YAAW,uBAAuB;IAC9E,QAAQ,CAAC,QAAQ,QAAQ;IAEzB;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB;IAkC5C;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAuBjC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;OAEG;IACH,OAAO,CAAC,iCAAiC;IAqCzC;;;;;;;;OAQG;IACH,OAAO,CAAC,yBAAyB;IA4CjC;;;;;;OAMG;IACH,OAAO,CAAC,4BAA4B;CAmErC;AAGD,eAAO,MAAM,gCAAgC,kCAAyC,CAAC"}

package/dist/tokenizers/ms.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Malay Tokenizer
+ *
+ * Tokenizes Malay hyperscript input.
+ * Word order: SVO
+ * Direction: ltr
+ * Uses spaces: true
+ *
+ * This tokenizer derives keywords from the Malay profile (single source of truth)
+ * with extras for literals, positional words, and event names.
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class MalayTokenizer extends BaseTokenizer {
+    readonly language = "ms";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+}
+export declare const malayTokenizer: MalayTokenizer;
+//# sourceMappingURL=ms.d.ts.map

package/dist/tokenizers/ms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ms.d.ts","sourceRoot":"","sources":["../../src/tokenizers/ms.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEtE,OAAO,EACL,aAAa,EAUd,MAAM,QAAQ,CAAC;AA4ChB,qBAAa,cAAe,SAAQ,aAAa;IAC/C,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAoHpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;CAgBxC;AAED,eAAO,MAAM,cAAc,gBAAuB,CAAC"}

package/dist/tokenizers/polish.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Polish Tokenizer
+ *
+ * Tokenizes Polish hyperscript input.
+ * Polish characteristics:
+ * - SVO word order (relatively free, but SVO is common)
+ * - Space-separated words
+ * - Prepositions
+ * - Polish diacritics: ą, ę, ć, ń, ó, ś, ź, ż, ł
+ * - Fusional morphology with verb conjugations
+ * - Imperative form used in software UI (unlike most languages that use infinitive)
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class PolishTokenizer extends BaseTokenizer {
+    readonly language = "pl";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    private extractNumber;
+}
+export declare const polishTokenizer: PolishTokenizer;
+//# sourceMappingURL=polish.d.ts.map

package/dist/tokenizers/polish.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"polish.d.ts","sourceRoot":"","sources":["../../src/tokenizers/polish.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAWd,MAAM,QAAQ,CAAC;AA2IhB,qBAAa,eAAgB,SAAQ,aAAa;IAChD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IASpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IA0BnB,OAAO,CAAC,aAAa;CAsDtB;AAED,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/tokenizers/portuguese.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Portuguese Tokenizer
+ *
+ * Tokenizes Portuguese hyperscript input.
+ * Portuguese is similar to Spanish:
+ * - SVO word order
+ * - Space-separated words
+ * - Prepositions
+ * - Accent marks
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class PortugueseTokenizer extends BaseTokenizer {
+    readonly language = "pt";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    /**
+     * Extract a number, including Portuguese time unit suffixes.
+     */
+    private extractNumber;
+}
+export declare const portugueseTokenizer: PortugueseTokenizer;
+//# sourceMappingURL=portuguese.d.ts.map

package/dist/tokenizers/portuguese.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"portuguese.d.ts","sourceRoot":"","sources":["../../src/tokenizers/portuguese.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AAyIhB,qBAAa,mBAAoB,SAAQ,aAAa;IACpD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAQpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IAyBnB;;OAEG;IACH,OAAO,CAAC,aAAa;CAMtB;AAED,eAAO,MAAM,mBAAmB,qBAA4B,CAAC"}

package/dist/tokenizers/quechua.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Quechua Tokenizer
+ *
+ * Tokenizes Quechua (Runasimi) hyperscript input.
+ * Quechua characteristics:
+ * - SOV word order
+ * - Agglutinative/polysynthetic morphology
+ * - Postpositions (suffixes)
+ * - Case suffixes: -ta (accusative), -man (allative), -manta (ablative), etc.
+ * - Evidential markers
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class QuechuaTokenizer extends BaseTokenizer {
+    readonly language = "qu";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private trySuffix;
+    /**
+     * Split a selector token if it has a Quechua suffix attached.
+     * E.g., ".openta" -> [".open", "-ta"]
+     * Returns array with 1 token (no suffix) or 2 tokens (selector + suffix)
+     */
+    private splitSelectorSuffix;
+    /**
+     * Try to match multi-word keywords that should be treated as a single unit.
+     * E.g., "mana qhawachiy" (not focus = blur)
+     */
+    private tryMultiWordKeyword;
+    private extractWord;
+    private extractNumber;
+}
+export declare const quechuaTokenizer: QuechuaTokenizer;
+//# sourceMappingURL=quechua.d.ts.map

package/dist/tokenizers/quechua.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quechua.d.ts","sourceRoot":"","sources":["../../src/tokenizers/quechua.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAWd,MAAM,QAAQ,CAAC;AAmIhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAOpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAwHpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,SAAS;IAcjB;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAgC3B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,WAAW;IA0BnB,OAAO,CAAC,aAAa;CAyCtB;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/russian.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Russian Tokenizer
+ *
+ * Tokenizes Russian hyperscript input.
+ * Russian characteristics:
+ * - SVO word order (relatively free, but SVO is common)
+ * - Space-separated words
+ * - Prepositions
+ * - Cyrillic alphabet
+ * - Fusional morphology with verb conjugations
+ * - Imperative form used in software UI (infinitive also common)
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class RussianTokenizer extends BaseTokenizer {
+    readonly language = "ru";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    private extractWord;
+    private extractNumber;
+}
+export declare const russianTokenizer: RussianTokenizer;
+//# sourceMappingURL=russian.d.ts.map

package/dist/tokenizers/russian.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"russian.d.ts","sourceRoot":"","sources":["../../src/tokenizers/russian.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAWd,MAAM,QAAQ,CAAC;AAmIhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IASpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA4FpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAkBvC,OAAO,CAAC,WAAW;IA0BnB,OAAO,CAAC,aAAa;CAmFtB;AAED,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}

package/dist/tokenizers/spanish.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Spanish Tokenizer
+ *
+ * Tokenizes Spanish hyperscript input.
+ * Spanish is relatively straightforward as it:
+ * - Uses space-separated words like English
+ * - Has similar preposition structure (SVO)
+ * - Uses accent marks that need proper handling
+ */
+import type { TokenKind, TokenStream } from '../types';
+import { BaseTokenizer } from './base';
+export declare class SpanishTokenizer extends BaseTokenizer {
+    readonly language = "es";
+    readonly direction: "ltr";
+    constructor();
+    tokenize(input: string): TokenStream;
+    classifyToken(token: string): TokenKind;
+    /**
+     * Try to match multi-word phrases that function as single units.
+     * Multi-word phrases are included in profileKeywords and sorted longest-first,
+     * so they'll be matched before their constituent words.
+     */
+    private tryMultiWordPhrase;
+    /**
+     * Extract a Spanish word.
+     *
+     * Uses morphological normalization to handle:
+     * - Reflexive verbs (mostrarse → mostrar)
+     * - Verb conjugations (alternando → alternar)
+     */
+    private extractSpanishWord;
+    /**
+     * Extract a number, including Spanish time unit suffixes.
+     */
+    private extractSpanishNumber;
+}
+/**
+ * Singleton instance.
+ */
+export declare const spanishTokenizer: SpanishTokenizer;
+//# sourceMappingURL=spanish.d.ts.map

package/dist/tokenizers/spanish.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spanish.d.ts","sourceRoot":"","sources":["../../src/tokenizers/spanish.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAiB,SAAS,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACtE,OAAO,EACL,aAAa,EAYd,MAAM,QAAQ,CAAC;AAmKhB,qBAAa,gBAAiB,SAAQ,aAAa;IACjD,QAAQ,CAAC,QAAQ,QAAQ;IACzB,QAAQ,CAAC,SAAS,EAAG,KAAK,CAAU;;IAUpC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA6GpC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAqBvC;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IA6B1B;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAM7B;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,kBAAyB,CAAC"}