@lokascript/framework 2.5.0 → 2.6.0

package/dist/core/tokenization/base-tokenizer.d.ts

@@ -45,8 +45,27 @@ export declare abstract class BaseTokenizer implements LanguageTokenizer {
     protected normalizer?: MorphologicalNormalizer;
     /** Keywords derived from profile, sorted longest-first for greedy matching */
     protected profileKeywords: KeywordEntry[];
+    /**
+     * Space-containing profile keywords (multi-word phrases), longest-first.
+     * Used by `tryMultiWordKeyword` so natural spaced forms (hi `मेल खाता`,
+     * vi `chuyển đổi`, es `tecla abajo`, …) tokenize as ONE keyword — the
+     * profile-driven replacement for the per-language hardcoded compound lists.
+     * Empty for no-space (CJK) languages, so they are unaffected.
+     */
+    protected multiWordKeywords: KeywordEntry[];
     /** Map for O(1) keyword lookups by lowercase native word */
     protected profileKeywordMap: Map<string, KeywordEntry>;
+    /**
+     * The raw EXTRAS list passed to initializeKeywordsFromProfile, kept pre-dedup.
+     * The keyword map is keyed by native word with last-wins insertion, so a
+     * duplicate native word inside the extras silently shadows the earlier entry
+     * (e.g. a `nächste→closest` entry shadowing `nächste→next` broke German
+     * positional expressions). Exposed so consistency tests can detect such
+     * intra-extras collisions, which are invisible in the deduplicated map.
+     */
+    private rawExtraEntries;
+    /** Raw extras as passed in, pre-dedup — for consistency tests. */
+    getExtraKeywordEntries(): readonly KeywordEntry[];
     /**
      * Pluggable value extractors for domain-specific syntax.
      * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.
@@ -143,6 +162,20 @@ export declare abstract class BaseTokenizer implements LanguageTokenizer {
      * @returns Token if matched, null otherwise
      */
     protected tryProfileKeyword(input: string, pos: number): LanguageToken | null;
+    /**
+     * Match the longest multi-word (space-containing) profile keyword at `pos`,
+     * requiring the match to end at a word boundary. The profile-driven
+     * counterpart of the per-language hardcoded compound lists (the hindi and
+     * vietnamese keyword extractors). Returns a keyword token (with the normalized
+     * form) or null. Case-sensitive against the stored native form, mirroring
+     * `tryProfileKeyword`/`isKeywordStart` (the i18n dicts emit a fixed surface
+     * case). No-op when `multiWordKeywords` is empty (no-space/CJK languages).
+     *
+     * @param input - Input string
+     * @param pos - Current position (must be a token-start boundary)
+     * @param isWordChar - End-boundary predicate (defaults to Unicode letter/digit/_)
+     */
+    protected tryMultiWordKeyword(input: string, pos: number, isWordChar?: (char: string) => boolean): LanguageToken | null;
     /**
      * Check if the remaining input starts with any known keyword.
      * Useful for non-space languages to detect word boundaries.
@@ -152,6 +185,25 @@ export declare abstract class BaseTokenizer implements LanguageTokenizer {
      * @returns true if a keyword starts at this position
      */
     protected isKeywordStart(input: string, pos: number): boolean;
+    /**
+     * Check if a known keyword starts at the given position AND ends at a word
+     * boundary (end of input or a non-word character).
+     *
+     * Space-delimited languages must use this (not `isKeywordStart`) for
+     * word-walk break checks: the keyword table includes English canonical
+     * fallbacks (me, it, you, …), so a raw `startsWith` check splits any native
+     * word with an embedded fallback mid-word (e.g. Quechua ñit'iy contains
+     * "it"). CJK/no-space tokenizers rely on mid-text keyword starts and must
+     * keep using `isKeywordStart`.
+     *
+     * @param input - Input string
+     * @param pos - Current position
+     * @param isWordChar - Language-specific word-character predicate; pass the
+     *   tokenizer's letter classifier so e.g. the Quechua glottal apostrophe
+     *   counts as part of a word. Defaults to Unicode letters/digits/underscore.
+     * @returns true if a keyword starts here and is not followed by a word char
+     */
+    protected isKeywordStartAtBoundary(input: string, pos: number, isWordChar?: (char: string) => boolean): boolean;
     /**
      * Look up a keyword by native word (case-insensitive).
      * O(1) lookup using the keyword map.

package/dist/core/tokenization/base-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAOL,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AAevB,YAAY,EAAE,YAAY,EAAE,CAAC;AAE7B;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CACxB,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAClE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAC3B,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE;QACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;QACxE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5C,CAAC;CACH;AAMD;;;GAGG;AACH,8BAAsB,aAAc,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAE3C,0DAA0D;IAC1D,SAAS,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAE/C,8EAA8E;IAC9E,SAAS,CAAC,eAAe,EAAE,YAAY,EAAE,CAAM;IAE/C,4DAA4D;IAC5D,SAAS,CAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAEnE;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,CAAM;IAE5C;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAYpC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAEhD;;;;;;OAMG;IACH,iBAAiB,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAOlD;;;;OAIG;IACH,kBAAkB,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI;IAMtD;;;OAGG;IACH,eAAe,IAAI,IAAI;IAIvB;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,OAAO;IAItC;;;;;;OAMG;IACH,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAiE5D;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAMtD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IAgCzF;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,6BAA6B,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,GAAE,YAAY,EAAO,GAC1B,IAAI;IAuFP;;;;;;;OAOG;IACH,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAMhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc7E;;;;;;;OAOG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IAK7D;;;;;;OAMG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIjE;;;;;;OAMG;IACH,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,uBAAuB,GAAG,IAAI;IAIxD;;;;;;;OAOG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAahE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,oBAAoB,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,aAAa,GAAG,IAAI;IAgBvB;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQvE;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAmC5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAKvE;IAEF;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CACxB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,cAAc,UAAQ,GACrB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAuC5C;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,SAAS,UAAO,GACf;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAgC5C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,sBAAsB,CAC9B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,SAAS,eAAe,EAAE,EAC3C,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,aAAa,GAAG,IAAI;IAqBvB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQlE;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc1E;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAqBvE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,aAAa,GAAG,IAAI;CAQxB;AAMD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC1B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,wDAAwD;IACxD,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;IAC/B,wEAAwE;IACxE,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,uGAAuG;IACvG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wDAAwD;IACxD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;CACrC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CA2CtF"}
+{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAOL,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AAsEvB,YAAY,EAAE,YAAY,EAAE,CAAC;AAmC7B;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CACxB,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAClE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAC3B,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE;QACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;QACxE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5C,CAAC;CACH;AAMD;;;GAGG;AACH,8BAAsB,aAAc,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAE3C,0DAA0D;IAC1D,SAAS,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAE/C,8EAA8E;IAC9E,SAAS,CAAC,eAAe,EAAE,YAAY,EAAE,CAAM;IAE/C;;;;;;OAMG;IACH,SAAS,CAAC,iBAAiB,EAAE,YAAY,EAAE,CAAM;IAEjD,4DAA4D;IAC5D,SAAS,CAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAEnE;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe,CAAsB;IAE7C,kEAAkE;IAClE,sBAAsB,IAAI,SAAS,YAAY,EAAE;IAIjD;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,CAAM;IAE5C;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAYpC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAEhD;;;;;;OAMG;IACH,iBAAiB,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAOlD;;;;OAIG;IACH,kBAAkB,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI;IAMtD;;;OAGG;IACH,eAAe,IAAI,IAAI;IAIvB;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,OAAO;IAItC;;;;;;OAMG;IACH,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA8E5D;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAMtD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IAgCzF;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,6BAA6B,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,GAAE,YAAY,EAAO,GAC1B,IAAI;IAoHP;;;;;;;OAOG;IACH,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAMhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc7E;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,mBAAmB,CAC3B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,aAAa,GAAG,IAAI;IAiBvB;;;;;;;OAOG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IAK7D;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,CAAC,wBAAwB,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,OAAO;IASV;;;;;;OAMG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIjE;;;;;;OAMG;IACH,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,uBAAuB,GAAG,IAAI;IAIxD;;;;;;;OAOG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAahE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,oBAAoB,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,aAAa,GAAG,IAAI;IAgBvB;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQvE;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAmC5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAKvE;IAEF;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CACxB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,cAAc,UAAQ,GACrB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAuC5C;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,SAAS,UAAO,GACf;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAgC5C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,sBAAsB,CAC9B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,SAAS,eAAe,EAAE,EAC3C,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,aAAa,GAAG,IAAI;IAqBvB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQlE;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc1E;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAqBvE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,aAAa,GAAG,IAAI;CAQxB;AAMD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC1B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,wDAAwD;IACxD,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;IAC/B,wEAAwE;IACxE,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,uGAAuG;IACvG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wDAAwD;IACxD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;CACrC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CA2CtF"}

package/dist/core/tokenization/index.js

@@ -641,7 +641,8 @@ function createTokenizerContext(tokenizer) {
     direction: tokenizer.direction,
     lookupKeyword: tokenizer.lookupKeyword.bind(tokenizer),
     isKeyword: tokenizer.isKeyword.bind(tokenizer),
-    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer)
+    isKeywordStart: tokenizer.isKeywordStart.bind(tokenizer),
+    ...tokenizer.isKeywordStartAtBoundary ? { isKeywordStartAtBoundary: tokenizer.isKeywordStartAtBoundary.bind(tokenizer) } : {}
   };
   if (tokenizer.normalizer) {
     return { ...ctx, normalizer: tokenizer.normalizer };
@@ -689,18 +690,94 @@ function createLatinCharClassifiers(letterPattern) {
 
 // src/core/tokenization/base-tokenizer.ts
 var SIMPLE_TOKENIZER_OPERATOR_SET = new Set(DEFAULT_OPERATORS);
+var MARKER_CONCEPT_NORMALIZEDS = /* @__PURE__ */ new Set([
+  // Role-marker role names (profile.roleMarkers normalizeds)
+  "patient",
+  "destination",
+  "source",
+  "style",
+  "event",
+  "eventMarker",
+  "agent",
+  "goal",
+  "manner",
+  // Prepositional / positional modifier concepts matched via the role mechanism
+  // (profile.keywords "Modifiers"). `before`/`after`/`until` are intentionally
+  // NOT here — they are pattern literals (see the note above).
+  "into",
+  "from",
+  "to",
+  "with",
+  "at",
+  "of",
+  "as",
+  "by",
+  "in",
+  "on",
+  "over",
+  "under",
+  "between",
+  "through",
+  "without"
+]);
+var ENGLISH_DOM_EVENT_NAMES = [
+  "click",
+  "dblclick",
+  "input",
+  "change",
+  "submit",
+  "keydown",
+  "keyup",
+  "keypress",
+  "mousedown",
+  "mouseup",
+  "mouseover",
+  "mouseout",
+  "mouseenter",
+  "mouseleave",
+  "mousemove",
+  "pointerdown",
+  "pointerup",
+  "pointermove",
+  "focus",
+  "blur",
+  "load",
+  "resize",
+  "scroll"
+];
 var _BaseTokenizer = class _BaseTokenizer {
   constructor() {
     /** Keywords derived from profile, sorted longest-first for greedy matching */
     this.profileKeywords = [];
+    /**
+     * Space-containing profile keywords (multi-word phrases), longest-first.
+     * Used by `tryMultiWordKeyword` so natural spaced forms (hi `मेल खाता`,
+     * vi `chuyển đổi`, es `tecla abajo`, …) tokenize as ONE keyword — the
+     * profile-driven replacement for the per-language hardcoded compound lists.
+     * Empty for no-space (CJK) languages, so they are unaffected.
+     */
+    this.multiWordKeywords = [];
     /** Map for O(1) keyword lookups by lowercase native word */
     this.profileKeywordMap = /* @__PURE__ */ new Map();
+    /**
+     * The raw EXTRAS list passed to initializeKeywordsFromProfile, kept pre-dedup.
+     * The keyword map is keyed by native word with last-wins insertion, so a
+     * duplicate native word inside the extras silently shadows the earlier entry
+     * (e.g. a `nächste→closest` entry shadowing `nächste→next` broke German
+     * positional expressions). Exposed so consistency tests can detect such
+     * intra-extras collisions, which are invisible in the deduplicated map.
+     */
+    this.rawExtraEntries = [];
     /**
      * Pluggable value extractors for domain-specific syntax.
      * When registered, BaseTokenizer will use extractor-based tokenization instead of legacy methods.
      */
     this.extractors = [];
   }
+  /** Raw extras as passed in, pre-dedup — for consistency tests. */
+  getExtraKeywordEntries() {
+    return this.rawExtraEntries;
+  }
   /**
    * Tokenize input string to token stream.
    * Delegates to extractor-based tokenization if extractors are registered,
@@ -769,6 +846,12 @@ var _BaseTokenizer = class _BaseTokenizer {
         pos++;
       }
       if (pos >= input.length) break;
+      const multiWord = this.tryMultiWordKeyword(input, pos);
+      if (multiWord) {
+        tokens.push(multiWord);
+        pos = multiWord.position.end;
+        continue;
+      }
       let extracted = false;
       for (const extractor of this.extractors) {
         if (extractor.canExtract(input, pos)) {
@@ -868,6 +951,7 @@ var _BaseTokenizer = class _BaseTokenizer {
    */
   initializeKeywordsFromProfile(profile, extras = []) {
     const keywordMap = /* @__PURE__ */ new Map();
+    this.rawExtraEntries = extras;
     if (profile.keywords) {
       for (const [normalized2, translation] of Object.entries(profile.keywords)) {
         keywordMap.set(translation.primary, {
@@ -911,12 +995,20 @@ var _BaseTokenizer = class _BaseTokenizer {
         keywordMap.set(native, { native, normalized: normalized2 });
       }
     }
+    for (const evt of ENGLISH_DOM_EVENT_NAMES) {
+      if (!keywordMap.has(evt)) {
+        keywordMap.set(evt, { native: evt, normalized: evt });
+      }
+    }
     for (const extra of extras) {
       keywordMap.set(extra.native, extra);
     }
     this.profileKeywords = Array.from(keywordMap.values()).sort(
       (a, b) => b.native.length - a.native.length
     );
+    this.multiWordKeywords = this.profileKeywords.filter(
+      (k) => k.native.includes(" ") && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
+    );
     this.profileKeywordMap = /* @__PURE__ */ new Map();
     for (const keyword of this.profileKeywords) {
       this.profileKeywordMap.set(keyword.native.toLowerCase(), keyword);
@@ -958,6 +1050,35 @@ var _BaseTokenizer = class _BaseTokenizer {
     }
     return null;
   }
+  /**
+   * Match the longest multi-word (space-containing) profile keyword at `pos`,
+   * requiring the match to end at a word boundary. The profile-driven
+   * counterpart of the per-language hardcoded compound lists (the hindi and
+   * vietnamese keyword extractors). Returns a keyword token (with the normalized
+   * form) or null. Case-sensitive against the stored native form, mirroring
+   * `tryProfileKeyword`/`isKeywordStart` (the i18n dicts emit a fixed surface
+   * case). No-op when `multiWordKeywords` is empty (no-space/CJK languages).
+   *
+   * @param input - Input string
+   * @param pos - Current position (must be a token-start boundary)
+   * @param isWordChar - End-boundary predicate (defaults to Unicode letter/digit/_)
+   */
+  tryMultiWordKeyword(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    if (this.multiWordKeywords.length === 0) return null;
+    const rest = input.slice(pos);
+    for (const entry of this.multiWordKeywords) {
+      if (!rest.startsWith(entry.native)) continue;
+      const after = input[pos + entry.native.length];
+      if (after !== void 0 && isWordChar(after)) continue;
+      return createToken(
+        entry.native,
+        "keyword",
+        createPosition(pos, pos + entry.native.length),
+        entry.normalized
+      );
+    }
+    return null;
+  }
   /**
    * Check if the remaining input starts with any known keyword.
    * Useful for non-space languages to detect word boundaries.
@@ -970,6 +1091,32 @@ var _BaseTokenizer = class _BaseTokenizer {
     const remaining = input.slice(pos);
     return this.profileKeywords.some((entry) => remaining.startsWith(entry.native));
   }
+  /**
+   * Check if a known keyword starts at the given position AND ends at a word
+   * boundary (end of input or a non-word character).
+   *
+   * Space-delimited languages must use this (not `isKeywordStart`) for
+   * word-walk break checks: the keyword table includes English canonical
+   * fallbacks (me, it, you, …), so a raw `startsWith` check splits any native
+   * word with an embedded fallback mid-word (e.g. Quechua ñit'iy contains
+   * "it"). CJK/no-space tokenizers rely on mid-text keyword starts and must
+   * keep using `isKeywordStart`.
+   *
+   * @param input - Input string
+   * @param pos - Current position
+   * @param isWordChar - Language-specific word-character predicate; pass the
+   *   tokenizer's letter classifier so e.g. the Quechua glottal apostrophe
+   *   counts as part of a word. Defaults to Unicode letters/digits/underscore.
+   * @returns true if a keyword starts here and is not followed by a word char
+   */
+  isKeywordStartAtBoundary(input, pos, isWordChar = (ch) => /[\p{L}\p{N}_]/u.test(ch)) {
+    const remaining = input.slice(pos);
+    return this.profileKeywords.some((entry) => {
+      if (!remaining.startsWith(entry.native)) return false;
+      const after = input[pos + entry.native.length];
+      return after === void 0 || !isWordChar(after);
+    });
+  }
   /**
    * Look up a keyword by native word (case-insensitive).
    * O(1) lookup using the keyword map.