@lokascript/framework 2.8.0 → 2.9.0

package/dist/interfaces/value-extractor.d.ts

@@ -116,6 +116,11 @@ export declare class LatinExtendedIdentifierExtractor implements ValueExtractor
     canExtract(input: string, position: number): boolean;
     extract(input: string, position: number): ExtractionResult | null;
 }
+export declare class CssSelectorExtractor implements ValueExtractor {
+    readonly name = "css-selector";
+    canExtract(input: string, position: number): boolean;
+    extract(input: string, position: number): ExtractionResult | null;
+}
 /**
  * Whitespace extractor - handles spaces, tabs, newlines.
  */

package/dist/interfaces/value-extractor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"value-extractor.d.ts","sourceRoot":"","sources":["../../src/interfaces/value-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAMrF;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,6CAA6C;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAErD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CACnE;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,cAAc;IAC3D,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAWpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CA8DlE;AAED;;GAEG;AACH,qBAAa,eAAgB,YAAW,cAAc;IACpD,QAAQ,CAAC,IAAI,YAAY;IAEzB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAgFlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAmBlE;AAED;;;;;;GAMG;AACH,qBAAa,0BAA2B,YAAW,cAAc;IAC/D,QAAQ,CAAC,IAAI,wBAAwB;IAErC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAQpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAelE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gCAAiC,YAAW,cAAc;IACrE,QAAQ,CAAC,IAAI,+BAA+B;IAE5C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAQlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAclE;AAMD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,qBAAqB;IACrB,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAElC;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IAExD;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEnC;;;OAGG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzD;;;;;;;;OAQG;IACH,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IAEX;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;CAC/C;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;CAC7C;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,cAAc,GAAG,qBAAqB,GAChD,SAAS,IAAI,qBAAqB,CAEpC;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IACxD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACzD,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IACX,UAAU,CAAC,EAAE,uBAAuB,CAAC;CACtC,GAAG,gBAAgB,CAiBnB"}
+{"version":3,"file":"value-extractor.d.ts","sourceRoot":"","sources":["../../src/interfaces/value-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAMrF;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,6CAA6C;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAErD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CACnE;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,cAAc;IAC3D,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAWpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CA8DlE;AAED;;GAEG;AACH,qBAAa,eAAgB,YAAW,cAAc;IACpD,QAAQ,CAAC,IAAI,YAAY;IAEzB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAgFlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAmBlE;AAED;;;;;;GAMG;AACH,qBAAa,0BAA2B,YAAW,cAAc;IAC/D,QAAQ,CAAC,IAAI,wBAAwB;IAErC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAQpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAelE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gCAAiC,YAAW,cAAc;IACrE,QAAQ,CAAC,IAAI,+BAA+B;IAE5C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAQlE;AA6BD,qBAAa,oBAAqB,YAAW,cAAc;IACzD,QAAQ,CAAC,IAAI,kBAAkB;IAE/B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAUpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAQlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAclE;AAMD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,qBAAqB;IACrB,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAElC;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IAExD;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEnC;;;OAGG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzD;;;;;;;;OAQG;IACH,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IAEX;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;CAC/C;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;CAC7C;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,cAAc,GAAG,qBAAqB,GAChD,SAAS,IAAI,qBAAqB,CAEpC;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IACxD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACzD,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IACX,UAAU,CAAC,EAAE,uBAAuB,CAAC;CACtC,GAAG,gBAAgB,CAiBnB"}

package/dist/multilingual/index.js

@@ -323,6 +323,9 @@ function isQuote(char) {
 function isDigit(char) {
   return /\d/.test(char);
 }
+function stripOptionalDiacritics(word) {
+  return word.replace(/[ً-ْٰ]/g, "");
+}
 function isAsciiLetter(char) {
   return /[a-zA-Z]/.test(char);
 }
@@ -1055,7 +1058,7 @@ var _BaseTokenizer = class _BaseTokenizer {
    * @returns Word without diacritics
    */
   removeDiacritics(word) {
-    return word.replace(/[\u064B-\u0652\u0670]/g, "");
+    return stripOptionalDiacritics(word);
   }
   /**
    * Try to match a keyword from profile at the current position.
@@ -1146,24 +1149,40 @@ var _BaseTokenizer = class _BaseTokenizer {
     });
   }
   /**
-   * Look up a keyword by native word (case-insensitive).
+   * Look up a keyword by native word (case-insensitive, diacritic-insensitive).
    * O(1) lookup using the keyword map.
    *
+   * The map is INDEXED both with and without diacritics (see
+   * `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
+   * that: it lets a surface form carrying harakat the profile does not happen to
+   * spell still find its entry. Only consulted after the exact lookup misses, so
+   * every previously-matching word resolves byte-identically.
+   *
+   * Half-implementing this — indexing stripped but querying exact — is what made
+   * diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
+   * `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
+   * exists to prevent exactly that handed the word on, and the single-char `ب`
+   * bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
+   *
    * @param native - Native word to look up
    * @returns KeywordEntry if found, undefined otherwise
    */
   lookupKeyword(native) {
-    return this.profileKeywordMap.get(native.toLowerCase());
+    const exact = this.profileKeywordMap.get(native.toLowerCase());
+    if (exact) return exact;
+    const stripped = this.removeDiacritics(native);
+    if (stripped === native) return void 0;
+    return this.profileKeywordMap.get(stripped.toLowerCase());
   }
   /**
-   * Check if a word is a known keyword (case-insensitive).
-   * O(1) lookup using the keyword map.
+   * Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
+   * O(1) lookup using the keyword map. See {@link lookupKeyword}.
    *
    * @param native - Native word to check
    * @returns true if the word is a keyword
    */
   isKeyword(native) {
-    return this.profileKeywordMap.has(native.toLowerCase());
+    return this.lookupKeyword(native) !== void 0;
   }
   /**
    * Set the morphological normalizer for this tokenizer.