@lokascript/framework 2.8.0 → 2.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +393 -0
- package/dist/api/create-dsl.d.ts +93 -1
- package/dist/api/create-dsl.d.ts.map +1 -1
- package/dist/api/domain-registry.d.ts +5 -3
- package/dist/api/domain-registry.d.ts.map +1 -1
- package/dist/api/index.js +232 -18
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.js +26 -6
- package/dist/core/index.js.map +1 -1
- package/dist/core/tokenization/base-tokenizer.d.ts +15 -3
- package/dist/core/tokenization/base-tokenizer.d.ts.map +1 -1
- package/dist/core/tokenization/char-classifiers.d.ts +2 -2
- package/dist/core/tokenization/index.js +26 -6
- package/dist/core/tokenization/index.js.map +1 -1
- package/dist/core/tokenization/token-utils.d.ts +15 -0
- package/dist/core/tokenization/token-utils.d.ts.map +1 -1
- package/dist/generation/index.js +73 -47
- package/dist/generation/index.js.map +1 -1
- package/dist/generation/pattern-generator.d.ts +8 -1
- package/dist/generation/pattern-generator.d.ts.map +1 -1
- package/dist/generation/renderer.d.ts +53 -1
- package/dist/generation/renderer.d.ts.map +1 -1
- package/dist/index.cjs +302 -115
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +298 -115
- package/dist/index.js.map +1 -1
- package/dist/interfaces/value-extractor.d.ts +5 -0
- package/dist/interfaces/value-extractor.d.ts.map +1 -1
- package/dist/multilingual/index.js +25 -6
- package/dist/multilingual/index.js.map +1 -1
- package/package.json +4 -3
- package/src/api/create-dsl.test.ts +11 -0
- package/src/api/create-dsl.ts +278 -9
- package/src/api/domain-registry.ts +15 -10
- package/src/api/extensions.test.ts +322 -0
- package/src/core/tokenization/base-tokenizer.ts +23 -8
- package/src/core/tokenization/char-classifiers.ts +2 -2
- package/src/core/tokenization/css-selector-extractor.test.ts +67 -0
- package/src/core/tokenization/token-utils.ts +18 -0
- package/src/generation/domain-renderer.test.ts +172 -0
- package/src/generation/pattern-generator.test.ts +102 -0
- package/src/generation/pattern-generator.ts +27 -19
- package/src/generation/renderer.test.ts +243 -4
- package/src/generation/renderer.ts +188 -45
- package/src/index.ts +9 -1
- package/src/interfaces/value-extractor.ts +50 -0
|
@@ -229,16 +229,28 @@ export declare abstract class BaseTokenizer implements LanguageTokenizer {
|
|
|
229
229
|
*/
|
|
230
230
|
protected isKeywordStartAtBoundary(input: string, pos: number, isWordChar?: (char: string) => boolean): boolean;
|
|
231
231
|
/**
|
|
232
|
-
* Look up a keyword by native word (case-insensitive).
|
|
232
|
+
* Look up a keyword by native word (case-insensitive, diacritic-insensitive).
|
|
233
233
|
* O(1) lookup using the keyword map.
|
|
234
234
|
*
|
|
235
|
+
* The map is INDEXED both with and without diacritics (see
|
|
236
|
+
* `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
|
|
237
|
+
* that: it lets a surface form carrying harakat the profile does not happen to
|
|
238
|
+
* spell still find its entry. Only consulted after the exact lookup misses, so
|
|
239
|
+
* every previously-matching word resolves byte-identically.
|
|
240
|
+
*
|
|
241
|
+
* Half-implementing this — indexing stripped but querying exact — is what made
|
|
242
|
+
* diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
|
|
243
|
+
* `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
|
|
244
|
+
* exists to prevent exactly that handed the word on, and the single-char `ب`
|
|
245
|
+
* bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
|
|
246
|
+
*
|
|
235
247
|
* @param native - Native word to look up
|
|
236
248
|
* @returns KeywordEntry if found, undefined otherwise
|
|
237
249
|
*/
|
|
238
250
|
protected lookupKeyword(native: string): KeywordEntry | undefined;
|
|
239
251
|
/**
|
|
240
|
-
* Check if a word is a known keyword (case-insensitive).
|
|
241
|
-
* O(1) lookup using the keyword map.
|
|
252
|
+
* Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
|
|
253
|
+
* O(1) lookup using the keyword map. See {@link lookupKeyword}.
|
|
242
254
|
*
|
|
243
255
|
* @param native - Native word to check
|
|
244
256
|
* @returns true if the word is a keyword
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,
|
|
1
|
+
{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAQL,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AAsEvB,YAAY,EAAE,YAAY,EAAE,CAAC;AAmC7B;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CACxB,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAClE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAC3B,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE;QACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;QACxE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5C,CAAC;CACH;AAMD;;;GAGG;AACH,8BAAsB,aAAc,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAE3C,0DAA0D;IAC1D,SAAS,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAE/C,8EAA8E;IAC9E,SAAS,CAAC,eAAe,EAAE,YAAY,EAAE,CAAM;IAE/C;;;;;;OAMG;IACH,SAAS,CAAC,iBAAiB,EAAE,YAAY,EAAE,CAAM;IAEjD,4DAA4D;IAC5D,SAAS,CAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAEnE;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe,CAAsB;IAE7C,kEAAkE;IAClE,sBAAsB,IAAI,SAAS,YAAY,EAAE;IAIjD;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,CAAM;IAE5C;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAYpC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAEhD;;;;;;OAMG;IACH,iBAAiB,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAOlD;;;;OAIG;IACH,kBAAkB,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI;IAMtD;;;OAGG;IACH,eAAe,IAAI,IAAI;IAIvB;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,OAAO;IAItC;;;;;;OAMG;IACH,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA8E5D;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA8B;IAEhE,iFAAiF;IACjF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA+B;IAEtE;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,wBAAwB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE;IA6B5E;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAMtD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IAgCzF;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,6BAA6B,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,GAAE,YAAY,EAAO,GAC1B,IAAI;IAoHP;;;;;;;OAOG;IACH,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAIhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc7E;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,mBAAmB,CAC3B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,aAAa,GAAG,IAAI;IAiBvB;;;;;;;OAOG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IAK7D;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,CAAC,wBAAwB,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,OAAO;IASV;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAQjE;;;;;;OAMG;IACH,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,uBAAuB,GAAG,IAAI;IAIxD;;;;;;;OAOG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAahE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,oBAAoB,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,aAAa,GAAG,IAAI;IAgBvB;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQvE;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAmC5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAKvE;IAEF;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CACxB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,cAAc,UAAQ,GACrB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAuC5C;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,SAAS,UAAO,GACf;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAgC5C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,sBAAsB,CAC9B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,SAAS,eAAe,EAAE,EAC3C,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,aAAa,GAAG,IAAI;IAqBvB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQlE;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc1E;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAqBvE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,aAAa,GAAG,IAAI;CAQxB;AAMD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC1B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,wDAAwD;IACxD,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;IAC/B,wEAAwE;IACxE,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,uGAAuG;IACvG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wDAAwD;IACxD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;CACrC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CA2CtF"}
|
|
@@ -47,10 +47,10 @@ export interface LatinCharClassifiers {
|
|
|
47
47
|
*
|
|
48
48
|
* @example
|
|
49
49
|
* // Spanish letters
|
|
50
|
-
* const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-
|
|
50
|
+
* const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-Z\u00e1\u00e9\u00ed\u00f3\u00fa\u00fc\u00f1\u00c1\u00c9\u00cd\u00d3\u00da\u00dc\u00d1]/);
|
|
51
51
|
*
|
|
52
52
|
* // German letters
|
|
53
|
-
* const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-
|
|
53
|
+
* const { isLetter, isIdentifierChar } = createLatinCharClassifiers(/[a-zA-Z\u00e4\u00f6\u00fc\u00c4\u00d6\u00dc\u00df]/);
|
|
54
54
|
*/
|
|
55
55
|
export declare function createLatinCharClassifiers(letterPattern: RegExp): LatinCharClassifiers;
|
|
56
56
|
//# sourceMappingURL=char-classifiers.d.ts.map
|
|
@@ -104,6 +104,9 @@ function isQuote(char) {
|
|
|
104
104
|
function isDigit(char) {
|
|
105
105
|
return /\d/.test(char);
|
|
106
106
|
}
|
|
107
|
+
function stripOptionalDiacritics(word) {
|
|
108
|
+
return word.replace(/[\u064b-\u0652\u0670]/g, "");
|
|
109
|
+
}
|
|
107
110
|
function isAsciiLetter(char) {
|
|
108
111
|
return /[a-zA-Z]/.test(char);
|
|
109
112
|
}
|
|
@@ -1059,7 +1062,7 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
1059
1062
|
* @returns Word without diacritics
|
|
1060
1063
|
*/
|
|
1061
1064
|
removeDiacritics(word) {
|
|
1062
|
-
return word
|
|
1065
|
+
return stripOptionalDiacritics(word);
|
|
1063
1066
|
}
|
|
1064
1067
|
/**
|
|
1065
1068
|
* Try to match a keyword from profile at the current position.
|
|
@@ -1150,24 +1153,40 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
1150
1153
|
});
|
|
1151
1154
|
}
|
|
1152
1155
|
/**
|
|
1153
|
-
* Look up a keyword by native word (case-insensitive).
|
|
1156
|
+
* Look up a keyword by native word (case-insensitive, diacritic-insensitive).
|
|
1154
1157
|
* O(1) lookup using the keyword map.
|
|
1155
1158
|
*
|
|
1159
|
+
* The map is INDEXED both with and without diacritics (see
|
|
1160
|
+
* `initializeKeywordsFromProfile`), so a stripped QUERY is the other half of
|
|
1161
|
+
* that: it lets a surface form carrying harakat the profile does not happen to
|
|
1162
|
+
* spell still find its entry. Only consulted after the exact lookup misses, so
|
|
1163
|
+
* every previously-matching word resolves byte-identically.
|
|
1164
|
+
*
|
|
1165
|
+
* Half-implementing this — indexing stripped but querying exact — is what made
|
|
1166
|
+
* diacritized `بَدِّل` (toggle) tokenize as `kind=particle normalized=with`:
|
|
1167
|
+
* `isKeyword` returned false, so the guard in `ArabicProcliticExtractor` that
|
|
1168
|
+
* exists to prevent exactly that handed the word on, and the single-char `ب`
|
|
1169
|
+
* bi- proclitic claimed it. A wrong CONCEPT, not a failed parse.
|
|
1170
|
+
*
|
|
1156
1171
|
* @param native - Native word to look up
|
|
1157
1172
|
* @returns KeywordEntry if found, undefined otherwise
|
|
1158
1173
|
*/
|
|
1159
1174
|
lookupKeyword(native) {
|
|
1160
|
-
|
|
1175
|
+
const exact = this.profileKeywordMap.get(native.toLowerCase());
|
|
1176
|
+
if (exact) return exact;
|
|
1177
|
+
const stripped = this.removeDiacritics(native);
|
|
1178
|
+
if (stripped === native) return void 0;
|
|
1179
|
+
return this.profileKeywordMap.get(stripped.toLowerCase());
|
|
1161
1180
|
}
|
|
1162
1181
|
/**
|
|
1163
|
-
* Check if a word is a known keyword (case-insensitive).
|
|
1164
|
-
* O(1) lookup using the keyword map.
|
|
1182
|
+
* Check if a word is a known keyword (case-insensitive, diacritic-insensitive).
|
|
1183
|
+
* O(1) lookup using the keyword map. See {@link lookupKeyword}.
|
|
1165
1184
|
*
|
|
1166
1185
|
* @param native - Native word to check
|
|
1167
1186
|
* @returns true if the word is a keyword
|
|
1168
1187
|
*/
|
|
1169
1188
|
isKeyword(native) {
|
|
1170
|
-
return this.
|
|
1189
|
+
return this.lookupKeyword(native) !== void 0;
|
|
1171
1190
|
}
|
|
1172
1191
|
/**
|
|
1173
1192
|
* Set the morphological normalizer for this tokenizer.
|
|
@@ -1660,6 +1679,7 @@ export {
|
|
|
1660
1679
|
isWhitespace,
|
|
1661
1680
|
noChange,
|
|
1662
1681
|
normalized,
|
|
1682
|
+
stripOptionalDiacritics,
|
|
1663
1683
|
withDefaultExtractors
|
|
1664
1684
|
};
|
|
1665
1685
|
//# sourceMappingURL=index.js.map
|