@lokascript/framework 2.11.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -0
- package/dist/api/domain-registry.d.ts +2 -2
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.js +7 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/tokenization/base-tokenizer.d.ts.map +1 -1
- package/dist/core/tokenization/index.js +7 -1
- package/dist/core/tokenization/index.js.map +1 -1
- package/dist/index.cjs +7 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/interfaces/value-extractor.d.ts.map +1 -1
- package/dist/multilingual/index.js +7 -1
- package/dist/multilingual/index.js.map +1 -1
- package/dist/prompts/prompt-generator.d.ts +1 -1
- package/package.json +2 -2
- package/src/api/domain-registry.ts +2 -2
- package/src/core/tokenization/apostrophe-possessive.test.ts +42 -0
- package/src/core/tokenization/base-tokenizer.ts +21 -3
- package/src/interfaces/value-extractor.ts +10 -0
- package/src/prompts/prompt-generator.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAQL,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"base-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/tokenization/base-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AACzF,OAAO,KAAK,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EAGlB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAQL,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AA+EvB,YAAY,EAAE,YAAY,EAAE,CAAC;AAmC7B;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CACxB,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAClE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAC3B,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChE,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,EAAE;QACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,cAAc,EAAE,cAAc,GAAG,SAAS,GAAG,iBAAiB,CAAC;QACxE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5C,CAAC;CACH;AAMD;;;GAGG;AACH,8BAAsB,aAAc,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAE3C,0DAA0D;IAC1D,SAAS,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAE/C,8EAA8E;IAC9E,SAAS,CAAC,eAAe,EAAE,YAAY,EAAE,CAAM;IAE/C;;;;;;OAMG;IACH,SAAS,CAAC,iBAAiB,EAAE,YAAY,EAAE,CAAM;IAEjD,4DAA4D;IAC5D,SAAS,CAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAa;IAEnE;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe,CAAsB;IAE7C,kEAAkE;IAClE,sBAAsB,IAAI,SAAS,YAAY,EAAE;IAIjD;;;OAGG;IACH,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,CAAM;IAE5C;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IAYpC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS;IAEhD;;;;;;OAMG;IACH,iBAAiB,CAAC,SAAS,EAAE,cAAc,GAAG,IAAI;IAOlD;;;;OAIG;IACH,kBAAkB,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI;IAMtD;;;OAGG;IACH,eAAe,IAAI,IAAI;IAIvB;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,OAAO;IAItC;;;;;;OAMG;IACH,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW;IA8E5D;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA8B;IAEhE,iFAAiF;IACjF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA+B;IAEtE;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,wBAAwB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE;IA6B5E;;;;;;OAMG;IACH,SAAS,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS;IAMtD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IAgCzF;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,6BAA6B,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,GAAE,YAAY,EAAO,GAC1B,IAAI;IA6HP;;;;;;;OAOG;IACH,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAIhD;;;;;;;OAOG;IACH,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc7E;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,mBAAmB,CAC3B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,aAAa,GAAG,IAAI;IAiBvB;;;;;;;OAOG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IAK7D;;;;;;;;;;;;;;;;;OAiBG;IACH,SAAS,CAAC,wBAAwB,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAyC,GACtE,OAAO;IASV;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAQjE;;;;;;OAMG;IACH,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,uBAAuB,GAAG,IAAI;IAIxD;;;;;;;OAOG;IACH,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI;IAahE;;;;;;;;;;;;;;;OAeG;IACH,SAAS,CAAC,oBAAoB,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACb,aAAa,GAAG,IAAI;IAgBvB;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQvE;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAmC5E;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;OAEG;IACH,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQrE;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAKvE;IAEF;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CACxB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,cAAc,UAAQ,GACrB;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAuC5C;;;;;;;;OAQG;IACH,SAAS,CAAC,eAAe,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,SAAS,UAAO,GACf;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAgC5C;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,sBAAsB,CAC9B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,SAAS,eAAe,EAAE,EAC3C,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAC9D,aAAa,GAAG,IAAI;IAqBvB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAQlE;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAc1E;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAqBvE;;;;;;;;;;OAUG;IACH,SAAS,CAAC,oBAAoB,CAC5B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,MAAM,EAAE,GAC3B,aAAa,GAAG,IAAI;CAQxB;AAMD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,qBAAqB;IACpC,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,sCAAsC;IACtC,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC1B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,wDAAwD;IACxD,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;IAC/B,wEAAwE;IACxE,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,uGAAuG;IACvG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,wDAAwD;IACxD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,cAAc,EAAE,CAAC;CACrC;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,qBAAqB,GAAG,iBAAiB,CA2CtF"}
|
|
@@ -457,6 +457,9 @@ var StringLiteralExtractor = class {
|
|
|
457
457
|
}
|
|
458
458
|
canExtract(input, position) {
|
|
459
459
|
const char = input[position];
|
|
460
|
+
if (char === "'" && position > 0 && /[\p{L}\p{N}_)\]]/u.test(input[position - 1])) {
|
|
461
|
+
return false;
|
|
462
|
+
}
|
|
460
463
|
return char === '"' || char === "'" || char === "`" || char === "\u201C" || // Chinese double quote open "
|
|
461
464
|
char === "\u2018";
|
|
462
465
|
}
|
|
@@ -723,6 +726,9 @@ var MARKER_CONCEPT_NORMALIZEDS = /* @__PURE__ */ new Set([
|
|
|
723
726
|
"through",
|
|
724
727
|
"without"
|
|
725
728
|
]);
|
|
729
|
+
function isHyphenatedWord(native) {
|
|
730
|
+
return native.includes("-") && /^[\p{L}\p{N}_]+(-[\p{L}\p{N}_]+)+$/u.test(native);
|
|
731
|
+
}
|
|
726
732
|
var ENGLISH_DOM_EVENT_NAMES = [
|
|
727
733
|
"click",
|
|
728
734
|
"dblclick",
|
|
@@ -1042,7 +1048,7 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
1042
1048
|
(a, b) => b.native.length - a.native.length
|
|
1043
1049
|
);
|
|
1044
1050
|
this.multiWordKeywords = this.profileKeywords.filter(
|
|
1045
|
-
(k) => k.native.includes(" ") && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
|
|
1051
|
+
(k) => (k.native.includes(" ") || isHyphenatedWord(k.native)) && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
|
|
1046
1052
|
);
|
|
1047
1053
|
this.profileKeywordMap = /* @__PURE__ */ new Map();
|
|
1048
1054
|
for (const keyword of this.profileKeywords) {
|