@lokascript/framework 2.11.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +301 -1
- package/dist/api/domain-registry.d.ts +2 -2
- package/dist/api/index.js.map +1 -1
- package/dist/core/index.js +7 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/tokenization/base-tokenizer.d.ts.map +1 -1
- package/dist/core/tokenization/index.js +7 -1
- package/dist/core/tokenization/index.js.map +1 -1
- package/dist/index.cjs +7 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/interfaces/value-extractor.d.ts.map +1 -1
- package/dist/multilingual/index.js +7 -1
- package/dist/multilingual/index.js.map +1 -1
- package/dist/prompts/prompt-generator.d.ts +1 -1
- package/package.json +2 -2
- package/src/api/domain-registry.ts +2 -2
- package/src/core/tokenization/apostrophe-possessive.test.ts +42 -0
- package/src/core/tokenization/base-tokenizer.ts +21 -3
- package/src/interfaces/value-extractor.ts +10 -0
- package/src/prompts/prompt-generator.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"value-extractor.d.ts","sourceRoot":"","sources":["../../src/interfaces/value-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAMrF;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,6CAA6C;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAErD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CACnE;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,cAAc;IAC3D,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;
|
|
1
|
+
{"version":3,"file":"value-extractor.d.ts","sourceRoot":"","sources":["../../src/interfaces/value-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAMrF;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,6CAA6C;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAErD;;;;;;;;;;OAUG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CACnE;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,cAAc;IAC3D,QAAQ,CAAC,IAAI,oBAAoB;IAEjC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAqBpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CA8DlE;AAED;;GAEG;AACH,qBAAa,eAAgB,YAAW,cAAc;IACpD,QAAQ,CAAC,IAAI,YAAY;IAEzB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAgFlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAmBlE;AAED;;;;;;GAMG;AACH,qBAAa,0BAA2B,YAAW,cAAc;IAC/D,QAAQ,CAAC,IAAI,wBAAwB;IAErC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAQpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAelE;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gCAAiC,YAAW,cAAc;IACrE,QAAQ,CAAC,IAAI,+BAA+B;IAE5C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAQlE;AA6BD,qBAAa,oBAAqB,YAAW,cAAc;IACzD,QAAQ,CAAC,IAAI,kBAAkB;IAE/B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAUpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAQlE;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIpD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAclE;AAMD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,qBAAqB;IACrB,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IAElC;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IAExD;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAEnC;;;OAGG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IAEzD;;;;;;;;OAQG;IACH,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IAEX;;OAEG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;CAC/C;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC3D;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;CAC7C;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,cAAc,GAAG,qBAAqB,GAChD,SAAS,IAAI,qBAAqB,CAEpC;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,KAAK,GAAG,KAAK,CAAC;IACzB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAC;IACxD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACzD,wBAAwB,CAAC,CACvB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,GACrC,OAAO,CAAC;IACX,UAAU,CAAC,EAAE,uBAAuB,CAAC;CACtC,GAAG,gBAAgB,CAiBnB"}
|
|
@@ -5,6 +5,9 @@ var StringLiteralExtractor = class {
|
|
|
5
5
|
}
|
|
6
6
|
canExtract(input, position) {
|
|
7
7
|
const char = input[position];
|
|
8
|
+
if (char === "'" && position > 0 && /[\p{L}\p{N}_)\]]/u.test(input[position - 1])) {
|
|
9
|
+
return false;
|
|
10
|
+
}
|
|
8
11
|
return char === '"' || char === "'" || char === "`" || char === "\u201C" || // Chinese double quote open "
|
|
9
12
|
char === "\u2018";
|
|
10
13
|
}
|
|
@@ -719,6 +722,9 @@ var MARKER_CONCEPT_NORMALIZEDS = /* @__PURE__ */ new Set([
|
|
|
719
722
|
"through",
|
|
720
723
|
"without"
|
|
721
724
|
]);
|
|
725
|
+
function isHyphenatedWord(native) {
|
|
726
|
+
return native.includes("-") && /^[\p{L}\p{N}_]+(-[\p{L}\p{N}_]+)+$/u.test(native);
|
|
727
|
+
}
|
|
722
728
|
var ENGLISH_DOM_EVENT_NAMES = [
|
|
723
729
|
"click",
|
|
724
730
|
"dblclick",
|
|
@@ -1038,7 +1044,7 @@ var _BaseTokenizer = class _BaseTokenizer {
|
|
|
1038
1044
|
(a, b) => b.native.length - a.native.length
|
|
1039
1045
|
);
|
|
1040
1046
|
this.multiWordKeywords = this.profileKeywords.filter(
|
|
1041
|
-
(k) => k.native.includes(" ") && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
|
|
1047
|
+
(k) => (k.native.includes(" ") || isHyphenatedWord(k.native)) && !MARKER_CONCEPT_NORMALIZEDS.has(k.normalized)
|
|
1042
1048
|
);
|
|
1043
1049
|
this.profileKeywordMap = /* @__PURE__ */ new Map();
|
|
1044
1050
|
for (const keyword of this.profileKeywords) {
|