@inseefr/lunatic 3.12.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/utils/search/tokenizer.js +7 -3
- package/esm/utils/search/tokenizer.js.map +1 -1
- package/esm/utils/search/tokenizer.spec.js +5 -0
- package/esm/utils/search/tokenizer.spec.js.map +1 -1
- package/package.json +2 -2
- package/src/utils/search/tokenizer.spec.ts +7 -0
- package/src/utils/search/tokenizer.ts +11 -3
- package/tsconfig.build.tsbuildinfo +1 -1
- package/utils/search/tokenizer.js +7 -3
- package/utils/search/tokenizer.js.map +1 -1
- package/utils/search/tokenizer.spec.js +5 -0
- package/utils/search/tokenizer.spec.js.map +1 -1
|
@@ -36,6 +36,7 @@ exports.tokenizeQuery = tokenizeQuery;
|
|
|
36
36
|
* Tokenizer used for indexing (based on "fields" specification)
|
|
37
37
|
*/
|
|
38
38
|
const tokenizeIndex = (str, info, stopWords) => {
|
|
39
|
+
let normalizedStr = normalizeStr(str);
|
|
39
40
|
const wordRegex = info.rules && info.rules !== 'soft'
|
|
40
41
|
? new RegExp(info.rules[0], 'gi')
|
|
41
42
|
: /\w+/gi;
|
|
@@ -43,12 +44,15 @@ const tokenizeIndex = (str, info, stopWords) => {
|
|
|
43
44
|
// For synonyms, add the synonyms to the string
|
|
44
45
|
if (info.synonyms) {
|
|
45
46
|
for (const source in info.synonyms) {
|
|
46
|
-
const
|
|
47
|
-
|
|
47
|
+
const normalizedSource = normalizeStr(source);
|
|
48
|
+
const synonyms = info.synonyms[source]
|
|
49
|
+
.map((synonym) => normalizeStr(synonym))
|
|
50
|
+
.join(' ');
|
|
51
|
+
normalizedStr = normalizedStr.replaceAll(normalizedSource, `${normalizedSource} ${synonyms}`);
|
|
48
52
|
}
|
|
49
53
|
}
|
|
50
54
|
// We remove the stopWords from the string
|
|
51
|
-
return (filterStopWords(
|
|
55
|
+
return (filterStopWords(normalizedStr, stopWords)
|
|
52
56
|
.match(wordRegex)
|
|
53
57
|
?.filter((w) => w.length >= minLength) ?? []);
|
|
54
58
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.ts"],"names":[],"mappings":";;;AAgGA,0CAUC;AAvGD;;;GAGG;AACI,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,OAAO,KAAK;QACX,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;QACtC,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AARU,QAAA,SAAS,aAQnB;AAEH;;GAEG;AACI,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,YAAY,CAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAEvC,OAAO,CACN,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAlBW,QAAA,aAAa,iBAkBxB;AAEF;;GAEG;AACI,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EAClC,SAAoB,EACnB,EAAE;IACH,IAAI,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;iBACpC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;iBACvC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,aAAa,GAAG,aAAa,CAAC,UAAU,CACvC,gBAAgB,EAChB,GAAG,gBAAgB,IAAI,QAAQ,EAAE,CACjC,CAAC;QACH,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,CACN,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC;SACvC,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAjCW,QAAA,aAAa,iBAiCxB;AAEF;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,OAAO,GAAG;SACR,WAAW,EAAE;SACb,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF;;GAEG;AACH,SAAgB,eAAe,CAAC,KAAa,EAAE,SAAoB;IAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC1D,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC"}
|
|
@@ -125,6 +125,11 @@ const mockSearchInfo = {
|
|
|
125
125
|
const result = (0, tokenizer_1.tokenizeIndex)('The car is fast', fieldInfo);
|
|
126
126
|
(0, vitest_1.expect)(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
|
|
127
127
|
});
|
|
128
|
+
(0, vitest_1.it)('should tokenize and apply synonyms regardless of case', () => {
|
|
129
|
+
const fieldInfo = mockSearchInfo.fields[0];
|
|
130
|
+
const result = (0, tokenizer_1.tokenizeIndex)('The Car is fast', fieldInfo);
|
|
131
|
+
(0, vitest_1.expect)(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
|
|
132
|
+
});
|
|
128
133
|
(0, vitest_1.it)('should normalize the input', () => {
|
|
129
134
|
const fieldInfo = mockSearchInfo.fields[0];
|
|
130
135
|
const result = (0, tokenizer_1.tokenizeIndex)('Élève Étudiant!', fieldInfo);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.spec.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.spec.ts"],"names":[],"mappings":";;AAAA,mCAA8C;AAC9C,2CAKqB;AAGrB,MAAM,cAAc,GAAe;IAClC,IAAI,EAAE,UAAU;IAChB,MAAM,EAAE;QACP;YACC,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC,QAAQ,CAAC;YACjB,QAAQ,EAAE;gBACT,GAAG,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;aAC9B;SACD;KACD;IACD,WAAW,EAAE;QACZ,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE;YACP,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,CAAC;SACN;KACD;CACD,CAAC;AAEF,IAAA,iBAAQ,EAAC,iBAAiB,EAAE,GAAG,EAAE;IAChC,IAAA,WAAE,EAAC,oDAAoD,EAAE,GAAG,EAAE;QAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yDAAyD,EAAE,GAAG,EAAE;QAClE,MAAM,KAAK,GAAG,2BAA2B,CAAC;QAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,oEAAoE,EAAE,GAAG,EAAE;QAC7E,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,CAAC,CAAC;QACtC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,wEAAwE,EAAE,GAAG,EAAE;QACjF,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,0DAA0D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,gBAAgB,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,sDAAsD,EAAE,GAAG,EAAE;QAC/D,MAAM,KAAK,GAAG,qBAAqB,CAAC;QACpC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,kCAAkC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,GAAG,EAAE;QAC9C,MAAM,KAAK,GAAG,+BAA+B,CAAC;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE;SACT,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;SACN,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QAClD,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,6BAA6B,EAAE,WAAW,CAAC,CAAC;QACzE,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,cAAc;SAC7C,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAA,WAAE,EAAC,kEAAkE,EAAE,GAAG,EAAE;QAC3E,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QAClD,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,6BAA6B,EAAE,SAAS,CAAC,CAAC;QACvE,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,IAAA,yBAAa,EAC3B,+BAA+B,EAC/B,SAAS,EACT,SAAS,CACT,CAAC;QACF,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,eAAe;QAEnF,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,WAAW,EAAE,GAAG,EAAE;IAC1B,IAAA,WAAE,EAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QACpD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4DAA4D,EAAE,GAAG,EAAE;QACrE,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,IAAA,eAAM,EAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAA,eAAM,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"tokenizer.spec.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.spec.ts"],"names":[],"mappings":";;AAAA,mCAA8C;AAC9C,2CAKqB;AAGrB,MAAM,cAAc,GAAe;IAClC,IAAI,EAAE,UAAU;IAChB,MAAM,EAAE;QACP;YACC,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC,QAAQ,CAAC;YACjB,QAAQ,EAAE;gBACT,GAAG,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;aAC9B;SACD;KACD;IACD,WAAW,EAAE;QACZ,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE;YACP,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,CAAC;SACN;KACD;CACD,CAAC;AAEF,IAAA,iBAAQ,EAAC,iBAAiB,EAAE,GAAG,EAAE;IAChC,IAAA,WAAE,EAAC,oDAAoD,EAAE,GAAG,EAAE;QAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yDAAyD,EAAE,GAAG,EAAE;QAClE,MAAM,KAAK,GAAG,2BAA2B,CAAC;QAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,oEAAoE,EAAE,GAAG,EAAE;QAC7E,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,CAAC,CAAC;QACtC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,wEAAwE,EAAE,GAAG,EAAE;QACjF,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,0DAA0D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,gBAAgB,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,sDAAsD,EAAE,GAAG,EAAE;QAC/D,MAAM,KAAK,GAAG,qBAAqB,CAAC;QACpC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,kCAAkC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,GAAG,EAAE;QAC9C,MAAM,KAAK,GAAG,+BAA+B,CAAC;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAA,2BAAe,EAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE;SACT,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;SACN,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QAClD,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,6BAA6B,EAAE,WAAW,CAAC,CAAC;QACzE,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,cAAc;SAC7C,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAA,WAAE,EAAC,kEAAkE,EAAE,GAAG,EAAE;QAC3E,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QAClD,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,6BAA6B,EAAE,SAAS,CAAC,CAAC;QACvE,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,IAAA,yBAAa,EAC3B,+BAA+B,EAC/B,SAAS,EACT,SAAS,CACT,CAAC;QACF,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,eAAe;QAEnF,MAAM,MAAM,GAAG,IAAA,yBAAa,EAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,WAAW,EAAE,GAAG,EAAE;IAC1B,IAAA,WAAE,EAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QACpD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4DAA4D,EAAE,GAAG,EAAE;QACrE,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,IAAA,qBAAS,EAAC,cAAc,CAAC,CAAC;QAE3C,IAAA,eAAM,EAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAA,eAAM,EAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|