@inseefr/lunatic 3.4.7 → 3.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/type.source.d.ts +25 -40
- package/esm/type.source.js +1 -0
- package/esm/type.source.js.map +1 -1
- package/esm/utils/search/SearchMiniSearch.spec.d.ts +1 -0
- package/esm/utils/search/SearchMiniSearch.spec.js +51 -0
- package/esm/utils/search/SearchMiniSearch.spec.js.map +1 -0
- package/esm/utils/search/melauto.js +1 -1
- package/esm/utils/search/melauto.spec.d.ts +1 -0
- package/esm/utils/search/melauto.spec.js +67 -0
- package/esm/utils/search/melauto.spec.js.map +1 -0
- package/esm/utils/search/tokenizer.d.ts +7 -2
- package/esm/utils/search/tokenizer.js +23 -8
- package/esm/utils/search/tokenizer.js.map +1 -1
- package/esm/utils/search/tokenizer.spec.d.ts +1 -0
- package/esm/utils/search/tokenizer.spec.js +160 -0
- package/esm/utils/search/tokenizer.spec.js.map +1 -0
- package/package.json +23 -1
- package/src/type.source.ts +93 -108
- package/src/utils/search/SearchMiniSearch.spec.ts +58 -0
- package/src/utils/search/melauto.spec.ts +75 -0
- package/src/utils/search/melauto.ts +1 -1
- package/src/utils/search/tokenizer.spec.ts +205 -0
- package/src/utils/search/tokenizer.ts +27 -8
- package/tsconfig.build.tsbuildinfo +1 -1
- package/type.source.d.ts +25 -40
- package/type.source.js +1 -0
- package/type.source.js.map +1 -1
- package/utils/search/SearchMiniSearch.spec.d.ts +1 -0
- package/utils/search/SearchMiniSearch.spec.js +51 -0
- package/utils/search/SearchMiniSearch.spec.js.map +1 -0
- package/utils/search/melauto.js +1 -1
- package/utils/search/melauto.spec.d.ts +1 -0
- package/utils/search/melauto.spec.js +69 -0
- package/utils/search/melauto.spec.js.map +1 -0
- package/utils/search/tokenizer.d.ts +7 -2
- package/utils/search/tokenizer.js +24 -8
- package/utils/search/tokenizer.js.map +1 -1
- package/utils/search/tokenizer.spec.d.ts +1 -0
- package/utils/search/tokenizer.spec.js +162 -0
- package/utils/search/tokenizer.spec.js.map +1 -0
package/esm/type.source.d.ts
CHANGED
|
@@ -9,20 +9,18 @@ export type ComponentDefinitionWithPage = ComponentDefinition & ({
|
|
|
9
9
|
goToPage: string;
|
|
10
10
|
});
|
|
11
11
|
export type ComponentDefinition = ComponentInputDefinition | ComponentSequenceDefinition | ComponentRoundaboutDefinition | ComponentLoopDefinition | ComponentRosterForLoopDefinition | ComponentTableDefinition | ComponentNumberDefinition | ComponentDurationDefinition | ComponentDatePickerDefinition | ComponentCheckboxGroupDefinition | ComponentCheckboxBooleanDefinition | ComponentRadioDefinition | ComponentDropdownDefinition | ComponentQuestionDefinition | ComponentCheckboxOneDefinition | ComponentSuggesterDefinition | ComponentPairWiseLinksDefinition | ComponentSummaryDefinition | ComponentText | ComponentAccordion;
|
|
12
|
-
export type ComponentInputDefinition =
|
|
12
|
+
export type ComponentInputDefinition = ComponentDefinitionBaseWithResponse & {
|
|
13
13
|
componentType: 'Input' | 'Textarea';
|
|
14
14
|
maxLength?: number;
|
|
15
15
|
};
|
|
16
|
-
export type ComponentInputDefinition1 = ComponentDefinitionBaseWithResponse;
|
|
17
16
|
export type ComponentDefinitionBaseWithResponse = ComponentDefinitionBase & {
|
|
18
17
|
response: ResponseDefinition;
|
|
19
18
|
};
|
|
20
|
-
export type ComponentSequenceDefinition =
|
|
19
|
+
export type ComponentSequenceDefinition = ComponentDefinitionBase & {
|
|
21
20
|
componentType: 'Sequence' | 'Subsequence';
|
|
22
21
|
goToPage?: string;
|
|
23
22
|
};
|
|
24
|
-
export type
|
|
25
|
-
export type ComponentRoundaboutDefinition = ComponentRoundaboutDefinition1 & {
|
|
23
|
+
export type ComponentRoundaboutDefinition = ComponentDefinitionBase & {
|
|
26
24
|
componentType: 'Roundabout';
|
|
27
25
|
iterations: VTLScalarExpression;
|
|
28
26
|
locked: boolean;
|
|
@@ -34,25 +32,22 @@ export type ComponentRoundaboutDefinition = ComponentRoundaboutDefinition1 & {
|
|
|
34
32
|
};
|
|
35
33
|
components: ComponentDefinitionWithPage[];
|
|
36
34
|
};
|
|
37
|
-
export type ComponentRoundaboutDefinition1 = ComponentDefinitionBase;
|
|
38
35
|
export type ComponentLoopDefinition = {
|
|
39
36
|
componentType: 'Loop';
|
|
40
37
|
loopDependencies?: string[];
|
|
41
38
|
} & ComponentLoopDefinition1;
|
|
42
39
|
export type ComponentLoopDefinition1 = PaginatedLoop | BlockLoop;
|
|
43
|
-
export type PaginatedLoop =
|
|
40
|
+
export type PaginatedLoop = ComponentDefinitionBase & {
|
|
44
41
|
components: ComponentDefinitionWithPage[];
|
|
45
42
|
iterations: VTLScalarExpression;
|
|
46
43
|
maxPage: string;
|
|
47
44
|
paginatedLoop: true;
|
|
48
45
|
};
|
|
49
|
-
export type
|
|
50
|
-
export type BlockLoop = BlockLoop1 & {
|
|
46
|
+
export type BlockLoop = ComponentDefinitionBase & {
|
|
51
47
|
paginatedLoop: false;
|
|
52
48
|
components: ComponentDefinition[];
|
|
53
|
-
} &
|
|
54
|
-
export type BlockLoop1 =
|
|
55
|
-
export type BlockLoop2 = {
|
|
49
|
+
} & BlockLoop1;
|
|
50
|
+
export type BlockLoop1 = {
|
|
56
51
|
lines: {
|
|
57
52
|
min: VTLExpression;
|
|
58
53
|
max: VTLExpression;
|
|
@@ -60,7 +55,7 @@ export type BlockLoop2 = {
|
|
|
60
55
|
} | {
|
|
61
56
|
iterations: VTLExpression;
|
|
62
57
|
};
|
|
63
|
-
export type ComponentRosterForLoopDefinition =
|
|
58
|
+
export type ComponentRosterForLoopDefinition = ComponentDefinitionBase & {
|
|
64
59
|
componentType: 'RosterForLoop';
|
|
65
60
|
components: ComponentDefinition[];
|
|
66
61
|
lines: {
|
|
@@ -69,7 +64,6 @@ export type ComponentRosterForLoopDefinition = ComponentRosterForLoopDefinition1
|
|
|
69
64
|
};
|
|
70
65
|
header?: TableHeader;
|
|
71
66
|
};
|
|
72
|
-
export type ComponentRosterForLoopDefinition1 = ComponentDefinitionBase;
|
|
73
67
|
export type Options = {
|
|
74
68
|
value: string | boolean;
|
|
75
69
|
label: VTLExpression;
|
|
@@ -83,35 +77,31 @@ export type TableHeader = {
|
|
|
83
77
|
rowspan?: number;
|
|
84
78
|
options?: Options;
|
|
85
79
|
}[];
|
|
86
|
-
export type ComponentTableDefinition =
|
|
80
|
+
export type ComponentTableDefinition = ComponentDefinitionBase & {
|
|
87
81
|
componentType: 'Table';
|
|
88
82
|
header?: TableHeader;
|
|
89
83
|
body: (ComponentDefinition | {
|
|
90
84
|
label: VTLExpression;
|
|
91
85
|
})[][];
|
|
92
86
|
};
|
|
93
|
-
export type
|
|
94
|
-
export type ComponentNumberDefinition = ComponentNumberDefinition1 & {
|
|
87
|
+
export type ComponentNumberDefinition = ComponentDefinitionBaseWithResponse & {
|
|
95
88
|
componentType: 'InputNumber';
|
|
96
89
|
unit?: string | VTLExpression;
|
|
97
90
|
min?: number;
|
|
98
91
|
max?: number;
|
|
99
92
|
decimals?: number;
|
|
100
93
|
};
|
|
101
|
-
export type
|
|
102
|
-
export type ComponentDurationDefinition = ComponentDurationDefinition1 & {
|
|
94
|
+
export type ComponentDurationDefinition = ComponentDefinitionBaseWithResponse & {
|
|
103
95
|
componentType: 'Duration';
|
|
104
96
|
format: 'PnYnM' | 'PTnHnM';
|
|
105
97
|
};
|
|
106
|
-
export type
|
|
107
|
-
export type ComponentDatePickerDefinition = ComponentDatePickerDefinition1 & {
|
|
98
|
+
export type ComponentDatePickerDefinition = ComponentDefinitionBaseWithResponse & {
|
|
108
99
|
componentType: 'Datepicker';
|
|
109
100
|
dateFormat: 'YYYY-MM-DD' | 'YYYY' | 'YYYY-MM';
|
|
110
101
|
min?: string;
|
|
111
102
|
max?: string;
|
|
112
103
|
};
|
|
113
|
-
export type
|
|
114
|
-
export type ComponentCheckboxGroupDefinition = ComponentCheckboxGroupDefinition1 & {
|
|
104
|
+
export type ComponentCheckboxGroupDefinition = ComponentDefinitionBase & {
|
|
115
105
|
componentType: 'CheckboxGroup';
|
|
116
106
|
orientation?: 'horizontal' | 'vertical';
|
|
117
107
|
responses: {
|
|
@@ -126,17 +116,14 @@ export type ComponentCheckboxGroupDefinition = ComponentCheckboxGroupDefinition1
|
|
|
126
116
|
};
|
|
127
117
|
}[];
|
|
128
118
|
};
|
|
129
|
-
export type
|
|
130
|
-
export type ComponentCheckboxBooleanDefinition = ComponentCheckboxBooleanDefinition1 & {
|
|
119
|
+
export type ComponentCheckboxBooleanDefinition = ComponentDefinitionBaseWithResponse & {
|
|
131
120
|
componentType: 'CheckboxBoolean';
|
|
132
121
|
};
|
|
133
|
-
export type
|
|
134
|
-
export type ComponentRadioDefinition = ComponentRadioDefinition1 & {
|
|
122
|
+
export type ComponentRadioDefinition = ComponentDefinitionBaseWithResponse & {
|
|
135
123
|
componentType: 'Radio';
|
|
136
124
|
orientation?: 'horizontal' | 'vertical';
|
|
137
125
|
options: OptionsWithDetail;
|
|
138
126
|
};
|
|
139
|
-
export type ComponentRadioDefinition1 = ComponentDefinitionBaseWithResponse;
|
|
140
127
|
export type OptionsWithDetail = {
|
|
141
128
|
value: string | boolean;
|
|
142
129
|
label: VTLExpression;
|
|
@@ -148,22 +135,19 @@ export type OptionsWithDetail = {
|
|
|
148
135
|
};
|
|
149
136
|
};
|
|
150
137
|
}[];
|
|
151
|
-
export type ComponentDropdownDefinition =
|
|
138
|
+
export type ComponentDropdownDefinition = ComponentDefinitionBaseWithResponse & {
|
|
152
139
|
componentType: 'Dropdown';
|
|
153
140
|
options: Options;
|
|
154
141
|
};
|
|
155
|
-
export type
|
|
156
|
-
export type ComponentQuestionDefinition = ComponentQuestionDefinition1 & {
|
|
142
|
+
export type ComponentQuestionDefinition = ComponentDefinitionBase & {
|
|
157
143
|
componentType: 'Question';
|
|
158
144
|
components: ComponentDefinition[];
|
|
159
145
|
};
|
|
160
|
-
export type
|
|
161
|
-
export type ComponentCheckboxOneDefinition = ComponentCheckboxOneDefinition1 & {
|
|
146
|
+
export type ComponentCheckboxOneDefinition = ComponentDefinitionBaseWithResponse & {
|
|
162
147
|
componentType: 'CheckboxOne';
|
|
163
148
|
options: OptionsWithDetail;
|
|
164
149
|
};
|
|
165
|
-
export type
|
|
166
|
-
export type ComponentSuggesterDefinition = ComponentSuggesterDefinition1 & {
|
|
150
|
+
export type ComponentSuggesterDefinition = ComponentDefinitionBaseWithResponse & {
|
|
167
151
|
componentType: 'Suggester';
|
|
168
152
|
/**
|
|
169
153
|
* Nom / Index du référentiel à utiliser
|
|
@@ -189,8 +173,7 @@ export type ComponentSuggesterDefinition = ComponentSuggesterDefinition1 & {
|
|
|
189
173
|
attribute: string;
|
|
190
174
|
}[];
|
|
191
175
|
};
|
|
192
|
-
export type
|
|
193
|
-
export type ComponentPairWiseLinksDefinition = ComponentPairWiseLinksDefinition1 & {
|
|
176
|
+
export type ComponentPairWiseLinksDefinition = ComponentDefinitionBase & {
|
|
194
177
|
componentType: 'PairwiseLinks';
|
|
195
178
|
xAxisIterations: VTLScalarExpression;
|
|
196
179
|
yAxisIterations: VTLScalarExpression;
|
|
@@ -201,8 +184,7 @@ export type ComponentPairWiseLinksDefinition = ComponentPairWiseLinksDefinition1
|
|
|
201
184
|
};
|
|
202
185
|
components: ComponentDefinition[];
|
|
203
186
|
};
|
|
204
|
-
export type
|
|
205
|
-
export type ComponentSummaryDefinition = ComponentSummaryDefinition1 & {
|
|
187
|
+
export type ComponentSummaryDefinition = ComponentDefinitionBase & {
|
|
206
188
|
componentType: 'Summary';
|
|
207
189
|
sections: {
|
|
208
190
|
id: string;
|
|
@@ -215,7 +197,6 @@ export type ComponentSummaryDefinition = ComponentSummaryDefinition1 & {
|
|
|
215
197
|
}[];
|
|
216
198
|
}[];
|
|
217
199
|
};
|
|
218
|
-
export type ComponentSummaryDefinition1 = ComponentDefinitionBase;
|
|
219
200
|
export type Variable = {
|
|
220
201
|
variableType: 'EXTERNAL';
|
|
221
202
|
name: string;
|
|
@@ -405,4 +386,8 @@ export type SuggesterDefinition = {
|
|
|
405
386
|
} | {
|
|
406
387
|
type: 'soft';
|
|
407
388
|
};
|
|
389
|
+
/**
|
|
390
|
+
* list of words to exclude from the searching
|
|
391
|
+
*/
|
|
392
|
+
stopWords?: string[];
|
|
408
393
|
};
|
package/esm/type.source.js
CHANGED
package/esm/type.source.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"type.source.js","sourceRoot":"","sources":["../src/type.source.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
1
|
+
{"version":3,"file":"type.source.js","sourceRoot":"","sources":["../src/type.source.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB;;;;GAIG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeAll, afterEach } from 'vitest';
|
|
2
|
+
import { SearchMinisearch } from './SearchMinisearch';
|
|
3
|
+
vi.mock('minisearch', () => {
|
|
4
|
+
return {
|
|
5
|
+
default: vi.fn().mockImplementation(() => ({
|
|
6
|
+
addAll: vi.fn(),
|
|
7
|
+
search: vi.fn(),
|
|
8
|
+
})),
|
|
9
|
+
};
|
|
10
|
+
});
|
|
11
|
+
vi.mock('./melauto', () => ({
|
|
12
|
+
applyMelauto: vi.fn(),
|
|
13
|
+
}));
|
|
14
|
+
describe('SearchMinisearch', () => {
|
|
15
|
+
let searchInstance;
|
|
16
|
+
const mockData = [
|
|
17
|
+
{ id: '1', label: 'First Item' },
|
|
18
|
+
{ id: '2', label: 'Second Item' },
|
|
19
|
+
];
|
|
20
|
+
beforeAll(() => {
|
|
21
|
+
searchInstance = new SearchMinisearch({
|
|
22
|
+
name: 'test-suggester',
|
|
23
|
+
fields: [{ name: 'id' }, { name: 'label' }],
|
|
24
|
+
queryParser: {
|
|
25
|
+
type: 'tokenized',
|
|
26
|
+
params: { language: 'English', pattern: '\\w+', min: 1 },
|
|
27
|
+
},
|
|
28
|
+
max: 10,
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
const miniSearchMock = searchInstance.db;
|
|
33
|
+
miniSearchMock.addAll.mockClear();
|
|
34
|
+
});
|
|
35
|
+
it('should initialize and index data correctly', async () => {
|
|
36
|
+
var _a;
|
|
37
|
+
await searchInstance.index(mockData);
|
|
38
|
+
// Check if MiniSearch instance was created and indexed
|
|
39
|
+
expect(searchInstance.db).not.toBeNull();
|
|
40
|
+
expect(searchInstance.isIndexed()).toBe(true);
|
|
41
|
+
// Check if addAll was called with the correct data
|
|
42
|
+
expect((_a = searchInstance.db) === null || _a === void 0 ? void 0 : _a.addAll).toHaveBeenCalledWith(mockData);
|
|
43
|
+
});
|
|
44
|
+
it('should not re-index if already indexed', async () => {
|
|
45
|
+
var _a;
|
|
46
|
+
searchInstance.indexed = true;
|
|
47
|
+
await searchInstance.index(mockData);
|
|
48
|
+
expect((_a = searchInstance.db) === null || _a === void 0 ? void 0 : _a.addAll).not.toHaveBeenCalled();
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
//# sourceMappingURL=SearchMiniSearch.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SearchMiniSearch.spec.js","sourceRoot":"","sources":["../../../src/utils/search/SearchMiniSearch.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,OAAO;QACN,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;YACf,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;SACf,CAAC,CAAC;KACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3B,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;CACrB,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,cAAqC,CAAC;IAC1C,MAAM,QAAQ,GAAG;QAChB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE;QAChC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;KACjC,CAAC;IAEF,SAAS,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,IAAI,gBAAgB,CAAC;YACrC,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YAC3C,WAAW,EAAE;gBACZ,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;aACxD;YACD,GAAG,EAAE,EAAE;SACP,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,cAAc,GAAG,cAAc,CAAC,EAAS,CAAC;QAChD,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;;QAC3D,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,uDAAuD;QACvD,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,mDAAmD;QACnD,MAAM,CAAC,MAAA,cAAc,CAAC,EAAE,0CAAE,MAAM,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;;QACvD,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC;QAC9B,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,MAAM,CAAC,MAAA,cAAc,CAAC,EAAE,0CAAE,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAC1D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
export function applyMelauto(query, data) {
|
|
5
5
|
return data.sort((a, b) => {
|
|
6
6
|
var _a, _b;
|
|
7
|
-
return melautoScore((_a = b.label) !== null && _a !== void 0 ? _a :
|
|
7
|
+
return melautoScore((_a = b.label) !== null && _a !== void 0 ? _a : b.id, query) -
|
|
8
8
|
melautoScore((_b = a.label) !== null && _b !== void 0 ? _b : a.id, query);
|
|
9
9
|
});
|
|
10
10
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { applyMelauto, melautoScore } from './melauto';
|
|
3
|
+
// Mock data to test the sorting and scoring
|
|
4
|
+
const data = [
|
|
5
|
+
{ id: '1', label: 'Hello world' },
|
|
6
|
+
{ id: '2', label: 'Bonjour le monde' },
|
|
7
|
+
{ id: '3', label: 'Hello everyone' },
|
|
8
|
+
{ id: '4', label: 'Greetings planet' },
|
|
9
|
+
];
|
|
10
|
+
describe('applyMelauto', () => {
|
|
11
|
+
it('should sort data by relevance to the query', () => {
|
|
12
|
+
const sortedData = applyMelauto('hello', data);
|
|
13
|
+
const expectedSortedData = [
|
|
14
|
+
{ id: '1', label: 'Hello world' },
|
|
15
|
+
{ id: '3', label: 'Hello everyone' },
|
|
16
|
+
{ id: '2', label: 'Bonjour le monde' },
|
|
17
|
+
{ id: '4', label: 'Greetings planet' },
|
|
18
|
+
];
|
|
19
|
+
expect(sortedData).toStrictEqual(expectedSortedData);
|
|
20
|
+
});
|
|
21
|
+
it('should return data in original order if query is empty', () => {
|
|
22
|
+
const sortedData = applyMelauto('', data);
|
|
23
|
+
expect(sortedData).toEqual(data);
|
|
24
|
+
});
|
|
25
|
+
it('should handle data without labels by using id instead', () => {
|
|
26
|
+
const noLabelData = [
|
|
27
|
+
{ id: 'a' },
|
|
28
|
+
{ id: 'b', label: 'Hello' },
|
|
29
|
+
{ id: 'hello-w' },
|
|
30
|
+
];
|
|
31
|
+
const sortedData = applyMelauto('hello', noLabelData);
|
|
32
|
+
const expectedSortedData = [
|
|
33
|
+
{ id: 'b', label: 'Hello' },
|
|
34
|
+
{ id: 'hello-w' },
|
|
35
|
+
{ id: 'a' },
|
|
36
|
+
];
|
|
37
|
+
expect(sortedData).toStrictEqual(expectedSortedData);
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
describe('melautoScore', () => {
|
|
41
|
+
it('should return a non-null score', () => {
|
|
42
|
+
const score = melautoScore('Hello world', 'hello');
|
|
43
|
+
const expectedScore = 1;
|
|
44
|
+
expect(score).toBeCloseTo(expectedScore, 2);
|
|
45
|
+
});
|
|
46
|
+
it('should give a higher score for a closer match', () => {
|
|
47
|
+
const score1 = melautoScore('Hello beautiful world', 'hello world');
|
|
48
|
+
const score2 = melautoScore('Hello beautiful world', 'hello');
|
|
49
|
+
expect(score1).toBeGreaterThan(score2);
|
|
50
|
+
});
|
|
51
|
+
it('should give the same score for a same query if the comparison string is longer', () => {
|
|
52
|
+
const score1 = melautoScore('Hello world', 'hello');
|
|
53
|
+
const score2 = melautoScore('Hello beautiful world', 'hello');
|
|
54
|
+
expect(score1).toBeCloseTo(score2, 2);
|
|
55
|
+
});
|
|
56
|
+
it('should calculate a null score', () => {
|
|
57
|
+
const score = melautoScore('Greetings planet', 'hello');
|
|
58
|
+
const expectedScore = 0;
|
|
59
|
+
expect(score).toBe(expectedScore);
|
|
60
|
+
});
|
|
61
|
+
it('should handle accent and special characters correctly', () => {
|
|
62
|
+
const score1 = melautoScore('Héllo wörld', 'hello world');
|
|
63
|
+
const score2 = melautoScore('Hello world', 'héllo-wOrld');
|
|
64
|
+
expect(score1).toBeCloseTo(score2, 2);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
//# sourceMappingURL=melauto.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"melauto.spec.js","sourceRoot":"","sources":["../../../src/utils/search/melauto.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEvD,4CAA4C;AAC5C,MAAM,IAAI,GAAG;IACZ,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;IACjC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;IACtC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,gBAAgB,EAAE;IACpC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;CACtC,CAAC;AAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACrD,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,kBAAkB,GAAG;YAC1B,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;YACjC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,gBAAgB,EAAE;YACpC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;YACtC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;SACtC,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QACjE,MAAM,UAAU,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,WAAW,GAAG;YACnB,EAAE,EAAE,EAAE,GAAG,EAAE;YACX,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE;YAC3B,EAAE,EAAE,EAAE,SAAS,EAAE;SACjB,CAAC;QACF,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAG;YAC1B,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE;YAC3B,EAAE,EAAE,EAAE,SAAS,EAAE;YACjB,EAAE,EAAE,EAAE,GAAG,EAAE;SACX,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACzC,MAAM,KAAK,GAAG,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,aAAa,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,GAAG,EAAE;QACzF,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACxC,MAAM,KAAK,GAAG,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { SearchInfo } from './SearchInterface';
|
|
2
2
|
import type { ItemOf } from '../../type.utils';
|
|
3
3
|
/**
|
|
4
|
-
* Generates a tokenize method
|
|
4
|
+
* Generates a tokenize method.
|
|
5
|
+
* When used for tokenizing a search query instead of the indexing, the fieldName is undefined.
|
|
5
6
|
*/
|
|
6
7
|
export declare const tokenizer: (info: SearchInfo) => (str: string, fieldName?: string) => string[];
|
|
7
8
|
/**
|
|
@@ -11,4 +12,8 @@ export declare const tokenizeQuery: (str: string, info: SearchInfo["queryParser"
|
|
|
11
12
|
/**
|
|
12
13
|
* Tokenizer used for indexing (based on "fields" specification)
|
|
13
14
|
*/
|
|
14
|
-
export declare const tokenizeIndex: (str: string, info: ItemOf<SearchInfo["fields"]
|
|
15
|
+
export declare const tokenizeIndex: (str: string, info: ItemOf<SearchInfo["fields"]>, stopWords?: string[]) => string[];
|
|
16
|
+
/**
|
|
17
|
+
* remove from a string all the words that are included in a stopwords list
|
|
18
|
+
*/
|
|
19
|
+
export declare function filterStopWords(input: string, stopWords?: string[]): string;
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Generates a tokenize method
|
|
2
|
+
* Generates a tokenize method.
|
|
3
|
+
* When used for tokenizing a search query instead of the indexing, the fieldName is undefined.
|
|
3
4
|
*/
|
|
4
5
|
export const tokenizer = (info) => (str, fieldName) => {
|
|
5
6
|
const field = info.fields.find((f) => f.name === fieldName);
|
|
7
|
+
const stopWords = info.stopWords;
|
|
6
8
|
return field
|
|
7
|
-
? tokenizeIndex(str, field)
|
|
9
|
+
? tokenizeIndex(str, field, stopWords)
|
|
8
10
|
: tokenizeQuery(str, info.queryParser);
|
|
9
11
|
};
|
|
10
12
|
/**
|
|
@@ -18,8 +20,8 @@ export const tokenizeQuery = (str, info) => {
|
|
|
18
20
|
.filter((w) => w.length > 0);
|
|
19
21
|
}
|
|
20
22
|
const wordRegex = info.params.pattern && info.params.pattern !== 'soft'
|
|
21
|
-
?
|
|
22
|
-
:
|
|
23
|
+
? new RegExp(info.params.pattern, 'gi')
|
|
24
|
+
: /\w+/gi;
|
|
23
25
|
const minLength = (_a = info.params.min) !== null && _a !== void 0 ? _a : 1;
|
|
24
26
|
return ((_c = (_b = normalizeStr(str)
|
|
25
27
|
.match(wordRegex)) === null || _b === void 0 ? void 0 : _b.filter((w) => w.length >= minLength)) !== null && _c !== void 0 ? _c : []);
|
|
@@ -27,11 +29,11 @@ export const tokenizeQuery = (str, info) => {
|
|
|
27
29
|
/**
|
|
28
30
|
* Tokenizer used for indexing (based on "fields" specification)
|
|
29
31
|
*/
|
|
30
|
-
export const tokenizeIndex = (str, info) => {
|
|
32
|
+
export const tokenizeIndex = (str, info, stopWords) => {
|
|
31
33
|
var _a, _b, _c;
|
|
32
34
|
const wordRegex = info.rules && info.rules !== 'soft'
|
|
33
|
-
?
|
|
34
|
-
:
|
|
35
|
+
? new RegExp(info.rules[0], 'gi')
|
|
36
|
+
: /\w+/gi;
|
|
35
37
|
const minLength = (_a = info.min) !== null && _a !== void 0 ? _a : 1;
|
|
36
38
|
// For synonyms, add the synonyms to the string
|
|
37
39
|
if (info.synonyms) {
|
|
@@ -40,7 +42,8 @@ export const tokenizeIndex = (str, info) => {
|
|
|
40
42
|
str = str.replaceAll(source, `${source} ${synonyms}`);
|
|
41
43
|
}
|
|
42
44
|
}
|
|
43
|
-
|
|
45
|
+
// We remove the stopWords from the string
|
|
46
|
+
return ((_c = (_b = filterStopWords(normalizeStr(str), stopWords)
|
|
44
47
|
.match(wordRegex)) === null || _b === void 0 ? void 0 : _b.filter((w) => w.length >= minLength)) !== null && _c !== void 0 ? _c : []);
|
|
45
48
|
};
|
|
46
49
|
/**
|
|
@@ -54,4 +57,16 @@ const normalizeStr = (str) => {
|
|
|
54
57
|
.replace(/[\u0300-\u036f]/g, '')
|
|
55
58
|
.toLowerCase();
|
|
56
59
|
};
|
|
60
|
+
/**
|
|
61
|
+
* remove from a string all the words that are included in a stopwords list
|
|
62
|
+
*/
|
|
63
|
+
export function filterStopWords(input, stopWords) {
|
|
64
|
+
if (!stopWords) {
|
|
65
|
+
return input;
|
|
66
|
+
}
|
|
67
|
+
const lowerCaseStopWords = stopWords.map((word) => word.toLowerCase());
|
|
68
|
+
const words = input.split(/\s+/);
|
|
69
|
+
const filteredWords = words.filter((word) => !lowerCaseStopWords.includes(word.toLowerCase()));
|
|
70
|
+
return filteredWords.join(' ');
|
|
71
|
+
}
|
|
57
72
|
//# sourceMappingURL=tokenizer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.ts"],"names":[],"mappings":"AAGA
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,OAAO,KAAK;QACX,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;QACtC,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,YAAY,CAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEvC,OAAO,CACN,MAAA,MAAA,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EAClC,SAAoB,EACnB,EAAE;;IACH,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;QACvD,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,CACN,MAAA,MAAA,eAAe,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC;SAC3C,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,OAAO,GAAG;SACR,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,WAAW,EAAE,CAAC;AACjB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,SAAoB;IAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC1D,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { tokenizer, tokenizeQuery, tokenizeIndex, filterStopWords, } from './tokenizer';
|
|
3
|
+
const mockSearchInfo = {
|
|
4
|
+
name: 'Products',
|
|
5
|
+
fields: [
|
|
6
|
+
{
|
|
7
|
+
name: 'title',
|
|
8
|
+
min: 3,
|
|
9
|
+
rules: ['[\\w]+'],
|
|
10
|
+
synonyms: {
|
|
11
|
+
car: ['vehicle', 'automobile'],
|
|
12
|
+
},
|
|
13
|
+
},
|
|
14
|
+
],
|
|
15
|
+
queryParser: {
|
|
16
|
+
type: 'tokenized',
|
|
17
|
+
params: {
|
|
18
|
+
language: 'English',
|
|
19
|
+
pattern: '[\\w.]+',
|
|
20
|
+
min: 2,
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
describe('filterStopWords', () => {
|
|
25
|
+
it('should remove only stopwords from the input string', () => {
|
|
26
|
+
const input = 'This is a test.';
|
|
27
|
+
const stopWords = ['is', 'a'];
|
|
28
|
+
const result = filterStopWords(input, stopWords);
|
|
29
|
+
expect(result).toBe('This test.');
|
|
30
|
+
});
|
|
31
|
+
it('should not alter words that are substrings of stopwords', () => {
|
|
32
|
+
const input = 'this is a testing example';
|
|
33
|
+
const stopWords = ['test'];
|
|
34
|
+
const result = filterStopWords(input, stopWords);
|
|
35
|
+
expect(result).toBe('this is a testing example');
|
|
36
|
+
});
|
|
37
|
+
it('should be case-insensitive', () => {
|
|
38
|
+
const input = 'This Is A Test.';
|
|
39
|
+
const stopWords = ['is', 'a'];
|
|
40
|
+
const result = filterStopWords(input, stopWords);
|
|
41
|
+
expect(result).toBe('This Test.');
|
|
42
|
+
});
|
|
43
|
+
it('should return the input string unchanged if stopWords is undefined', () => {
|
|
44
|
+
const input = 'This is a test.';
|
|
45
|
+
const result = filterStopWords(input);
|
|
46
|
+
expect(result).toBe(input);
|
|
47
|
+
});
|
|
48
|
+
it('should return the input string unchanged if stopWords is a empty array', () => {
|
|
49
|
+
const input = 'This is a test.';
|
|
50
|
+
const stopWords = [];
|
|
51
|
+
const result = filterStopWords(input, stopWords);
|
|
52
|
+
expect(result).toBe(input);
|
|
53
|
+
});
|
|
54
|
+
it('should return an empty string if all words are stopwords', () => {
|
|
55
|
+
const input = 'this is a test';
|
|
56
|
+
const stopWords = ['this', 'is', 'a', 'test'];
|
|
57
|
+
const result = filterStopWords(input, stopWords);
|
|
58
|
+
expect(result).toBe('');
|
|
59
|
+
});
|
|
60
|
+
it('should handle strings with multiple spaces correctly', () => {
|
|
61
|
+
const input = 'This is a test.';
|
|
62
|
+
const stopWords = ['is', 'a'];
|
|
63
|
+
const result = filterStopWords(input, stopWords);
|
|
64
|
+
expect(result).toBe('This test.');
|
|
65
|
+
});
|
|
66
|
+
it('should handle empty input string', () => {
|
|
67
|
+
const input = '';
|
|
68
|
+
const stopWords = ['is', 'a'];
|
|
69
|
+
const result = filterStopWords(input, stopWords);
|
|
70
|
+
expect(result).toBe('');
|
|
71
|
+
});
|
|
72
|
+
it('should handle punctuation correctly', () => {
|
|
73
|
+
const input = 'Hello, world! This is a test.';
|
|
74
|
+
const stopWords = ['is', 'a'];
|
|
75
|
+
const result = filterStopWords(input, stopWords);
|
|
76
|
+
expect(result).toBe('Hello, world! This test.');
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
describe('tokenizeQuery', () => {
|
|
80
|
+
it('should tokenize based on soft type', () => {
|
|
81
|
+
const queryParser = { type: 'soft' };
|
|
82
|
+
const result = tokenizeQuery('This is a test!', queryParser);
|
|
83
|
+
expect(result).toEqual(['this', 'is', 'a', 'test']);
|
|
84
|
+
});
|
|
85
|
+
it('should tokenize with a custom regex and a min', () => {
|
|
86
|
+
const queryParser = {
|
|
87
|
+
type: 'tokenized',
|
|
88
|
+
params: { pattern: '[\\w.]+', min: 3 },
|
|
89
|
+
};
|
|
90
|
+
const result = tokenizeQuery('This is a test !', queryParser);
|
|
91
|
+
expect(result).toEqual(['this', 'test']);
|
|
92
|
+
});
|
|
93
|
+
it('should normalize the input', () => {
|
|
94
|
+
const queryParser = {
|
|
95
|
+
type: 'tokenized',
|
|
96
|
+
params: { pattern: '\\w+', min: 1 },
|
|
97
|
+
};
|
|
98
|
+
const result = tokenizeQuery('Élève Étudiant!', queryParser);
|
|
99
|
+
expect(result).toEqual(['eleve', 'etudiant']);
|
|
100
|
+
});
|
|
101
|
+
it('should return an empty array for unmatched patterns', () => {
|
|
102
|
+
const queryParser = {
|
|
103
|
+
type: 'tokenized',
|
|
104
|
+
params: { language: 'French', pattern: '[\\d.]+', min: 1 }, // only digits
|
|
105
|
+
};
|
|
106
|
+
const result = tokenizeQuery('No numbers here!', queryParser);
|
|
107
|
+
expect(result).toEqual([]);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
describe('tokenizeIndex', () => {
|
|
111
|
+
it('should filter out words shorter than the required minimum length', () => {
|
|
112
|
+
const fieldInfo = mockSearchInfo.fields[0];
|
|
113
|
+
const result = tokenizeIndex('The bus is so slow', fieldInfo);
|
|
114
|
+
expect(result).toEqual(['the', 'bus', 'slow']);
|
|
115
|
+
});
|
|
116
|
+
it('should tokenize and apply synonyms', () => {
|
|
117
|
+
const fieldInfo = mockSearchInfo.fields[0];
|
|
118
|
+
const result = tokenizeIndex('The car is fast', fieldInfo);
|
|
119
|
+
expect(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
|
|
120
|
+
});
|
|
121
|
+
it('should normalize the input', () => {
|
|
122
|
+
const fieldInfo = mockSearchInfo.fields[0];
|
|
123
|
+
const result = tokenizeIndex('Élève Étudiant!', fieldInfo);
|
|
124
|
+
expect(result).toEqual(['eleve', 'etudiant']);
|
|
125
|
+
});
|
|
126
|
+
it('should filter out stopWords', () => {
|
|
127
|
+
const fieldInfo = { ...mockSearchInfo.fields[0], min: 1 };
|
|
128
|
+
const stopWords = ['is', 'the', 'of', 'this', 'a'];
|
|
129
|
+
const result = tokenizeIndex('This is a test of stopWords !', fieldInfo, stopWords);
|
|
130
|
+
expect(result).toEqual(['test', 'stopwords']);
|
|
131
|
+
});
|
|
132
|
+
it('should return an empty array for unmatched patterns', () => {
|
|
133
|
+
const fieldInfo = { ...mockSearchInfo.fields[0], rules: ['\\d+'] }; // Only numbers
|
|
134
|
+
const result = tokenizeIndex('No numbers here', fieldInfo);
|
|
135
|
+
expect(result).toEqual([]);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
describe('tokenizer', () => {
|
|
139
|
+
it('should tokenize using field rules', () => {
|
|
140
|
+
const tokenize = tokenizer(mockSearchInfo);
|
|
141
|
+
const result = tokenize('The car is fast', 'title');
|
|
142
|
+
expect(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
|
|
143
|
+
});
|
|
144
|
+
it('should tokenize using query parser when field is not found', () => {
|
|
145
|
+
const tokenize = tokenizer(mockSearchInfo);
|
|
146
|
+
const result = tokenize('This is a test!');
|
|
147
|
+
expect(result).toEqual(['this', 'is', 'test']);
|
|
148
|
+
});
|
|
149
|
+
it('should normalize the input', () => {
|
|
150
|
+
const tokenize = tokenizer(mockSearchInfo);
|
|
151
|
+
const result = tokenize('Élève Étudiant!');
|
|
152
|
+
expect(result).toEqual(['eleve', 'etudiant']);
|
|
153
|
+
});
|
|
154
|
+
it('should handle empty strings', () => {
|
|
155
|
+
const tokenize = tokenizer(mockSearchInfo);
|
|
156
|
+
expect(tokenize('', 'title')).toEqual([]);
|
|
157
|
+
expect(tokenize('')).toEqual([]);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
//# sourceMappingURL=tokenizer.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.spec.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACN,SAAS,EACT,aAAa,EACb,aAAa,EACb,eAAe,GACf,MAAM,aAAa,CAAC;AAGrB,MAAM,cAAc,GAAe;IAClC,IAAI,EAAE,UAAU;IAChB,MAAM,EAAE;QACP;YACC,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC,QAAQ,CAAC;YACjB,QAAQ,EAAE;gBACT,GAAG,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;aAC9B;SACD;KACD;IACD,WAAW,EAAE;QACZ,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE;YACP,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,CAAC;SACN;KACD;CACD,CAAC;AAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QAClE,MAAM,KAAK,GAAG,2BAA2B,CAAC;QAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC7E,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QACjF,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,gBAAgB,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC/D,MAAM,KAAK,GAAG,qBAAqB,CAAC;QACpC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC9C,MAAM,KAAK,GAAG,+BAA+B,CAAC;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE;SACT,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;SACN,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,cAAc;SAC7C,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC3E,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,aAAa,CAC3B,+BAA+B,EAC/B,SAAS,EACT,SAAS,CACT,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,eAAe;QAEnF,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|