@inseefr/lunatic 3.4.7 → 3.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/esm/type.source.d.ts +25 -40
  2. package/esm/type.source.js +1 -0
  3. package/esm/type.source.js.map +1 -1
  4. package/esm/utils/search/SearchMiniSearch.spec.d.ts +1 -0
  5. package/esm/utils/search/SearchMiniSearch.spec.js +51 -0
  6. package/esm/utils/search/SearchMiniSearch.spec.js.map +1 -0
  7. package/esm/utils/search/melauto.js +1 -1
  8. package/esm/utils/search/melauto.spec.d.ts +1 -0
  9. package/esm/utils/search/melauto.spec.js +67 -0
  10. package/esm/utils/search/melauto.spec.js.map +1 -0
  11. package/esm/utils/search/tokenizer.d.ts +7 -2
  12. package/esm/utils/search/tokenizer.js +23 -8
  13. package/esm/utils/search/tokenizer.js.map +1 -1
  14. package/esm/utils/search/tokenizer.spec.d.ts +1 -0
  15. package/esm/utils/search/tokenizer.spec.js +160 -0
  16. package/esm/utils/search/tokenizer.spec.js.map +1 -0
  17. package/package.json +23 -1
  18. package/src/type.source.ts +93 -108
  19. package/src/utils/search/SearchMiniSearch.spec.ts +58 -0
  20. package/src/utils/search/melauto.spec.ts +75 -0
  21. package/src/utils/search/melauto.ts +1 -1
  22. package/src/utils/search/tokenizer.spec.ts +205 -0
  23. package/src/utils/search/tokenizer.ts +27 -8
  24. package/tsconfig.build.tsbuildinfo +1 -1
  25. package/type.source.d.ts +25 -40
  26. package/type.source.js +1 -0
  27. package/type.source.js.map +1 -1
  28. package/utils/search/SearchMiniSearch.spec.d.ts +1 -0
  29. package/utils/search/SearchMiniSearch.spec.js +51 -0
  30. package/utils/search/SearchMiniSearch.spec.js.map +1 -0
  31. package/utils/search/melauto.js +1 -1
  32. package/utils/search/melauto.spec.d.ts +1 -0
  33. package/utils/search/melauto.spec.js +69 -0
  34. package/utils/search/melauto.spec.js.map +1 -0
  35. package/utils/search/tokenizer.d.ts +7 -2
  36. package/utils/search/tokenizer.js +24 -8
  37. package/utils/search/tokenizer.js.map +1 -1
  38. package/utils/search/tokenizer.spec.d.ts +1 -0
  39. package/utils/search/tokenizer.spec.js +162 -0
  40. package/utils/search/tokenizer.spec.js.map +1 -0
@@ -9,20 +9,18 @@ export type ComponentDefinitionWithPage = ComponentDefinition & ({
9
9
  goToPage: string;
10
10
  });
11
11
  export type ComponentDefinition = ComponentInputDefinition | ComponentSequenceDefinition | ComponentRoundaboutDefinition | ComponentLoopDefinition | ComponentRosterForLoopDefinition | ComponentTableDefinition | ComponentNumberDefinition | ComponentDurationDefinition | ComponentDatePickerDefinition | ComponentCheckboxGroupDefinition | ComponentCheckboxBooleanDefinition | ComponentRadioDefinition | ComponentDropdownDefinition | ComponentQuestionDefinition | ComponentCheckboxOneDefinition | ComponentSuggesterDefinition | ComponentPairWiseLinksDefinition | ComponentSummaryDefinition | ComponentText | ComponentAccordion;
12
- export type ComponentInputDefinition = ComponentInputDefinition1 & {
12
+ export type ComponentInputDefinition = ComponentDefinitionBaseWithResponse & {
13
13
  componentType: 'Input' | 'Textarea';
14
14
  maxLength?: number;
15
15
  };
16
- export type ComponentInputDefinition1 = ComponentDefinitionBaseWithResponse;
17
16
  export type ComponentDefinitionBaseWithResponse = ComponentDefinitionBase & {
18
17
  response: ResponseDefinition;
19
18
  };
20
- export type ComponentSequenceDefinition = ComponentSequenceDefinition1 & {
19
+ export type ComponentSequenceDefinition = ComponentDefinitionBase & {
21
20
  componentType: 'Sequence' | 'Subsequence';
22
21
  goToPage?: string;
23
22
  };
24
- export type ComponentSequenceDefinition1 = ComponentDefinitionBase;
25
- export type ComponentRoundaboutDefinition = ComponentRoundaboutDefinition1 & {
23
+ export type ComponentRoundaboutDefinition = ComponentDefinitionBase & {
26
24
  componentType: 'Roundabout';
27
25
  iterations: VTLScalarExpression;
28
26
  locked: boolean;
@@ -34,25 +32,22 @@ export type ComponentRoundaboutDefinition = ComponentRoundaboutDefinition1 & {
34
32
  };
35
33
  components: ComponentDefinitionWithPage[];
36
34
  };
37
- export type ComponentRoundaboutDefinition1 = ComponentDefinitionBase;
38
35
  export type ComponentLoopDefinition = {
39
36
  componentType: 'Loop';
40
37
  loopDependencies?: string[];
41
38
  } & ComponentLoopDefinition1;
42
39
  export type ComponentLoopDefinition1 = PaginatedLoop | BlockLoop;
43
- export type PaginatedLoop = PaginatedLoop1 & {
40
+ export type PaginatedLoop = ComponentDefinitionBase & {
44
41
  components: ComponentDefinitionWithPage[];
45
42
  iterations: VTLScalarExpression;
46
43
  maxPage: string;
47
44
  paginatedLoop: true;
48
45
  };
49
- export type PaginatedLoop1 = ComponentDefinitionBase;
50
- export type BlockLoop = BlockLoop1 & {
46
+ export type BlockLoop = ComponentDefinitionBase & {
51
47
  paginatedLoop: false;
52
48
  components: ComponentDefinition[];
53
- } & BlockLoop2;
54
- export type BlockLoop1 = ComponentDefinitionBase;
55
- export type BlockLoop2 = {
49
+ } & BlockLoop1;
50
+ export type BlockLoop1 = {
56
51
  lines: {
57
52
  min: VTLExpression;
58
53
  max: VTLExpression;
@@ -60,7 +55,7 @@ export type BlockLoop2 = {
60
55
  } | {
61
56
  iterations: VTLExpression;
62
57
  };
63
- export type ComponentRosterForLoopDefinition = ComponentRosterForLoopDefinition1 & {
58
+ export type ComponentRosterForLoopDefinition = ComponentDefinitionBase & {
64
59
  componentType: 'RosterForLoop';
65
60
  components: ComponentDefinition[];
66
61
  lines: {
@@ -69,7 +64,6 @@ export type ComponentRosterForLoopDefinition = ComponentRosterForLoopDefinition1
69
64
  };
70
65
  header?: TableHeader;
71
66
  };
72
- export type ComponentRosterForLoopDefinition1 = ComponentDefinitionBase;
73
67
  export type Options = {
74
68
  value: string | boolean;
75
69
  label: VTLExpression;
@@ -83,35 +77,31 @@ export type TableHeader = {
83
77
  rowspan?: number;
84
78
  options?: Options;
85
79
  }[];
86
- export type ComponentTableDefinition = ComponentTableDefinition1 & {
80
+ export type ComponentTableDefinition = ComponentDefinitionBase & {
87
81
  componentType: 'Table';
88
82
  header?: TableHeader;
89
83
  body: (ComponentDefinition | {
90
84
  label: VTLExpression;
91
85
  })[][];
92
86
  };
93
- export type ComponentTableDefinition1 = ComponentDefinitionBase;
94
- export type ComponentNumberDefinition = ComponentNumberDefinition1 & {
87
+ export type ComponentNumberDefinition = ComponentDefinitionBaseWithResponse & {
95
88
  componentType: 'InputNumber';
96
89
  unit?: string | VTLExpression;
97
90
  min?: number;
98
91
  max?: number;
99
92
  decimals?: number;
100
93
  };
101
- export type ComponentNumberDefinition1 = ComponentDefinitionBaseWithResponse;
102
- export type ComponentDurationDefinition = ComponentDurationDefinition1 & {
94
+ export type ComponentDurationDefinition = ComponentDefinitionBaseWithResponse & {
103
95
  componentType: 'Duration';
104
96
  format: 'PnYnM' | 'PTnHnM';
105
97
  };
106
- export type ComponentDurationDefinition1 = ComponentDefinitionBaseWithResponse;
107
- export type ComponentDatePickerDefinition = ComponentDatePickerDefinition1 & {
98
+ export type ComponentDatePickerDefinition = ComponentDefinitionBaseWithResponse & {
108
99
  componentType: 'Datepicker';
109
100
  dateFormat: 'YYYY-MM-DD' | 'YYYY' | 'YYYY-MM';
110
101
  min?: string;
111
102
  max?: string;
112
103
  };
113
- export type ComponentDatePickerDefinition1 = ComponentDefinitionBaseWithResponse;
114
- export type ComponentCheckboxGroupDefinition = ComponentCheckboxGroupDefinition1 & {
104
+ export type ComponentCheckboxGroupDefinition = ComponentDefinitionBase & {
115
105
  componentType: 'CheckboxGroup';
116
106
  orientation?: 'horizontal' | 'vertical';
117
107
  responses: {
@@ -126,17 +116,14 @@ export type ComponentCheckboxGroupDefinition = ComponentCheckboxGroupDefinition1
126
116
  };
127
117
  }[];
128
118
  };
129
- export type ComponentCheckboxGroupDefinition1 = ComponentDefinitionBase;
130
- export type ComponentCheckboxBooleanDefinition = ComponentCheckboxBooleanDefinition1 & {
119
+ export type ComponentCheckboxBooleanDefinition = ComponentDefinitionBaseWithResponse & {
131
120
  componentType: 'CheckboxBoolean';
132
121
  };
133
- export type ComponentCheckboxBooleanDefinition1 = ComponentDefinitionBaseWithResponse;
134
- export type ComponentRadioDefinition = ComponentRadioDefinition1 & {
122
+ export type ComponentRadioDefinition = ComponentDefinitionBaseWithResponse & {
135
123
  componentType: 'Radio';
136
124
  orientation?: 'horizontal' | 'vertical';
137
125
  options: OptionsWithDetail;
138
126
  };
139
- export type ComponentRadioDefinition1 = ComponentDefinitionBaseWithResponse;
140
127
  export type OptionsWithDetail = {
141
128
  value: string | boolean;
142
129
  label: VTLExpression;
@@ -148,22 +135,19 @@ export type OptionsWithDetail = {
148
135
  };
149
136
  };
150
137
  }[];
151
- export type ComponentDropdownDefinition = ComponentDropdownDefinition1 & {
138
+ export type ComponentDropdownDefinition = ComponentDefinitionBaseWithResponse & {
152
139
  componentType: 'Dropdown';
153
140
  options: Options;
154
141
  };
155
- export type ComponentDropdownDefinition1 = ComponentDefinitionBaseWithResponse;
156
- export type ComponentQuestionDefinition = ComponentQuestionDefinition1 & {
142
+ export type ComponentQuestionDefinition = ComponentDefinitionBase & {
157
143
  componentType: 'Question';
158
144
  components: ComponentDefinition[];
159
145
  };
160
- export type ComponentQuestionDefinition1 = ComponentDefinitionBase;
161
- export type ComponentCheckboxOneDefinition = ComponentCheckboxOneDefinition1 & {
146
+ export type ComponentCheckboxOneDefinition = ComponentDefinitionBaseWithResponse & {
162
147
  componentType: 'CheckboxOne';
163
148
  options: OptionsWithDetail;
164
149
  };
165
- export type ComponentCheckboxOneDefinition1 = ComponentDefinitionBaseWithResponse;
166
- export type ComponentSuggesterDefinition = ComponentSuggesterDefinition1 & {
150
+ export type ComponentSuggesterDefinition = ComponentDefinitionBaseWithResponse & {
167
151
  componentType: 'Suggester';
168
152
  /**
169
153
  * Nom / Index du référentiel à utiliser
@@ -189,8 +173,7 @@ export type ComponentSuggesterDefinition = ComponentSuggesterDefinition1 & {
189
173
  attribute: string;
190
174
  }[];
191
175
  };
192
- export type ComponentSuggesterDefinition1 = ComponentDefinitionBaseWithResponse;
193
- export type ComponentPairWiseLinksDefinition = ComponentPairWiseLinksDefinition1 & {
176
+ export type ComponentPairWiseLinksDefinition = ComponentDefinitionBase & {
194
177
  componentType: 'PairwiseLinks';
195
178
  xAxisIterations: VTLScalarExpression;
196
179
  yAxisIterations: VTLScalarExpression;
@@ -201,8 +184,7 @@ export type ComponentPairWiseLinksDefinition = ComponentPairWiseLinksDefinition1
201
184
  };
202
185
  components: ComponentDefinition[];
203
186
  };
204
- export type ComponentPairWiseLinksDefinition1 = ComponentDefinitionBase;
205
- export type ComponentSummaryDefinition = ComponentSummaryDefinition1 & {
187
+ export type ComponentSummaryDefinition = ComponentDefinitionBase & {
206
188
  componentType: 'Summary';
207
189
  sections: {
208
190
  id: string;
@@ -215,7 +197,6 @@ export type ComponentSummaryDefinition = ComponentSummaryDefinition1 & {
215
197
  }[];
216
198
  }[];
217
199
  };
218
- export type ComponentSummaryDefinition1 = ComponentDefinitionBase;
219
200
  export type Variable = {
220
201
  variableType: 'EXTERNAL';
221
202
  name: string;
@@ -405,4 +386,8 @@ export type SuggesterDefinition = {
405
386
  } | {
406
387
  type: 'soft';
407
388
  };
389
+ /**
390
+ * list of words to exclude from the searching
391
+ */
392
+ stopWords?: string[];
408
393
  };
@@ -1,3 +1,4 @@
1
+ /* eslint-disable */
1
2
  /**
2
3
  * This file was automatically generated by json-schema-to-typescript.
3
4
  * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
@@ -1 +1 @@
1
- {"version":3,"file":"type.source.js","sourceRoot":"","sources":["../src/type.source.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
1
+ {"version":3,"file":"type.source.js","sourceRoot":"","sources":["../src/type.source.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB;;;;GAIG"}
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,51 @@
1
+ import { describe, it, expect, vi, beforeAll, afterEach } from 'vitest';
2
+ import { SearchMinisearch } from './SearchMinisearch';
3
+ vi.mock('minisearch', () => {
4
+ return {
5
+ default: vi.fn().mockImplementation(() => ({
6
+ addAll: vi.fn(),
7
+ search: vi.fn(),
8
+ })),
9
+ };
10
+ });
11
+ vi.mock('./melauto', () => ({
12
+ applyMelauto: vi.fn(),
13
+ }));
14
+ describe('SearchMinisearch', () => {
15
+ let searchInstance;
16
+ const mockData = [
17
+ { id: '1', label: 'First Item' },
18
+ { id: '2', label: 'Second Item' },
19
+ ];
20
+ beforeAll(() => {
21
+ searchInstance = new SearchMinisearch({
22
+ name: 'test-suggester',
23
+ fields: [{ name: 'id' }, { name: 'label' }],
24
+ queryParser: {
25
+ type: 'tokenized',
26
+ params: { language: 'English', pattern: '\\w+', min: 1 },
27
+ },
28
+ max: 10,
29
+ });
30
+ });
31
+ afterEach(() => {
32
+ const miniSearchMock = searchInstance.db;
33
+ miniSearchMock.addAll.mockClear();
34
+ });
35
+ it('should initialize and index data correctly', async () => {
36
+ var _a;
37
+ await searchInstance.index(mockData);
38
+ // Check if MiniSearch instance was created and indexed
39
+ expect(searchInstance.db).not.toBeNull();
40
+ expect(searchInstance.isIndexed()).toBe(true);
41
+ // Check if addAll was called with the correct data
42
+ expect((_a = searchInstance.db) === null || _a === void 0 ? void 0 : _a.addAll).toHaveBeenCalledWith(mockData);
43
+ });
44
+ it('should not re-index if already indexed', async () => {
45
+ var _a;
46
+ searchInstance.indexed = true;
47
+ await searchInstance.index(mockData);
48
+ expect((_a = searchInstance.db) === null || _a === void 0 ? void 0 : _a.addAll).not.toHaveBeenCalled();
49
+ });
50
+ });
51
+ //# sourceMappingURL=SearchMiniSearch.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SearchMiniSearch.spec.js","sourceRoot":"","sources":["../../../src/utils/search/SearchMiniSearch.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,OAAO;QACN,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;YACf,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE;SACf,CAAC,CAAC;KACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3B,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;CACrB,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,cAAqC,CAAC;IAC1C,MAAM,QAAQ,GAAG;QAChB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE;QAChC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;KACjC,CAAC;IAEF,SAAS,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,IAAI,gBAAgB,CAAC;YACrC,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YAC3C,WAAW,EAAE;gBACZ,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;aACxD;YACD,GAAG,EAAE,EAAE;SACP,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,cAAc,GAAG,cAAc,CAAC,EAAS,CAAC;QAChD,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;;QAC3D,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,uDAAuD;QACvD,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,mDAAmD;QACnD,MAAM,CAAC,MAAA,cAAc,CAAC,EAAE,0CAAE,MAAM,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;;QACvD,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC;QAC9B,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,MAAM,CAAC,MAAA,cAAc,CAAC,EAAE,0CAAE,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAC1D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
@@ -4,7 +4,7 @@
4
4
  export function applyMelauto(query, data) {
5
5
  return data.sort((a, b) => {
6
6
  var _a, _b;
7
- return melautoScore((_a = b.label) !== null && _a !== void 0 ? _a : a.id, query) -
7
+ return melautoScore((_a = b.label) !== null && _a !== void 0 ? _a : b.id, query) -
8
8
  melautoScore((_b = a.label) !== null && _b !== void 0 ? _b : a.id, query);
9
9
  });
10
10
  }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,67 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { applyMelauto, melautoScore } from './melauto';
3
+ // Mock data to test the sorting and scoring
4
+ const data = [
5
+ { id: '1', label: 'Hello world' },
6
+ { id: '2', label: 'Bonjour le monde' },
7
+ { id: '3', label: 'Hello everyone' },
8
+ { id: '4', label: 'Greetings planet' },
9
+ ];
10
+ describe('applyMelauto', () => {
11
+ it('should sort data by relevance to the query', () => {
12
+ const sortedData = applyMelauto('hello', data);
13
+ const expectedSortedData = [
14
+ { id: '1', label: 'Hello world' },
15
+ { id: '3', label: 'Hello everyone' },
16
+ { id: '2', label: 'Bonjour le monde' },
17
+ { id: '4', label: 'Greetings planet' },
18
+ ];
19
+ expect(sortedData).toStrictEqual(expectedSortedData);
20
+ });
21
+ it('should return data in original order if query is empty', () => {
22
+ const sortedData = applyMelauto('', data);
23
+ expect(sortedData).toEqual(data);
24
+ });
25
+ it('should handle data without labels by using id instead', () => {
26
+ const noLabelData = [
27
+ { id: 'a' },
28
+ { id: 'b', label: 'Hello' },
29
+ { id: 'hello-w' },
30
+ ];
31
+ const sortedData = applyMelauto('hello', noLabelData);
32
+ const expectedSortedData = [
33
+ { id: 'b', label: 'Hello' },
34
+ { id: 'hello-w' },
35
+ { id: 'a' },
36
+ ];
37
+ expect(sortedData).toStrictEqual(expectedSortedData);
38
+ });
39
+ });
40
+ describe('melautoScore', () => {
41
+ it('should return a non-null score', () => {
42
+ const score = melautoScore('Hello world', 'hello');
43
+ const expectedScore = 1;
44
+ expect(score).toBeCloseTo(expectedScore, 2);
45
+ });
46
+ it('should give a higher score for a closer match', () => {
47
+ const score1 = melautoScore('Hello beautiful world', 'hello world');
48
+ const score2 = melautoScore('Hello beautiful world', 'hello');
49
+ expect(score1).toBeGreaterThan(score2);
50
+ });
51
+ it('should give the same score for a same query if the comparison string is longer', () => {
52
+ const score1 = melautoScore('Hello world', 'hello');
53
+ const score2 = melautoScore('Hello beautiful world', 'hello');
54
+ expect(score1).toBeCloseTo(score2, 2);
55
+ });
56
+ it('should calculate a null score', () => {
57
+ const score = melautoScore('Greetings planet', 'hello');
58
+ const expectedScore = 0;
59
+ expect(score).toBe(expectedScore);
60
+ });
61
+ it('should handle accent and special characters correctly', () => {
62
+ const score1 = melautoScore('Héllo wörld', 'hello world');
63
+ const score2 = melautoScore('Hello world', 'héllo-wOrld');
64
+ expect(score1).toBeCloseTo(score2, 2);
65
+ });
66
+ });
67
+ //# sourceMappingURL=melauto.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"melauto.spec.js","sourceRoot":"","sources":["../../../src/utils/search/melauto.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEvD,4CAA4C;AAC5C,MAAM,IAAI,GAAG;IACZ,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;IACjC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;IACtC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,gBAAgB,EAAE;IACpC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;CACtC,CAAC;AAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACrD,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,kBAAkB,GAAG;YAC1B,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;YACjC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,gBAAgB,EAAE;YACpC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;YACtC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE;SACtC,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QACjE,MAAM,UAAU,GAAG,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,WAAW,GAAG;YACnB,EAAE,EAAE,EAAE,GAAG,EAAE;YACX,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE;YAC3B,EAAE,EAAE,EAAE,SAAS,EAAE;SACjB,CAAC;QACF,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,kBAAkB,GAAG;YAC1B,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE;YAC3B,EAAE,EAAE,EAAE,SAAS,EAAE;YACjB,EAAE,EAAE,EAAE,GAAG,EAAE;SACX,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACzC,MAAM,KAAK,GAAG,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,aAAa,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,aAAa,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,GAAG,EAAE;QACzF,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,YAAY,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACxC,MAAM,KAAK,GAAG,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,aAAa,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAChE,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
@@ -1,7 +1,8 @@
1
1
  import type { SearchInfo } from './SearchInterface';
2
2
  import type { ItemOf } from '../../type.utils';
3
3
  /**
4
- * Generates a tokenize method
4
+ * Generates a tokenize method.
5
+ * When used for tokenizing a search query instead of the indexing, the fieldName is undefined.
5
6
  */
6
7
  export declare const tokenizer: (info: SearchInfo) => (str: string, fieldName?: string) => string[];
7
8
  /**
@@ -11,4 +12,8 @@ export declare const tokenizeQuery: (str: string, info: SearchInfo["queryParser"
11
12
  /**
12
13
  * Tokenizer used for indexing (based on "fields" specification)
13
14
  */
14
- export declare const tokenizeIndex: (str: string, info: ItemOf<SearchInfo["fields"]>) => string[];
15
+ export declare const tokenizeIndex: (str: string, info: ItemOf<SearchInfo["fields"]>, stopWords?: string[]) => string[];
16
+ /**
17
+ * remove from a string all the words that are included in a stopwords list
18
+ */
19
+ export declare function filterStopWords(input: string, stopWords?: string[]): string;
@@ -1,10 +1,12 @@
1
1
  /**
2
- * Generates a tokenize method
2
+ * Generates a tokenize method.
3
+ * When used for tokenizing a search query instead of the indexing, the fieldName is undefined.
3
4
  */
4
5
  export const tokenizer = (info) => (str, fieldName) => {
5
6
  const field = info.fields.find((f) => f.name === fieldName);
7
+ const stopWords = info.stopWords;
6
8
  return field
7
- ? tokenizeIndex(str, field)
9
+ ? tokenizeIndex(str, field, stopWords)
8
10
  : tokenizeQuery(str, info.queryParser);
9
11
  };
10
12
  /**
@@ -18,8 +20,8 @@ export const tokenizeQuery = (str, info) => {
18
20
  .filter((w) => w.length > 0);
19
21
  }
20
22
  const wordRegex = info.params.pattern && info.params.pattern !== 'soft'
21
- ? /\w+/gi
22
- : new RegExp(info.params.pattern, 'gi');
23
+ ? new RegExp(info.params.pattern, 'gi')
24
+ : /\w+/gi;
23
25
  const minLength = (_a = info.params.min) !== null && _a !== void 0 ? _a : 1;
24
26
  return ((_c = (_b = normalizeStr(str)
25
27
  .match(wordRegex)) === null || _b === void 0 ? void 0 : _b.filter((w) => w.length >= minLength)) !== null && _c !== void 0 ? _c : []);
@@ -27,11 +29,11 @@ export const tokenizeQuery = (str, info) => {
27
29
  /**
28
30
  * Tokenizer used for indexing (based on "fields" specification)
29
31
  */
30
- export const tokenizeIndex = (str, info) => {
32
+ export const tokenizeIndex = (str, info, stopWords) => {
31
33
  var _a, _b, _c;
32
34
  const wordRegex = info.rules && info.rules !== 'soft'
33
- ? /\w+/gi
34
- : new RegExp(info.rules[0], 'gi');
35
+ ? new RegExp(info.rules[0], 'gi')
36
+ : /\w+/gi;
35
37
  const minLength = (_a = info.min) !== null && _a !== void 0 ? _a : 1;
36
38
  // For synonyms, add the synonyms to the string
37
39
  if (info.synonyms) {
@@ -40,7 +42,8 @@ export const tokenizeIndex = (str, info) => {
40
42
  str = str.replaceAll(source, `${source} ${synonyms}`);
41
43
  }
42
44
  }
43
- return ((_c = (_b = normalizeStr(str)
45
+ // We remove the stopWords from the string
46
+ return ((_c = (_b = filterStopWords(normalizeStr(str), stopWords)
44
47
  .match(wordRegex)) === null || _b === void 0 ? void 0 : _b.filter((w) => w.length >= minLength)) !== null && _c !== void 0 ? _c : []);
45
48
  };
46
49
  /**
@@ -54,4 +57,16 @@ const normalizeStr = (str) => {
54
57
  .replace(/[\u0300-\u036f]/g, '')
55
58
  .toLowerCase();
56
59
  };
60
+ /**
61
+ * remove from a string all the words that are included in a stopwords list
62
+ */
63
+ export function filterStopWords(input, stopWords) {
64
+ if (!stopWords) {
65
+ return input;
66
+ }
67
+ const lowerCaseStopWords = stopWords.map((word) => word.toLowerCase());
68
+ const words = input.split(/\s+/);
69
+ const filteredWords = words.filter((word) => !lowerCaseStopWords.includes(word.toLowerCase()));
70
+ return filteredWords.join(' ');
71
+ }
57
72
  //# sourceMappingURL=tokenizer.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAE5D,OAAO,KAAK;QACX,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC;QAC3B,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,YAAY,CAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEvC,OAAO,CACN,MAAA,MAAA,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EACjC,EAAE;;IACH,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;QACvD,CAAC;IACF,CAAC;IAED,OAAO,CACN,MAAA,MAAA,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,OAAO,GAAG;SACR,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,WAAW,EAAE,CAAC;AACjB,CAAC,CAAC"}
1
+ {"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,OAAO,KAAK;QACX,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;QACtC,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,YAAY,CAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEvC,OAAO,CACN,MAAA,MAAA,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EAClC,SAAoB,EACnB,EAAE;;IACH,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,MAAA,IAAI,CAAC,GAAG,mCAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjD,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;QACvD,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,CACN,MAAA,MAAA,eAAe,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC;SAC3C,KAAK,CAAC,SAAS,CAAC,0CACf,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,mCAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,OAAO,GAAG;SACR,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,WAAW,EAAE,CAAC;AACjB,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,SAAoB;IAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC1D,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC"}
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,160 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { tokenizer, tokenizeQuery, tokenizeIndex, filterStopWords, } from './tokenizer';
3
+ const mockSearchInfo = {
4
+ name: 'Products',
5
+ fields: [
6
+ {
7
+ name: 'title',
8
+ min: 3,
9
+ rules: ['[\\w]+'],
10
+ synonyms: {
11
+ car: ['vehicle', 'automobile'],
12
+ },
13
+ },
14
+ ],
15
+ queryParser: {
16
+ type: 'tokenized',
17
+ params: {
18
+ language: 'English',
19
+ pattern: '[\\w.]+',
20
+ min: 2,
21
+ },
22
+ },
23
+ };
24
+ describe('filterStopWords', () => {
25
+ it('should remove only stopwords from the input string', () => {
26
+ const input = 'This is a test.';
27
+ const stopWords = ['is', 'a'];
28
+ const result = filterStopWords(input, stopWords);
29
+ expect(result).toBe('This test.');
30
+ });
31
+ it('should not alter words that are substrings of stopwords', () => {
32
+ const input = 'this is a testing example';
33
+ const stopWords = ['test'];
34
+ const result = filterStopWords(input, stopWords);
35
+ expect(result).toBe('this is a testing example');
36
+ });
37
+ it('should be case-insensitive', () => {
38
+ const input = 'This Is A Test.';
39
+ const stopWords = ['is', 'a'];
40
+ const result = filterStopWords(input, stopWords);
41
+ expect(result).toBe('This Test.');
42
+ });
43
+ it('should return the input string unchanged if stopWords is undefined', () => {
44
+ const input = 'This is a test.';
45
+ const result = filterStopWords(input);
46
+ expect(result).toBe(input);
47
+ });
48
+ it('should return the input string unchanged if stopWords is a empty array', () => {
49
+ const input = 'This is a test.';
50
+ const stopWords = [];
51
+ const result = filterStopWords(input, stopWords);
52
+ expect(result).toBe(input);
53
+ });
54
+ it('should return an empty string if all words are stopwords', () => {
55
+ const input = 'this is a test';
56
+ const stopWords = ['this', 'is', 'a', 'test'];
57
+ const result = filterStopWords(input, stopWords);
58
+ expect(result).toBe('');
59
+ });
60
+ it('should handle strings with multiple spaces correctly', () => {
61
+ const input = 'This is a test.';
62
+ const stopWords = ['is', 'a'];
63
+ const result = filterStopWords(input, stopWords);
64
+ expect(result).toBe('This test.');
65
+ });
66
+ it('should handle empty input string', () => {
67
+ const input = '';
68
+ const stopWords = ['is', 'a'];
69
+ const result = filterStopWords(input, stopWords);
70
+ expect(result).toBe('');
71
+ });
72
+ it('should handle punctuation correctly', () => {
73
+ const input = 'Hello, world! This is a test.';
74
+ const stopWords = ['is', 'a'];
75
+ const result = filterStopWords(input, stopWords);
76
+ expect(result).toBe('Hello, world! This test.');
77
+ });
78
+ });
79
+ describe('tokenizeQuery', () => {
80
+ it('should tokenize based on soft type', () => {
81
+ const queryParser = { type: 'soft' };
82
+ const result = tokenizeQuery('This is a test!', queryParser);
83
+ expect(result).toEqual(['this', 'is', 'a', 'test']);
84
+ });
85
+ it('should tokenize with a custom regex and a min', () => {
86
+ const queryParser = {
87
+ type: 'tokenized',
88
+ params: { pattern: '[\\w.]+', min: 3 },
89
+ };
90
+ const result = tokenizeQuery('This is a test !', queryParser);
91
+ expect(result).toEqual(['this', 'test']);
92
+ });
93
+ it('should normalize the input', () => {
94
+ const queryParser = {
95
+ type: 'tokenized',
96
+ params: { pattern: '\\w+', min: 1 },
97
+ };
98
+ const result = tokenizeQuery('Élève Étudiant!', queryParser);
99
+ expect(result).toEqual(['eleve', 'etudiant']);
100
+ });
101
+ it('should return an empty array for unmatched patterns', () => {
102
+ const queryParser = {
103
+ type: 'tokenized',
104
+ params: { language: 'French', pattern: '[\\d.]+', min: 1 }, // only digits
105
+ };
106
+ const result = tokenizeQuery('No numbers here!', queryParser);
107
+ expect(result).toEqual([]);
108
+ });
109
+ });
110
+ describe('tokenizeIndex', () => {
111
+ it('should filter out words shorter than the required minimum length', () => {
112
+ const fieldInfo = mockSearchInfo.fields[0];
113
+ const result = tokenizeIndex('The bus is so slow', fieldInfo);
114
+ expect(result).toEqual(['the', 'bus', 'slow']);
115
+ });
116
+ it('should tokenize and apply synonyms', () => {
117
+ const fieldInfo = mockSearchInfo.fields[0];
118
+ const result = tokenizeIndex('The car is fast', fieldInfo);
119
+ expect(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
120
+ });
121
+ it('should normalize the input', () => {
122
+ const fieldInfo = mockSearchInfo.fields[0];
123
+ const result = tokenizeIndex('Élève Étudiant!', fieldInfo);
124
+ expect(result).toEqual(['eleve', 'etudiant']);
125
+ });
126
+ it('should filter out stopWords', () => {
127
+ const fieldInfo = { ...mockSearchInfo.fields[0], min: 1 };
128
+ const stopWords = ['is', 'the', 'of', 'this', 'a'];
129
+ const result = tokenizeIndex('This is a test of stopWords !', fieldInfo, stopWords);
130
+ expect(result).toEqual(['test', 'stopwords']);
131
+ });
132
+ it('should return an empty array for unmatched patterns', () => {
133
+ const fieldInfo = { ...mockSearchInfo.fields[0], rules: ['\\d+'] }; // Only numbers
134
+ const result = tokenizeIndex('No numbers here', fieldInfo);
135
+ expect(result).toEqual([]);
136
+ });
137
+ });
138
+ describe('tokenizer', () => {
139
+ it('should tokenize using field rules', () => {
140
+ const tokenize = tokenizer(mockSearchInfo);
141
+ const result = tokenize('The car is fast', 'title');
142
+ expect(result).toEqual(['the', 'car', 'vehicle', 'automobile', 'fast']);
143
+ });
144
+ it('should tokenize using query parser when field is not found', () => {
145
+ const tokenize = tokenizer(mockSearchInfo);
146
+ const result = tokenize('This is a test!');
147
+ expect(result).toEqual(['this', 'is', 'test']);
148
+ });
149
+ it('should normalize the input', () => {
150
+ const tokenize = tokenizer(mockSearchInfo);
151
+ const result = tokenize('Élève Étudiant!');
152
+ expect(result).toEqual(['eleve', 'etudiant']);
153
+ });
154
+ it('should handle empty strings', () => {
155
+ const tokenize = tokenizer(mockSearchInfo);
156
+ expect(tokenize('', 'title')).toEqual([]);
157
+ expect(tokenize('')).toEqual([]);
158
+ });
159
+ });
160
+ //# sourceMappingURL=tokenizer.spec.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tokenizer.spec.js","sourceRoot":"","sources":["../../../src/utils/search/tokenizer.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACN,SAAS,EACT,aAAa,EACb,aAAa,EACb,eAAe,GACf,MAAM,aAAa,CAAC;AAGrB,MAAM,cAAc,GAAe;IAClC,IAAI,EAAE,UAAU;IAChB,MAAM,EAAE;QACP;YACC,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC,QAAQ,CAAC;YACjB,QAAQ,EAAE;gBACT,GAAG,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;aAC9B;SACD;KACD;IACD,WAAW,EAAE;QACZ,IAAI,EAAE,WAAW;QACjB,MAAM,EAAE;YACP,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,CAAC;SACN;KACD;CACD,CAAC;AAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QAClE,MAAM,KAAK,GAAG,2BAA2B,CAAC;QAC1C,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC7E,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QACjF,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAChC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QACnE,MAAM,KAAK,GAAG,gBAAgB,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC/D,MAAM,KAAK,GAAG,qBAAqB,CAAC;QACpC,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC3C,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC9C,MAAM,KAAK,GAAG,+BAA+B,CAAC;QAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,EAA+B,CAAC;QAElE,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACxD,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE;SACT,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;SACN,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,WAAW,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,cAAc;SAC7C,CAAC;QAE/B,MAAM,MAAM,GAAG,aAAa,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC3E,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,aAAa,CAC3B,+BAA+B,EAC/B,SAAS,EACT,SAAS,CACT,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC9D,MAAM,SAAS,GAAG,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,eAAe;QAEnF,MAAM,MAAM,GAAG,aAAa,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAE3C,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}