@inseefr/lunatic 3.12.1 → 3.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  const vitest_1 = require("vitest");
4
4
  const SearchMinisearch_1 = require("./SearchMinisearch");
5
+ const melauto_1 = require("./melauto");
5
6
  vitest_1.vi.mock('minisearch', () => {
6
7
  return {
7
8
  default: vitest_1.vi.fn().mockImplementation(() => ({
@@ -22,7 +23,13 @@ vitest_1.vi.mock('./melauto', () => ({
22
23
  (0, vitest_1.beforeAll)(() => {
23
24
  searchInstance = new SearchMinisearch_1.SearchMinisearch({
24
25
  name: 'test-suggester',
25
- fields: [{ name: 'id' }, { name: 'label' }],
26
+ fields: [
27
+ { name: 'id' },
28
+ {
29
+ name: 'label',
30
+ synonyms: { accueil: ['ACCEUIL', 'ACUEIL'] },
31
+ },
32
+ ],
26
33
  queryParser: {
27
34
  type: 'tokenized',
28
35
  params: { language: 'English', pattern: '\\w+', min: 1 },
@@ -47,5 +54,12 @@ vitest_1.vi.mock('./melauto', () => ({
47
54
  await searchInstance.index(mockData);
48
55
  (0, vitest_1.expect)(searchInstance.db?.addAll).not.toHaveBeenCalled();
49
56
  });
57
+ (0, vitest_1.it)('should expand query synonyms before melauto sorting', async () => {
58
+ await searchInstance.index(mockData);
59
+ (searchInstance.db?.search).mockReturnValue(mockData);
60
+ vitest_1.vi.mocked(melauto_1.applyMelauto).mockReturnValue(mockData);
61
+ await searchInstance.search('agent acceuil');
62
+ (0, vitest_1.expect)(melauto_1.applyMelauto).toHaveBeenCalledWith('agent acceuil accueil', mockData);
63
+ });
50
64
  });
51
65
  //# sourceMappingURL=SearchMiniSearch.spec.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"SearchMiniSearch.spec.js","sourceRoot":"","sources":["../../src/utils/search/SearchMiniSearch.spec.ts"],"names":[],"mappings":";;AAAA,mCAAwE;AACxE,yDAAsD;AAEtD,WAAE,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,OAAO;QACN,OAAO,EAAE,WAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;YACf,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;SACf,CAAC,CAAC;KACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,WAAE,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3B,YAAY,EAAE,WAAE,CAAC,EAAE,EAAE;CACrB,CAAC,CAAC,CAAC;AAEJ,IAAA,iBAAQ,EAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,cAAqC,CAAC;IAC1C,MAAM,QAAQ,GAAG;QAChB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE;QAChC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;KACjC,CAAC;IAEF,IAAA,kBAAS,EAAC,GAAG,EAAE;QACd,cAAc,GAAG,IAAI,mCAAgB,CAAC;YACrC,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YAC3C,WAAW,EAAE;gBACZ,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;aACxD;YACD,GAAG,EAAE,EAAE;SACP,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,GAAG,EAAE;QACd,MAAM,cAAc,GAAG,cAAc,CAAC,EAAS,CAAC;QAChD,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,uDAAuD;QACvD,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,IAAA,eAAM,EAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,mDAAmD;QACnD,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC;QAC9B,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAC1D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"SearchMiniSearch.spec.js","sourceRoot":"","sources":["../../src/utils/search/SearchMiniSearch.spec.ts"],"names":[],"mappings":";;AAAA,mCAAwE;AACxE,yDAAsD;AACtD,uCAAyC;AAEzC,WAAE,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,OAAO;QACN,OAAO,EAAE,WAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;YACf,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;SACf,CAAC,CAAC;KACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,WAAE,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3B,YAAY,EAAE,WAAE,CAAC,EAAE,EAAE;CACrB,CAAC,CAAC,CAAC;AAEJ,IAAA,iBAAQ,EAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,IAAI,cAAqC,CAAC;IAC1C,MAAM,QAAQ,GAAG;QAChB,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE;QAChC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE;KACjC,CAAC;IAEF,IAAA,kBAAS,EAAC,GAAG,EAAE;QACd,cAAc,GAAG,IAAI,mCAAgB,CAAC;YACrC,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE;gBACP,EAAE,IAAI,EAAE,IAAI,EAAE;gBACd;oBACC,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE;iBAC5C;aACD;YACD,WAAW,EAAE;gBACZ,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;aACxD;YACD,GAAG,EAAE,EAAE;SACP,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,GAAG,EAAE;QACd,MAAM,cAAc,GAAG,cAAc,CAAC,EAAS,CAAC;QAChD,cAAc,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,uDAAuD;QACvD,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,IAAA,eAAM,EAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,mDAAmD;QACnD,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC;QAC9B,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErC,IAAA,eAAM,EAAC,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC,cAAc,CAAC,EAAE,EAAE,MAAc,CAAA,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC7D,WAAE,CAAC,MAAM,CAAC,sBAAY,CAAC,CAAC,eAAe,CAAC,QAAe,CAAC,CAAC;QAEzD,MAAM,cAAc,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAE7C,IAAA,eAAM,EAAC,sBAAY,CAAC,CAAC,oBAAoB,CACxC,uBAAuB,EACvB,QAAQ,CACR,CAAC;IACH,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
@@ -7,6 +7,31 @@ exports.SearchMinisearch = void 0;
7
7
  const melauto_1 = require("./melauto");
8
8
  const minisearch_1 = __importDefault(require("minisearch"));
9
9
  const tokenizer_1 = require("./tokenizer");
10
+ const utils_1 = require("./utils");
11
+ function getMelautoQuery(query, info) {
12
+ const tokens = (0, tokenizer_1.tokenizer)(info)(query);
13
+ // existing query tokens (already tokenized/normalized by tokenizer).
14
+ const expandedTokens = new Set(tokens);
15
+ // add synonyms to keep melauto ranking.
16
+ for (const field of info.fields) {
17
+ if (!field.synonyms) {
18
+ continue;
19
+ }
20
+ for (const source in field.synonyms) {
21
+ const normalizedSource = (0, utils_1.normalizeStr)(source);
22
+ const normalizedSynonyms = field.synonyms[source].map((synonym) => (0, utils_1.normalizeStr)(synonym));
23
+ // source -> synonyms
24
+ if (expandedTokens.has(normalizedSource)) {
25
+ normalizedSynonyms.forEach((synonym) => expandedTokens.add(synonym));
26
+ }
27
+ // synonym -> source
28
+ if (normalizedSynonyms.some((synonym) => expandedTokens.has(synonym))) {
29
+ expandedTokens.add(normalizedSource);
30
+ }
31
+ }
32
+ }
33
+ return Array.from(expandedTokens).join(' ');
34
+ }
10
35
  class SearchMinisearch {
11
36
  db = null;
12
37
  info;
@@ -38,7 +63,7 @@ class SearchMinisearch {
38
63
  prefix: (term) => term.length > 2,
39
64
  });
40
65
  // Apply melauto to classify results
41
- data = (0, melauto_1.applyMelauto)(q, data);
66
+ data = (0, melauto_1.applyMelauto)(getMelautoQuery(q, this.info), data);
42
67
  data = data.slice(0, this.info.max);
43
68
  return data;
44
69
  }
@@ -1 +1 @@
1
- {"version":3,"file":"SearchMinisearch.js","sourceRoot":"","sources":["../../src/utils/search/SearchMinisearch.ts"],"names":[],"mappings":";;;;;;AAKA,uCAAyC;AACzC,4DAAoC;AACpC,2CAAwC;AAExC,MAAa,gBAAgB;IAG5B,EAAE,GAAyB,IAAI,CAAC;IAChC,IAAI,CAAa;IACjB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,IAAgB;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,SAAS;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAS;QACpB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,EAAE,GAAG,IAAI,oBAAU,CAAC;YACxB,MAAM,EAAE,UAAU;YAClB,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU;YACxD,QAAQ,EAAE,IAAA,qBAAS,EAAC,IAAI,CAAC,IAAI,CAAC;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,CAAS;QACrB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACX,CAAC;QACD,IAAI,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE;YAC5B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;SACjC,CAAe,CAAC;QAEjB,oCAAoC;QACpC,IAAI,GAAG,IAAA,sBAAY,EAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAE7B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,aAAa,CAAC,EAAO;QACpB,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,EAAO,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,CAAM,CAAC;IACzC,CAAC;CACD;AAnDD,4CAmDC"}
1
+ {"version":3,"file":"SearchMinisearch.js","sourceRoot":"","sources":["../../src/utils/search/SearchMinisearch.ts"],"names":[],"mappings":";;;;;;AAKA,uCAAyC;AACzC,4DAAoC;AACpC,2CAAwC;AACxC,mCAAuC;AAEvC,SAAS,eAAe,CAAC,KAAa,EAAE,IAAgB;IACvD,MAAM,MAAM,GAAG,IAAA,qBAAS,EAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;IAEtC,qEAAqE;IACrE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAEvC,wCAAwC;IACxC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrB,SAAS;QACV,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,gBAAgB,GAAG,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,kBAAkB,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACjE,IAAA,oBAAY,EAAC,OAAO,CAAC,CACrB,CAAC;YAEF,qBAAqB;YACrB,IAAI,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC1C,kBAAkB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;YACtE,CAAC;YAED,oBAAoB;YACpB,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBACvE,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACtC,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED,MAAa,gBAAgB;IAG5B,EAAE,GAAyB,IAAI,CAAC;IAChC,IAAI,CAAa;IACjB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,IAAgB;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,SAAS;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAS;QACpB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,EAAE,GAAG,IAAI,oBAAU,CAAC;YACxB,MAAM,EAAE,UAAU;YAClB,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU;YACxD,QAAQ,EAAE,IAAA,qBAAS,EAAC,IAAI,CAAC,IAAI,CAAC;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,CAAS;QACrB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACX,CAAC;QACD,IAAI,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE;YAC5B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;SACjC,CAAe,CAAC;QAEjB,oCAAoC;QACpC,IAAI,GAAG,IAAA,sBAAY,EAAC,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QAEzD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,aAAa,CAAC,EAAO;QACpB,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACd,OAAO,EAAO,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,CAAM,CAAC;IACzC,CAAC;CACD;AAnDD,4CAmDC"}
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.tokenizeIndex = exports.tokenizeQuery = exports.tokenizer = void 0;
4
4
  exports.filterStopWords = filterStopWords;
5
+ const utils_1 = require("./utils");
5
6
  /**
6
7
  * Generates a tokenize method.
7
8
  * When used for tokenizing a search query instead of the indexing, the fieldName is undefined.
@@ -19,7 +20,7 @@ exports.tokenizer = tokenizer;
19
20
  */
20
21
  const tokenizeQuery = (str, info) => {
21
22
  if (info.type === 'soft') {
22
- return normalizeStr(str)
23
+ return (0, utils_1.normalizeStr)(str)
23
24
  .split(/[^a-z0-9]+/)
24
25
  .filter((w) => w.length > 0);
25
26
  }
@@ -27,7 +28,7 @@ const tokenizeQuery = (str, info) => {
27
28
  ? new RegExp(info.params.pattern, 'gi')
28
29
  : /\w+/gi;
29
30
  const minLength = info.params.min ?? 1;
30
- return (normalizeStr(str)
31
+ return ((0, utils_1.normalizeStr)(str)
31
32
  .match(wordRegex)
32
33
  ?.filter((w) => w.length >= minLength) ?? []);
33
34
  };
@@ -36,7 +37,7 @@ exports.tokenizeQuery = tokenizeQuery;
36
37
  * Tokenizer used for indexing (based on "fields" specification)
37
38
  */
38
39
  const tokenizeIndex = (str, info, stopWords) => {
39
- let normalizedStr = normalizeStr(str);
40
+ let normalizedStr = (0, utils_1.normalizeStr)(str);
40
41
  const wordRegex = info.rules && info.rules !== 'soft'
41
42
  ? new RegExp(info.rules[0], 'gi')
42
43
  : /\w+/gi;
@@ -44,9 +45,9 @@ const tokenizeIndex = (str, info, stopWords) => {
44
45
  // For synonyms, add the synonyms to the string
45
46
  if (info.synonyms) {
46
47
  for (const source in info.synonyms) {
47
- const normalizedSource = normalizeStr(source);
48
+ const normalizedSource = (0, utils_1.normalizeStr)(source);
48
49
  const synonyms = info.synonyms[source]
49
- .map((synonym) => normalizeStr(synonym))
50
+ .map((synonym) => (0, utils_1.normalizeStr)(synonym))
50
51
  .join(' ');
51
52
  normalizedStr = normalizedStr.replaceAll(normalizedSource, `${normalizedSource} ${synonyms}`);
52
53
  }
@@ -57,20 +58,6 @@ const tokenizeIndex = (str, info, stopWords) => {
57
58
  ?.filter((w) => w.length >= minLength) ?? []);
58
59
  };
59
60
  exports.tokenizeIndex = tokenizeIndex;
60
- /**
61
- * Normalize a string
62
- * - Remove accent (é => e, à => a)
63
- * - remove ligatures (æ => ae, , Æ => ae, œ => oe, Œ => oe)
64
- * - Lowercase
65
- */
66
- const normalizeStr = (str) => {
67
- return str
68
- .toLowerCase()
69
- .replaceAll('œ', 'oe')
70
- .replaceAll('æ', 'ae')
71
- .normalize('NFD')
72
- .replace(/[\u0300-\u036f]/g, '');
73
- };
74
61
  /**
75
62
  * remove from a string all the words that are included in a stopwords list
76
63
  */
@@ -1 +1 @@
1
- {"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.ts"],"names":[],"mappings":";;;AAgGA,0CAUC;AAvGD;;;GAGG;AACI,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,OAAO,KAAK;QACX,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;QACtC,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AARU,QAAA,SAAS,aAQnB;AAEH;;GAEG;AACI,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,YAAY,CAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAEvC,OAAO,CACN,YAAY,CAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAlBW,QAAA,aAAa,iBAkBxB;AAEF;;GAEG;AACI,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EAClC,SAAoB,EACnB,EAAE;IACH,IAAI,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;iBACpC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;iBACvC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,aAAa,GAAG,aAAa,CAAC,UAAU,CACvC,gBAAgB,EAChB,GAAG,gBAAgB,IAAI,QAAQ,EAAE,CACjC,CAAC;QACH,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,CACN,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC;SACvC,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAjCW,QAAA,aAAa,iBAiCxB;AAEF;;;;;GAKG;AACH,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACpC,OAAO,GAAG;SACR,WAAW,EAAE;SACb,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC,CAAC;AAEF;;GAEG;AACH,SAAgB,eAAe,CAAC,KAAa,EAAE,SAAoB;IAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC1D,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC"}
1
+ {"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/utils/search/tokenizer.ts"],"names":[],"mappings":";;;AAkFA,0CAUC;AA1FD,mCAAuC;AAEvC;;;GAGG;AACI,MAAM,SAAS,GACrB,CAAC,IAAgB,EAAE,EAAE,CAAC,CAAC,GAAW,EAAE,SAAkB,EAAE,EAAE;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,OAAO,KAAK;QACX,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;QACtC,CAAC,CAAC,IAAA,qBAAa,EAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC,CAAC;AARU,QAAA,SAAS,aAQnB;AAEH;;GAEG;AACI,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,IAA+B,EAAE,EAAE;IAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,IAAA,oBAAY,EAAC,GAAG,CAAC;aACtB,KAAK,CAAC,YAAY,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,SAAS,GACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM;QACpD,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAEvC,OAAO,CACN,IAAA,oBAAY,EAAC,GAAG,CAAC;SACf,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAlBW,QAAA,aAAa,iBAkBxB;AAEF;;GAEG;AACI,MAAM,aAAa,GAAG,CAC5B,GAAW,EACX,IAAkC,EAClC,SAAoB,EACnB,EAAE;IACH,IAAI,aAAa,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GACd,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAClC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QAClC,CAAC,CAAC,OAAO,CAAC;IACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IAEhC,+CAA+C;IAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,MAAM,gBAAgB,GAAG,IAAA,oBAAY,EAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;iBACpC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAA,oBAAY,EAAC,OAAO,CAAC,CAAC;iBACvC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEZ,aAAa,GAAG,aAAa,CAAC,UAAU,CACvC,gBAAgB,EAChB,GAAG,gBAAgB,IAAI,QAAQ,EAAE,CACjC,CAAC;QACH,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,CACN,eAAe,CAAC,aAAa,EAAE,SAAS,CAAC;SACvC,KAAK,CAAC,SAAS,CAAC;QACjB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAC7C,CAAC;AACH,CAAC,CAAC;AAjCW,QAAA,aAAa,iBAiCxB;AAEF;;GAEG;AACH,SAAgB,eAAe,CAAC,KAAa,EAAE,SAAoB;IAClE,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,KAAK,CAAC;IACd,CAAC;IACD,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC1D,CAAC;IACF,OAAO,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC"}
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Normalize a string
3
+ * - Remove accent (é => e, à => a)
4
+ * - remove ligatures (æ => ae, , Æ => ae, œ => oe, Œ => oe)
5
+ * - Lowercase
6
+ */
7
+ export declare const normalizeStr: (str: string) => string;
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.normalizeStr = void 0;
4
+ /**
5
+ * Normalize a string
6
+ * - Remove accent (é => e, à => a)
7
+ * - remove ligatures (æ => ae, , Æ => ae, œ => oe, Œ => oe)
8
+ * - Lowercase
9
+ */
10
+ const normalizeStr = (str) => {
11
+ return str
12
+ .toLowerCase()
13
+ .replaceAll('œ', 'oe')
14
+ .replaceAll('æ', 'ae')
15
+ .normalize('NFD')
16
+ .replace(/[\u0300-\u036f]/g, '');
17
+ };
18
+ exports.normalizeStr = normalizeStr;
19
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils/search/utils.ts"],"names":[],"mappings":";;;AAAA;;;;;GAKG;AACI,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IAC3C,OAAO,GAAG;SACR,WAAW,EAAE;SACb,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;SACrB,SAAS,CAAC,KAAK,CAAC;SAChB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC,CAAC;AAPW,QAAA,YAAY,gBAOvB"}