@inseefr/lunatic 2.0.2 → 2.0.3-v2

package/lib/utils/suggester-workers/commons-tokenizer/create-entity-tokenizer.js

@@ -0,0 +1,70 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _createFieldsTokenizer = _interopRequireDefault(require("./create-fields-tokenizer"));
+
+var _filters = require("./filters");
+
+var _filterStemmer = _interopRequireDefault(require("./filters/filter-stemmer"));
+
+var _filterLength = _interopRequireDefault(require("./filters/filter-length"));
+
+var _filterSynonyms = _interopRequireDefault(require("./filters/filter-synonyms"));
+
+var _filterToLower = _interopRequireDefault(require("./filters/filter-to-lower"));
+
+var _filterDouble = _interopRequireDefault(require("./filters/filter-double"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function createMapFieldsTokenizer(fields, filters) {
+  return fields.reduce(function (mapFieldTokenizers, field) {
+    var name = field.name;
+    return _objectSpread(_objectSpread({}, mapFieldTokenizers), {}, _defineProperty({}, name, (0, _createFieldsTokenizer["default"])(field, filters)));
+  }, {});
+}
+
+function createFilterTokens(fields, stopWords) {
+  var filterStopWords = (0, _filters.createFilterStopWords)(stopWords);
+  var getFilters = (0, _filters.composeFilters)(_filterDouble["default"], _filterStemmer["default"], _filterSynonyms["default"], _filterToLower["default"], filterStopWords, _filterLength["default"]);
+  var FIELDS_TOKENIZER_MAP = createMapFieldsTokenizer(fields || [], getFilters);
+  return function (field, entity) {
+    var name = field.name;
+    var tokenizeIt = FIELDS_TOKENIZER_MAP[name];
+
+    if (name in entity) {
+      var value = entity[name];
+
+      if (value === undefined || value === null) {
+        return [];
+      }
+
+      return tokenizeIt("".concat(value));
+    }
+
+    return [];
+  };
+}
+
+function createEntityTokenizer(fields, stopWords) {
+  var tokenizeAll = createFilterTokens(fields, stopWords);
+  return function (entity) {
+    return fields.reduce(function (a, field) {
+      var name = field.name;
+      return _objectSpread(_objectSpread({}, a), {}, _defineProperty({}, name, tokenizeAll(field, entity)));
+    }, {});
+  };
+}
+
+var _default = createEntityTokenizer;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/create-fields-tokenizer.js

@@ -0,0 +1,103 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _prepareStringIndexation = _interopRequireDefault(require("./prepare-string-indexation"));
+
+var _softTokenizer = _interopRequireDefault(require("./soft-tokenizer"));
+
+var _stringTokenizer = _interopRequireDefault(require("string-tokenizer"));
+
+var _getRegexpFromPattern = _interopRequireDefault(require("./get-regexp-from-pattern"));
+
+var _removeAccents = _interopRequireDefault(require("remove-accents"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+
+function _slicedToArray(arr, i) { return _arrayWithHoles(arr) || _iterableToArrayLimit(arr, i) || _unsupportedIterableToArray(arr, i) || _nonIterableRest(); }
+
+function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function _iterableToArrayLimit(arr, i) { var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"]; if (_i == null) return; var _arr = []; var _n = true; var _d = false; var _s, _e; try { for (_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"] != null) _i["return"](); } finally { if (_d) throw _e; } } return _arr; }
+
+function _arrayWithHoles(arr) { if (Array.isArray(arr)) return arr; }
+
+function defaultTokenizeIt(string) {
+  return [(0, _prepareStringIndexation["default"])(string)];
+}
+
+function toArray(content) {
+  return Array.isArray(content) ? content : [content];
+}
+
+function tokensToArray(tokenized) {
+  return Object.entries(tokenized).reduce(function (a, _ref) {
+    var _ref2 = _slicedToArray(_ref, 2),
+        k = _ref2[0],
+        values = _ref2[1];
+
+    if (k.startsWith('pattern')) {
+      return [].concat(_toConsumableArray(a), _toConsumableArray(toArray(values)));
+    }
+
+    return a;
+  }, []);
+}
+
+function createFieldsTokenizer(field, filtersStack) {
+  var name = field.name,
+      _field$rules = field.rules,
+      rules = _field$rules === void 0 ? [] : _field$rules,
+      min = field.min,
+      _field$language = field.language,
+      language = _field$language === void 0 ? 'French' : _field$language,
+      _field$stemmer = field.stemmer,
+      stemmer = _field$stemmer === void 0 ? true : _field$stemmer,
+      _field$synonyms = field.synonyms,
+      synonyms = _field$synonyms === void 0 ? {} : _field$synonyms;
+
+  if (rules === 'soft') {
+    return _softTokenizer["default"];
+  }
+
+  if (rules.length) {
+    var tokenRules = rules.reduce(function (rulesMap, pattern, index) {
+      return _objectSpread(_objectSpread({}, rulesMap), {}, _defineProperty({}, "pattern".concat(name).concat(index), (0, _getRegexpFromPattern["default"])(pattern)));
+    }, {});
+    return function (string) {
+      var what = (0, _stringTokenizer["default"])().input((0, _removeAccents["default"])(string)).tokens(tokenRules).resolve();
+      return filtersStack(tokensToArray(what), {
+        min: min,
+        language: language,
+        stemmer: stemmer,
+        synonyms: synonyms
+      });
+    };
+  }
+
+  return defaultTokenizeIt;
+}
+
+var _default = createFieldsTokenizer;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/create-filter-stop-words.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _stopWords = _interopRequireDefault(require("./stop-words"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function createFilterStopWords() {
+  var stops = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : _stopWords["default"];
+  var mapSW = stops.reduce(function (a, w) {
+    return _objectSpread(_objectSpread({}, a), {}, _defineProperty({}, w, undefined));
+  }, {});
+  return function filter(tokens) {
+    return tokens.reduce(function (a, t) {
+      if (t in mapSW) {
+        return a;
+      }
+
+      return [].concat(_toConsumableArray(a), [t]);
+    }, []);
+  };
+}
+
+var _default = createFilterStopWords;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/compose-filters.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.composeFilters = composeFilters;
+exports["default"] = void 0;
+
+function composeFilters() {
+  for (var _len = arguments.length, filters = new Array(_len), _key = 0; _key < _len; _key++) {
+    filters[_key] = arguments[_key];
+  }
+
+  return filters.reverse().reduce(function (next, current) {
+    return function (tokens, args) {
+      return next(current(tokens, args), args);
+    };
+  }, function (t) {
+    return t;
+  });
+}
+
+var _default = composeFilters;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _stopWords = _interopRequireDefault(require("./stop-words"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function createFilterStopWords() {
+  var stops = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : _stopWords["default"];
+  var mapSW = stops.reduce(function (a, w) {
+    return _objectSpread(_objectSpread({}, a), {}, _defineProperty({}, w, undefined));
+  }, {});
+  return function filter(tokens) {
+    return tokens.reduce(function (a, t) {
+      if (t in mapSW) {
+        return a;
+      }
+
+      return [].concat(_toConsumableArray(a), [t]);
+    }, []);
+  };
+}
+
+var _default = createFilterStopWords;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.spec.js

@@ -0,0 +1,16 @@
+"use strict";
+
+var _createFilterStopWords = _interopRequireDefault(require("./create-filter-stop-words"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('create-filter-stop-words', function () {
+  it('only last ', function () {
+    var sw = ['un', 'de'];
+    var tokens = ['un', 'de', 'trois'];
+    var filter = (0, _createFilterStopWords["default"])(sw);
+    var results = filter(tokens);
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(1);
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-accents.js

@@ -0,0 +1,24 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _removeAccents = _interopRequireDefault(require("remove-accents"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function filterAccents() {
+  var tokens = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
+  return tokens.map(function (token) {
+    if (typeof token === 'string') {
+      return (0, _removeAccents["default"])(token).toLowerCase();
+    }
+
+    return token;
+  });
+}
+
+var _default = filterAccents;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-accents.spec.js

@@ -0,0 +1,18 @@
+"use strict";
+
+var _filterAccents = _interopRequireDefault(require("./filter-accents"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-length', function () {
+  it('maj accent', function () {
+    var tokens = ['UN', 'éà ù'];
+    var results = (0, _filterAccents["default"])(tokens, {
+      min: 3
+    });
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(2);
+    expect(results[0]).toBe('un');
+    expect(results[1]).toBe('ea u');
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-double.js

@@ -0,0 +1,34 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function filterDouble() {
+  var tokens = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
+  var map = {};
+  return tokens.reduce(function (a, token) {
+    if (token in map) {
+      return a;
+    }
+
+    map[token] = true;
+    return [].concat(_toConsumableArray(a), [token]);
+  }, []);
+}
+
+var _default = filterDouble;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-double.spec.js

@@ -0,0 +1,25 @@
+"use strict";
+
+var _filterDouble = _interopRequireDefault(require("./filter-double"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-double', function () {
+  it('empty', function () {
+    var tokens = ['un', 'un'];
+    var results = (0, _filterDouble["default"])(tokens);
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(1);
+    expect(results[0]).toBe('un');
+  });
+  it('remove one', function () {
+    var tokens = ['un', 'un', 'long'];
+    var results = (0, _filterDouble["default"])(tokens, {
+      min: 3
+    });
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(2);
+    expect(results[0]).toBe('un');
+    expect(results[1]).toBe('long');
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-length.js

@@ -0,0 +1,17 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+function filterLength(tokens, _ref) {
+  var _ref$min = _ref.min,
+      min = _ref$min === void 0 ? 2 : _ref$min;
+  return tokens.filter(function (token) {
+    return token.length >= min;
+  });
+}
+
+var _default = filterLength;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-length.spec.js

@@ -0,0 +1,25 @@
+"use strict";
+
+var _filterLength = _interopRequireDefault(require("./filter-length"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-length', function () {
+  it('empty', function () {
+    var tokens = ['un', 'no'];
+    var results = (0, _filterLength["default"])(tokens, {
+      min: 3
+    });
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(0);
+  });
+  it('remove one', function () {
+    var tokens = ['un', 'long'];
+    var results = (0, _filterLength["default"])(tokens, {
+      min: 3
+    });
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(1);
+    expect(results[0]).toBe('long');
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.js

@@ -0,0 +1,29 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+var _getStemmer = _interopRequireDefault(require("../get-stemmer"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+function filterStemmer(tokens, _ref) {
+  var _ref$language = _ref.language,
+      language = _ref$language === void 0 ? 'French' : _ref$language,
+      _ref$stemmer = _ref.stemmer,
+      stemmer = _ref$stemmer === void 0 ? true : _ref$stemmer;
+
+  if (stemmer) {
+    var stemming = (0, _getStemmer["default"])(language);
+    return tokens.map(function (token) {
+      return stemming(token);
+    });
+  }
+
+  return tokens;
+}
+
+var _default = filterStemmer;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.spec.js

@@ -0,0 +1,18 @@
+"use strict";
+
+var _filterStemmer = _interopRequireDefault(require("./filter-stemmer"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-stemmer', function () {
+  it('commerce', function () {
+    var tokens = ['commerce', 'commercial'];
+    var results = (0, _filterStemmer["default"])(tokens, {
+      language: 'French'
+    });
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(2);
+    expect(results[0]).toBe('commerc');
+    expect(results[1]).toBe('commercial');
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.js

@@ -0,0 +1,71 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+function _typeof(obj) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (obj) { return typeof obj; } : function (obj) { return obj && "function" == typeof Symbol && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }, _typeof(obj); }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { _defineProperty(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
+
+function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+var MAPS_FROM_ARRAY = new Map(); // Pour éviter de recalculer une map à chaque fois
+
+function filterFromObject(tokens, synonyms) {
+  return tokens.reduce(function (prec, token) {
+    if (token in synonyms) {
+      return [].concat(_toConsumableArray(prec), [token], _toConsumableArray(synonyms[token]));
+    }
+
+    return [].concat(_toConsumableArray(prec), [token]);
+  }, []);
+}
+
+function buildSynonymsMap(array) {
+  return array.reduce(function (map, _ref) {
+    var source = _ref.source,
+        target = _ref.target;
+    return _objectSpread(_objectSpread({}, map), {}, _defineProperty({}, source, target));
+  }, {});
+}
+
+function filterFromArray(tokens, synonyms) {
+  if (!MAPS_FROM_ARRAY.has(synonyms)) {
+    MAPS_FROM_ARRAY.set(synonyms, buildSynonymsMap(synonyms));
+  }
+
+  return filterFromObject(tokens, MAPS_FROM_ARRAY.get(synonyms));
+}
+
+function filterSynonyms(tokens, _ref2) {
+  var synonyms = _ref2.synonyms;
+
+  if (Array.isArray(synonyms) && synonyms.length) {
+    return filterFromArray(tokens, synonyms);
+  }
+
+  if (_typeof(synonyms) === 'object') {
+    return filterFromObject(tokens, synonyms);
+  }
+
+  return tokens;
+}
+
+var _default = filterSynonyms;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.spec.js

@@ -0,0 +1,15 @@
+"use strict";
+
+var _filterSynonyms = _interopRequireDefault(require("./filter-synonyms"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-synonyms', function () {
+  it('', function () {// const tokens = ['saint', 'sainte'];
+    // const results = filterStemmer(tokens, { language: 'French' });
+    // expect(Array.isArray(results)).toBe(true);
+    // expect(results.length).toBe(2);
+    // expect(results[0]).toBe('commerc');
+    // expect(results[1]).toBe('commercial');
+  });
+});

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports["default"] = void 0;
+
+function filterToLower(tokens) {
+  return tokens.map(function (token) {
+    if (typeof token === 'string') {
+      return token.toLocaleLowerCase();
+    }
+
+    return token;
+  });
+}
+
+var _default = filterToLower;
+exports["default"] = _default;

package/lib/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.spec.js

@@ -0,0 +1,16 @@
+"use strict";
+
+var _filterToLower = _interopRequireDefault(require("./filter-to-lower"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+
+describe('filter-to-lower', function () {
+  it('UN', function () {
+    var tokens = ['UN', 'no'];
+    var results = (0, _filterToLower["default"])(tokens);
+    expect(Array.isArray(results)).toBe(true);
+    expect(results.length).toBe(2);
+    expect(results[0]).toBe('un');
+    expect(results[1]).toBe('no');
+  });
+});