@inseefr/lunatic 0.2.3-experimental → 0.2.3-prisme

package/src/utils/suggester-workers/commons-tokenizer/create-entity-tokenizer.js

@@ -0,0 +1,56 @@
+import createFieldsTokenizer from './create-fields-tokenizer';
+import { composeFilters, createFilterStopWords } from './filters';
+import filterStemmer from './filters/filter-stemmer';
+import filterLength from './filters/filter-length';
+import filterSynonyms from './filters/filter-synonyms';
+import filterAccents from './filters/filter-accents';
+import filterToLower from './filters/filter-to-lower';
+import filterDouble from './filters/filter-double';
+
+function createMapFieldsTokenizer(fields, filters) {
+	return fields.reduce(function (mapFieldTokenizers, field) {
+		const { name } = field;
+		return {
+			...mapFieldTokenizers,
+			[name]: createFieldsTokenizer(field, filters),
+		};
+	}, {});
+}
+
+function createFilterTokens(fields, stopWords) {
+	const filterStopWords = createFilterStopWords(stopWords);
+	const getFilters = composeFilters(
+		filterDouble,
+		filterAccents,
+		filterStemmer,
+		filterSynonyms,
+		filterToLower,
+		filterStopWords,
+		filterLength
+	);
+	const FIELDS_TOKENIZER_MAP = createMapFieldsTokenizer(
+		fields || [],
+		getFilters
+	);
+
+	return function (field, entity) {
+		const { name } = field;
+		const tokenizeIt = FIELDS_TOKENIZER_MAP[name];
+		if (name in entity) {
+			return tokenizeIt(`${entity[name]}`);
+		}
+		return [];
+	};
+}
+
+function createEntityTokenizer(fields, stopWords) {
+	const tokenizeAll = createFilterTokens(fields, stopWords);
+	return function (entity) {
+		return fields.reduce(function (a, field) {
+			const { name } = field;
+			return { ...a, [name]: tokenizeAll(field, entity) };
+		}, {});
+	};
+}
+
+export default createEntityTokenizer;

package/src/utils/suggester-workers/commons-tokenizer/create-fields-tokenizer.js

@@ -0,0 +1,56 @@
+import prepareStringIndexation from './prepare-string-indexation';
+import softTokenizer from './soft-tokenizer';
+import tokenizer from 'string-tokenizer';
+import getRegExpFromPattern from './get-regexp-from-pattern';
+
+function defaultTokenizeIt(string) {
+	return [prepareStringIndexation(string)];
+}
+
+function toArray(content) {
+	return Array.isArray(content) ? content : [content];
+}
+
+function tokensToArray(tokenized) {
+	return Object.entries(tokenized).reduce(function (a, [k, values]) {
+		if (k.startsWith('pattern')) {
+			return [...a, ...toArray(values)];
+		}
+		return a;
+	}, []);
+}
+
+function createFieldsTokenizer(field, filtersStack) {
+	const {
+		name,
+		rules = [],
+		min,
+		language = 'French',
+		stemmer = true,
+		synonyms = {},
+	} = field;
+	if (rules === 'soft') {
+		return softTokenizer;
+	}
+	if (rules.length) {
+		const tokenRules = rules.reduce(function (rulesMap, pattern, index) {
+			return {
+				...rulesMap,
+				[`pattern${name}${index}`]: getRegExpFromPattern(pattern),
+			};
+		}, {});
+
+		return function (string) {
+			const what = tokenizer().input(string).tokens(tokenRules).resolve();
+			return filtersStack(tokensToArray(what), {
+				min,
+				language,
+				stemmer,
+				synonyms,
+			});
+		};
+	}
+	return defaultTokenizeIt;
+}
+
+export default createFieldsTokenizer;

package/src/utils/suggester-workers/commons-tokenizer/create-filter-stop-words.js

@@ -1,17 +1,17 @@
-import defaultStopWords from './stop-words';
-
-function createFilterStopWords(stops = defaultStopWords) {
-  const mapSW = stops.reduce(function (a, w) {
-    return { ...a, [w]: undefined };
-  }, {});
-  return function filter(tokens) {
-    return tokens.reduce(function (a, t) {
-      if (t in mapSW) {
-        return a;
-      }
-      return [...a, t];
-    }, []);
-  };
-}
-
-export default createFilterStopWords;
+import defaultStopWords from './stop-words';
+
+function createFilterStopWords(stops = defaultStopWords) {
+	const mapSW = stops.reduce(function (a, w) {
+		return { ...a, [w]: undefined };
+	}, {});
+	return function filter(tokens) {
+		return tokens.reduce(function (a, t) {
+			if (t in mapSW) {
+				return a;
+			}
+			return [...a, t];
+		}, []);
+	};
+}
+
+export default createFilterStopWords;

package/src/utils/suggester-workers/commons-tokenizer/filters/compose-filters.js

@@ -0,0 +1,10 @@
+export function composeFilters(...filters) {
+	return filters.reduce(
+		function (next, current) {
+			return (tokens, args) => next(current(tokens, args), args);
+		},
+		(t) => t
+	);
+}
+
+export default composeFilters;

package/src/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.js

@@ -0,0 +1,17 @@
+import defaultStopWords from './stop-words';
+
+function createFilterStopWords(stops = defaultStopWords) {
+	const mapSW = stops.reduce(function (a, w) {
+		return { ...a, [w]: undefined };
+	}, {});
+	return function filter(tokens) {
+		return tokens.reduce(function (a, t) {
+			if (t in mapSW) {
+				return a;
+			}
+			return [...a, t];
+		}, []);
+	};
+}
+
+export default createFilterStopWords;

package/src/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.spec.js

@@ -0,0 +1,14 @@
+import createFilterSW from './create-filter-stop-words';
+
+describe('create-filter-stop-words', function () {
+	it('only last ', function () {
+		const sw = ['un', 'de'];
+		const tokens = ['un', 'de', 'trois'];
+
+		const filter = createFilterSW(sw);
+
+		const results = filter(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-accents.js

@@ -0,0 +1,12 @@
+import removeAccents from 'remove-accents';
+
+function filterAccents(tokens = []) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return removeAccents(token).toLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterAccents;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-accents.spec.js

@@ -0,0 +1,12 @@
+import filterATL from './filter-accents';
+
+describe('filter-length', function () {
+	it('maj accent', function () {
+		const tokens = ['UN', 'éà ù'];
+		const results = filterATL(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('ea u');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-double.js → filters/filter-double.js}

@@ -1,12 +1,12 @@
-function filterDouble(tokens = []) {
-	const map = {};
-	return tokens.reduce(function (a, token) {
-		if (token in map) {
-			return a;
-		}
-		map[token] = true;
-		return [...a, token];
-	}, []);
-}
-
-export default filterDouble;
+function filterDouble(tokens = []) {
+	const map = {};
+	return tokens.reduce(function (a, token) {
+		if (token in map) {
+			return a;
+		}
+		map[token] = true;
+		return [...a, token];
+	}, []);
+}
+
+export default filterDouble;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-double.spec.js

@@ -0,0 +1,20 @@
+import filterDouble from './filter-double';
+
+describe('filter-double', function () {
+	it('empty', function () {
+		const tokens = ['un', 'un'];
+		const results = filterDouble(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('un');
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'un', 'long'];
+		const results = filterDouble(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-length.js → filters/filter-length.js}

@@ -1,7 +1,7 @@
-function filterLength(tokens, { min = 2 }) {
-	return tokens.filter(function (token) {
-		return token.length >= min;
-	});
-}
-
-export default filterLength;
+function filterLength(tokens, { min = 2 }) {
+	return tokens.filter(function (token) {
+		return token.length >= min;
+	});
+}
+
+export default filterLength;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-length.spec.js

@@ -0,0 +1,18 @@
+import filterLength from './filter-length';
+
+describe('filter-length', function () {
+	it('empty', function () {
+		const tokens = ['un', 'no'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(0);
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'long'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-stemmer.js → filters/filter-stemmer.js}

@@ -1,13 +1,13 @@
-import getStemmer from './get-stemmer';
-
-function filterStemmer(tokens, { language, stemmer = true }) {
-	if (stemmer) {
-		const stemming = getStemmer(language);
-		return tokens.map(function (token) {
-			return stemming(token);
-		});
-	}
-	return tokens;
-}
-
-export default filterStemmer;
+import getStemmer from '../get-stemmer';
+
+function filterStemmer(tokens, { language = 'French', stemmer = true }) {
+	if (stemmer) {
+		const stemming = getStemmer(language);
+		return tokens.map(function (token) {
+			return stemming(token);
+		});
+	}
+	return tokens;
+}
+
+export default filterStemmer;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.spec.js

@@ -0,0 +1,12 @@
+import filterStemmer from './filter-stemmer';
+
+describe('filter-stemmer', function () {
+	it('commerce', function () {
+		const tokens = ['commerce', 'commercial'];
+		const results = filterStemmer(tokens, { language: 'French' });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('commerc');
+		expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.js

@@ -0,0 +1,36 @@
+const MAPS_FROM_ARRAY = new Map(); // Pour éviter de recalculer une map à chaque fois
+
+function filterFromObject(tokens, synonyms) {
+	return tokens.reduce(function (prec, token) {
+		if (token in synonyms) {
+			return [...prec, token, ...synonyms[token]];
+		}
+		return [...prec, token];
+	}, []);
+}
+
+function buildSynonymsMap(array) {
+	return array.reduce(function (map, { source, target }) {
+		return { ...map, [source]: target };
+	}, {});
+}
+
+function filterFromArray(tokens, synonyms) {
+	if (!MAPS_FROM_ARRAY.has(synonyms)) {
+		MAPS_FROM_ARRAY.set(synonyms, buildSynonymsMap(synonyms));
+	}
+	return filterFromObject(tokens, MAPS_FROM_ARRAY.get(synonyms));
+}
+
+function filterSynonyms(tokens, { synonyms }) {
+	if (Array.isArray(synonyms) && synonyms.length) {
+		return filterFromArray(tokens, synonyms);
+	}
+	if (typeof synonyms === 'object') {
+		return filterFromObject(tokens, synonyms);
+	}
+
+	return tokens;
+}
+
+export default filterSynonyms;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.spec.js

@@ -0,0 +1,12 @@
+import filterSynonyms from './filter-synonyms';
+
+describe('filter-synonyms', function () {
+	it('', function () {
+		// const tokens = ['saint', 'sainte'];
+		// const results = filterStemmer(tokens, { language: 'French' });
+		// expect(Array.isArray(results)).toBe(true);
+		// expect(results.length).toBe(2);
+		// expect(results[0]).toBe('commerc');
+		// expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.js

@@ -0,0 +1,10 @@
+function filterToLower(tokens) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return token.toLocaleLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterToLower;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.spec.js

@@ -0,0 +1,12 @@
+import filterToLower from './filter-to-lower';
+
+describe('filter-to-lower', function () {
+	it('UN', function () {
+		const tokens = ['UN', 'no'];
+		const results = filterToLower(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('no');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/index.js

@@ -0,0 +1,2 @@
+export { default as composeFilters } from './compose-filters';
+export { default as createFilterStopWords } from './create-filter-stop-words';

package/src/utils/suggester-workers/commons-tokenizer/{stop-words.js → filters/stop-words.js}

@@ -1,118 +1,118 @@
-const STOP_WORDS = [
-  'alors',
-  'au',
-  'aucuns',
-  'aussi',
-  'autre',
-  'avant',
-  'avec',
-  'avoir',
-  'bon',
-  'car',
-  'ce',
-  'cela',
-  'ces',
-  'ceux',
-  'chaque',
-  'ci',
-  'comme',
-  'comment',
-  'dans',
-  'des',
-  'du',
-  'dedans',
-  'dehors',
-  'depuis',
-  'devrait',
-  'doit',
-  'donc',
-  'dos',
-  'debut',
-  'elle',
-  'elles',
-  'en',
-  'encore',
-  'essai',
-  'est',
-  'et',
-  'eu',
-  'fait',
-  'faites',
-  'fois',
-  'font',
-  'hors',
-  'ici',
-  'il',
-  'ils',
-  'je',
-  'juste',
-  'le',
-  'les',
-  'leur',
-  'la',
-  'ma',
-  'maintenant',
-  'mais',
-  'mes',
-  'mien',
-  'moins',
-  'mon',
-  'mot',
-  'meme',
-  'ni',
-  'nommes',
-  'notre',
-  'nous',
-  'ou',
-  'par',
-  'parce',
-  'pas',
-  'peut',
-  'peu',
-  'plupart',
-  'pour',
-  'pourquoi',
-  'quand',
-  'que',
-  'quel',
-  'quelle',
-  'quelles',
-  'quels',
-  'qui',
-  'sa',
-  'sans',
-  'ses',
-  'seulement',
-  'si',
-  'sien',
-  'son',
-  'sont',
-  'sous',
-  'soyez',
-  'sois',
-  'sujet',
-  'sur',
-  'ta',
-  'tandis',
-  'tellement',
-  'tels',
-  'tes',
-  'ton',
-  'tous',
-  'tout',
-  'trop',
-  'tres',
-  'tu',
-  'voient',
-  'vont',
-  'votre',
-  'vous',
-  'vu',
-  'ca',
-  'etaient',
-  'etat',
-  'etions',
-  'ete',
-  'etre',
-];
-export default STOP_WORDS;
+const STOP_WORDS = [
+  'alors',
+  'au',
+  'aucuns',
+  'aussi',
+  'autre',
+  'avant',
+  'avec',
+  'avoir',
+  'bon',
+  'car',
+  'ce',
+  'cela',
+  'ces',
+  'ceux',
+  'chaque',
+  'ci',
+  'comme',
+  'comment',
+  'dans',
+  'des',
+  'du',
+  'dedans',
+  'dehors',
+  'depuis',
+  'devrait',
+  'doit',
+  'donc',
+  'dos',
+  'debut',
+  'elle',
+  'elles',
+  'en',
+  'encore',
+  'essai',
+  'est',
+  'et',
+  'eu',
+  'fait',
+  'faites',
+  'fois',
+  'font',
+  'hors',
+  'ici',
+  'il',
+  'ils',
+  'je',
+  'juste',
+  'le',
+  'les',
+  'leur',
+  'la',
+  'ma',
+  'maintenant',
+  'mais',
+  'mes',
+  'mien',
+  'moins',
+  'mon',
+  'mot',
+  'meme',
+  'ni',
+  'nommes',
+  'notre',
+  'nous',
+  'ou',
+  'par',
+  'parce',
+  'pas',
+  'peut',
+  'peu',
+  'plupart',
+  'pour',
+  'pourquoi',
+  'quand',
+  'que',
+  'quel',
+  'quelle',
+  'quelles',
+  'quels',
+  'qui',
+  'sa',
+  'sans',
+  'ses',
+  'seulement',
+  'si',
+  'sien',
+  'son',
+  'sont',
+  'sous',
+  'soyez',
+  'sois',
+  'sujet',
+  'sur',
+  'ta',
+  'tandis',
+  'tellement',
+  'tels',
+  'tes',
+  'ton',
+  'tous',
+  'tout',
+  'trop',
+  'tres',
+  'tu',
+  'voient',
+  'vont',
+  'votre',
+  'vous',
+  'vu',
+  'ca',
+  'etaient',
+  'etat',
+  'etions',
+  'ete',
+  'etre',
+];
+export default STOP_WORDS;

package/src/utils/suggester-workers/commons-tokenizer/get-regexp-from-pattern.js

@@ -1,8 +1,8 @@
-function get(pattern) {
-  if (typeof pattern === 'string') {
-    return new RegExp(pattern);
-  }
-  return pattern;
-}
-
-export default get;
+function get(pattern) {
+  if (typeof pattern === 'string') {
+    return new RegExp(pattern);
+  }
+  return pattern;
+}
+
+export default get;