@inseefr/lunatic 0.2.2-experimental → 0.2.2-prisme

package/src/utils/store-tools/open-or-create-store.js

@@ -1,47 +1,47 @@
-/* eslint-disable no-restricted-globals */
-import CONSTANTE from './constantes';
-import { getIDB } from '../idb-tools';
-
-const IDB_REF = getIDB();
-
-function openStorage(name, idbVersion = 1) {
-	return new Promise(function (resolve, reject) {
-		if (!IDB_REF) {
-			reject('indexedDb not supported !');
-		}
-		const request = IDB_REF.open(name, idbVersion);
-		let db;
-		let doIt = true;
-
-		request.onupgradeneeded = function (e) {
-			doIt = false;
-			db = e.target.result;
-			const store = db.createObjectStore(CONSTANTE.STORE_DATA_NAME, {
-				keyPath: 'id',
-			});
-			db.createObjectStore(CONSTANTE.STORE_INFO_NAME, {
-				keyPath: 'name',
-			});
-			store.createIndex(CONSTANTE.STORE_INDEX_NAME, 'tokens', {
-				multiEntry: true,
-			});
-
-			const txn = e.target.transaction;
-			txn.oncomplete = function () {
-				resolve(db);
-			};
-		};
-
-		request.onsuccess = function () {
-			if (doIt) {
-				resolve(request.result);
-			}
-		};
-
-		request.onerror = function (e) {
-			reject(e);
-		};
-	});
-}
-
-export default openStorage;
+/* eslint-disable no-restricted-globals */
+import CONSTANTE from './constantes';
+import { getIDB } from '../idb-tools';
+
+const IDB_REF = getIDB();
+
+function openStorage(name, idbVersion = 1) {
+	return new Promise(function (resolve, reject) {
+		if (!IDB_REF) {
+			reject('indexedDb not supported !');
+		}
+		const request = IDB_REF.open(name, idbVersion);
+		let db;
+		let doIt = true;
+
+		request.onupgradeneeded = function (e) {
+			doIt = false;
+			db = e.target.result;
+			const store = db.createObjectStore(CONSTANTE.STORE_DATA_NAME, {
+				keyPath: 'id',
+			});
+			db.createObjectStore(CONSTANTE.STORE_INFO_NAME, {
+				keyPath: 'name',
+			});
+			store.createIndex(CONSTANTE.STORE_INDEX_NAME, 'tokens', {
+				multiEntry: true,
+			});
+
+			const txn = e.target.transaction;
+			txn.oncomplete = function () {
+				resolve(db);
+			};
+		};
+
+		request.onsuccess = function () {
+			if (doIt) {
+				resolve(request.result);
+			}
+		};
+
+		request.onerror = function (e) {
+			reject(e);
+		};
+	});
+}
+
+export default openStorage;

package/src/utils/store-tools/use-store-index.js

@@ -1,24 +1,24 @@
-import { useEffect, useState } from 'react';
-import createStore from './create';
-
-function useStoreIndex(storeInfo, idbVersion) {
-	const [db, setDb] = useState(undefined);
-
-	useEffect(
-		function () {
-			async function init() {
-				if (storeInfo && idbVersion) {
-					const db = await createStore(storeInfo, idbVersion);
-					setDb(db);
-				}
-			}
-
-			init();
-			return function () {};
-		},
-		[storeInfo, idbVersion]
-	);
-	return db;
-}
-
-export default useStoreIndex;
+import { useEffect, useState } from 'react';
+import createStore from './create';
+
+function useStoreIndex(storeInfo, idbVersion) {
+	const [db, setDb] = useState(undefined);
+
+	useEffect(
+		function () {
+			async function init() {
+				if (storeInfo && idbVersion) {
+					const db = await createStore(storeInfo, idbVersion);
+					setDb(db);
+				}
+			}
+
+			init();
+			return function () {};
+		},
+		[storeInfo, idbVersion]
+	);
+	return db;
+}
+
+export default useStoreIndex;

package/src/utils/suggester-workers/append-to-index/create-append-task.js

@@ -1,3 +1,5 @@
+import { createWorker } from '../create-worker';
+
 const workerPath =
 	process.env.LUNATIC_LOADER_WORKER_PATH ||
 	process.env.REACT_APP_LUNATIC_LOADER_WORKER_PATH ||
@@ -8,7 +10,7 @@ const workerPath =
  */
 function task(info, version, log = () => null) {
 	const { name, fields, stopWords } = info;
-	const worker = new Worker(workerPath);
+	const worker = createWorker(workerPath);
 	let start = false;
 	let stop = false;
 

package/src/utils/suggester-workers/commons-tokenizer/create-entity-tokenizer.js

@@ -0,0 +1,56 @@
+import createFieldsTokenizer from './create-fields-tokenizer';
+import { composeFilters, createFilterStopWords } from './filters';
+import filterStemmer from './filters/filter-stemmer';
+import filterLength from './filters/filter-length';
+import filterSynonyms from './filters/filter-synonyms';
+import filterAccents from './filters/filter-accents';
+import filterToLower from './filters/filter-to-lower';
+import filterDouble from './filters/filter-double';
+
+function createMapFieldsTokenizer(fields, filters) {
+	return fields.reduce(function (mapFieldTokenizers, field) {
+		const { name } = field;
+		return {
+			...mapFieldTokenizers,
+			[name]: createFieldsTokenizer(field, filters),
+		};
+	}, {});
+}
+
+function createFilterTokens(fields, stopWords) {
+	const filterStopWords = createFilterStopWords(stopWords);
+	const getFilters = composeFilters(
+		filterDouble,
+		filterAccents,
+		filterStemmer,
+		filterSynonyms,
+		filterToLower,
+		filterStopWords,
+		filterLength
+	);
+	const FIELDS_TOKENIZER_MAP = createMapFieldsTokenizer(
+		fields || [],
+		getFilters
+	);
+
+	return function (field, entity) {
+		const { name } = field;
+		const tokenizeIt = FIELDS_TOKENIZER_MAP[name];
+		if (name in entity) {
+			return tokenizeIt(`${entity[name]}`);
+		}
+		return [];
+	};
+}
+
+function createEntityTokenizer(fields, stopWords) {
+	const tokenizeAll = createFilterTokens(fields, stopWords);
+	return function (entity) {
+		return fields.reduce(function (a, field) {
+			const { name } = field;
+			return { ...a, [name]: tokenizeAll(field, entity) };
+		}, {});
+	};
+}
+
+export default createEntityTokenizer;

package/src/utils/suggester-workers/commons-tokenizer/create-fields-tokenizer.js

@@ -0,0 +1,56 @@
+import prepareStringIndexation from './prepare-string-indexation';
+import softTokenizer from './soft-tokenizer';
+import tokenizer from 'string-tokenizer';
+import getRegExpFromPattern from './get-regexp-from-pattern';
+
+function defaultTokenizeIt(string) {
+	return [prepareStringIndexation(string)];
+}
+
+function toArray(content) {
+	return Array.isArray(content) ? content : [content];
+}
+
+function tokensToArray(tokenized) {
+	return Object.entries(tokenized).reduce(function (a, [k, values]) {
+		if (k.startsWith('pattern')) {
+			return [...a, ...toArray(values)];
+		}
+		return a;
+	}, []);
+}
+
+function createFieldsTokenizer(field, filtersStack) {
+	const {
+		name,
+		rules = [],
+		min,
+		language = 'French',
+		stemmer = true,
+		synonyms = {},
+	} = field;
+	if (rules === 'soft') {
+		return softTokenizer;
+	}
+	if (rules.length) {
+		const tokenRules = rules.reduce(function (rulesMap, pattern, index) {
+			return {
+				...rulesMap,
+				[`pattern${name}${index}`]: getRegExpFromPattern(pattern),
+			};
+		}, {});
+
+		return function (string) {
+			const what = tokenizer().input(string).tokens(tokenRules).resolve();
+			return filtersStack(tokensToArray(what), {
+				min,
+				language,
+				stemmer,
+				synonyms,
+			});
+		};
+	}
+	return defaultTokenizeIt;
+}
+
+export default createFieldsTokenizer;

package/src/utils/suggester-workers/commons-tokenizer/create-filter-stop-words.js

@@ -1,17 +1,17 @@
 import defaultStopWords from './stop-words';
 
 function createFilterStopWords(stops = defaultStopWords) {
-  const mapSW = stops.reduce(function (a, w) {
-    return { ...a, [w]: undefined };
-  }, {});
-  return function filter(tokens) {
-    return tokens.reduce(function (a, t) {
-      if (t in mapSW) {
-        return a;
-      }
-      return [...a, t];
-    }, []);
-  };
+	const mapSW = stops.reduce(function (a, w) {
+		return { ...a, [w]: undefined };
+	}, {});
+	return function filter(tokens) {
+		return tokens.reduce(function (a, t) {
+			if (t in mapSW) {
+				return a;
+			}
+			return [...a, t];
+		}, []);
+	};
 }
 
 export default createFilterStopWords;

package/src/utils/suggester-workers/commons-tokenizer/filters/compose-filters.js

@@ -0,0 +1,10 @@
+export function composeFilters(...filters) {
+	return filters.reduce(
+		function (next, current) {
+			return (tokens, args) => next(current(tokens, args), args);
+		},
+		(t) => t
+	);
+}
+
+export default composeFilters;

package/src/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.js

@@ -0,0 +1,17 @@
+import defaultStopWords from './stop-words';
+
+function createFilterStopWords(stops = defaultStopWords) {
+	const mapSW = stops.reduce(function (a, w) {
+		return { ...a, [w]: undefined };
+	}, {});
+	return function filter(tokens) {
+		return tokens.reduce(function (a, t) {
+			if (t in mapSW) {
+				return a;
+			}
+			return [...a, t];
+		}, []);
+	};
+}
+
+export default createFilterStopWords;

package/src/utils/suggester-workers/commons-tokenizer/filters/create-filter-stop-words.spec.js

@@ -0,0 +1,14 @@
+import createFilterSW from './create-filter-stop-words';
+
+describe('create-filter-stop-words', function () {
+	it('only last ', function () {
+		const sw = ['un', 'de'];
+		const tokens = ['un', 'de', 'trois'];
+
+		const filter = createFilterSW(sw);
+
+		const results = filter(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-accents.js

@@ -0,0 +1,12 @@
+import removeAccents from 'remove-accents';
+
+function filterAccents(tokens = []) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return removeAccents(token).toLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterAccents;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-accents.spec.js

@@ -0,0 +1,12 @@
+import filterATL from './filter-accents';
+
+describe('filter-length', function () {
+	it('maj accent', function () {
+		const tokens = ['UN', 'éà ù'];
+		const results = filterATL(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('ea u');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-double.spec.js

@@ -0,0 +1,20 @@
+import filterDouble from './filter-double';
+
+describe('filter-double', function () {
+	it('empty', function () {
+		const tokens = ['un', 'un'];
+		const results = filterDouble(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('un');
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'un', 'long'];
+		const results = filterDouble(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-length.spec.js

@@ -0,0 +1,18 @@
+import filterLength from './filter-length';
+
+describe('filter-length', function () {
+	it('empty', function () {
+		const tokens = ['un', 'no'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(0);
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'long'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-stemmer.js → filters/filter-stemmer.js}

@@ -1,6 +1,6 @@
-import getStemmer from './get-stemmer';
+import getStemmer from '../get-stemmer';
 
-function filterStemmer(tokens, { language, stemmer = true }) {
+function filterStemmer(tokens, { language = 'French', stemmer = true }) {
 	if (stemmer) {
 		const stemming = getStemmer(language);
 		return tokens.map(function (token) {

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.spec.js

@@ -0,0 +1,12 @@
+import filterStemmer from './filter-stemmer';
+
+describe('filter-stemmer', function () {
+	it('commerce', function () {
+		const tokens = ['commerce', 'commercial'];
+		const results = filterStemmer(tokens, { language: 'French' });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('commerc');
+		expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.js

@@ -0,0 +1,36 @@
+const MAPS_FROM_ARRAY = new Map(); // Pour éviter de recalculer une map à chaque fois
+
+function filterFromObject(tokens, synonyms) {
+	return tokens.reduce(function (prec, token) {
+		if (token in synonyms) {
+			return [...prec, token, ...synonyms[token]];
+		}
+		return [...prec, token];
+	}, []);
+}
+
+function buildSynonymsMap(array) {
+	return array.reduce(function (map, { source, target }) {
+		return { ...map, [source]: target };
+	}, {});
+}
+
+function filterFromArray(tokens, synonyms) {
+	if (!MAPS_FROM_ARRAY.has(synonyms)) {
+		MAPS_FROM_ARRAY.set(synonyms, buildSynonymsMap(synonyms));
+	}
+	return filterFromObject(tokens, MAPS_FROM_ARRAY.get(synonyms));
+}
+
+function filterSynonyms(tokens, { synonyms }) {
+	if (Array.isArray(synonyms) && synonyms.length) {
+		return filterFromArray(tokens, synonyms);
+	}
+	if (typeof synonyms === 'object') {
+		return filterFromObject(tokens, synonyms);
+	}
+
+	return tokens;
+}
+
+export default filterSynonyms;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.spec.js

@@ -0,0 +1,12 @@
+import filterSynonyms from './filter-synonyms';
+
+describe('filter-synonyms', function () {
+	it('', function () {
+		// const tokens = ['saint', 'sainte'];
+		// const results = filterStemmer(tokens, { language: 'French' });
+		// expect(Array.isArray(results)).toBe(true);
+		// expect(results.length).toBe(2);
+		// expect(results[0]).toBe('commerc');
+		// expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.js

@@ -0,0 +1,10 @@
+function filterToLower(tokens) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return token.toLocaleLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterToLower;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.spec.js

@@ -0,0 +1,12 @@
+import filterToLower from './filter-to-lower';
+
+describe('filter-to-lower', function () {
+	it('UN', function () {
+		const tokens = ['UN', 'no'];
+		const results = filterToLower(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('no');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/index.js

@@ -0,0 +1,2 @@
+export { default as composeFilters } from './compose-filters';
+export { default as createFilterStopWords } from './create-filter-stop-words';

package/src/utils/suggester-workers/commons-tokenizer/index.js

@@ -1,8 +1,9 @@
 export { default as getStemmer } from './get-stemmer';
-export { default as createTokenizer } from './create-tokenizer';
+export { default as createTokenizer } from './create-entity-tokenizer';
 export { default as prepareStringIndexation } from './prepare-string-indexation';
-export { default as filterStemmer } from './filter-stemmer';
-export { default as filterLength } from './filter-length';
-export { default as filterDouble } from './filter-double';
-export { default as filterAccentsToLower } from './filter-accents-to-lower';
+export { default as filterStemmer } from './filters/filter-stemmer';
+export { default as filterLength } from './filters/filter-length';
+export { default as filterDouble } from './filters/filter-double';
+export { default as filterAccents } from './filters/filter-accents';
+export { default as filterToLower } from './filters/filter-to-lower';
 export { default as getRegExpFromPattern } from './get-regexp-from-pattern';

package/src/utils/suggester-workers/commons-tokenizer/prepare-string-indexation.js

@@ -1,9 +1,13 @@
 import removeAccents from 'remove-accents';
 
 function prepare(token = '', replacement = '') {
-	return typeof token === 'string'
-		? removeAccents(token.toLowerCase()).replace(/[- ']/g, replacement)
-		: token;
+	if (typeof token === 'string') {
+		return removeAccents(token.trim().toLowerCase()).replace(
+			/[- ']/g,
+			replacement
+		);
+	}
+	return token;
 }
 
 export default prepare;

package/src/utils/suggester-workers/commons-tokenizer/soft-tokenizer.js

@@ -1,4 +1,4 @@
-import prepareStringIndexation from './prepare-string-indexation';
+import { prepareStringIndexation } from '../commons-tokenizer';
 
 function softTokenizer(string) {
 	return [prepareStringIndexation(string, '-')];

package/src/utils/suggester-workers/create-worker.js

@@ -0,0 +1,56 @@
+// Enable cross domain workers
+
+const testSameOrigin = (url) => {
+	var loc = window.location;
+	var a = document.createElement('a');
+	a.href = url;
+	return (
+		a.hostname === loc.hostname &&
+		a.port === loc.port &&
+		a.protocol === loc.protocol
+	);
+};
+
+const createWorkerFallback = (workerUrl) => {
+	console.log('Create worker for MFE');
+	var worker = null;
+	try {
+		var blob;
+		try {
+			blob = new Blob([`importScripts('${workerUrl}');`], {
+				type: 'application/javascript',
+			});
+		} catch (e) {
+			var blobBuilder = new (window.BlobBuilder ||
+				window.WebKitBlobBuilder ||
+				window.MozBlobBuilder)();
+			blobBuilder.append(`importScripts('${workerUrl}');`);
+			blob = blobBuilder.getBlob('application/javascript');
+		}
+		var url = window.URL || window.webkitURL;
+		var blobUrl = url.createObjectURL(blob);
+		worker = new Worker(blobUrl);
+	} catch (e1) {
+		console.error(`Lunatic-worker : Failed to load web worker : ${workerUrl}`);
+	}
+	return worker;
+};
+
+export const createWorker = (workerUrl) => {
+	var worker = null;
+	try {
+		if (testSameOrigin(workerUrl)) {
+			console.log('Trying create classic worker');
+			worker = new Worker(workerUrl);
+			worker.onerror = (event) => {
+				event.preventDefault();
+				worker = createWorkerFallback(workerUrl);
+			};
+		} else {
+			worker = createWorkerFallback(workerUrl);
+		}
+	} catch (e) {
+		worker = createWorkerFallback(workerUrl);
+	}
+	return worker;
+};

package/src/utils/suggester-workers/find-best-label/tokenize.js

@@ -1,5 +1,4 @@
 import tokenizer from 'string-tokenizer';
-import removeAccents from 'remove-accents';
 
 function toArray(tokens) {
 	if (Array.isArray(tokens)) {
@@ -12,13 +11,11 @@ function tokenize(object, pattern = /[\w]+/) {
 	if (typeof object === 'object') {
 		const map = {};
 		Object.entries(object).forEach(function ([key, string]) {
-			const input = removeAccents(string).toLowerCase().trim();
-			if (input.length) {
+			if (string.length) {
 				const { tokens } = tokenizer()
-					.input(removeAccents(string).toLowerCase())
+					.input(string)
 					.tokens({ tokens: pattern })
 					.resolve();
-
 				map[key] = toArray(tokens);
 			} else {
 				map[key] = [];

package/src/utils/suggester-workers/searching/query-parser/query-parser-soft.js

@@ -0,0 +1,7 @@
+import { prepareStringIndexation } from '../../commons-tokenizer';
+
+function queryParserSoft(string) {
+	return [prepareStringIndexation(string, '-')];
+}
+
+export default queryParserSoft;