@inseefr/lunatic 0.2.62 → 0.3.0-v2

package/src/utils/idb-tools/idb-bulk-insert.js

@@ -0,0 +1,96 @@
+const BULK_LIMIT = 300;
+
+export const BULK_INSERT_MESSAGES = {
+	bulkInsertComplete: { type: 'bulk-insert/complete' },
+	bulkInsertFinished: { type: 'bulk-insert/finished' },
+	bulkInsertError: { type: 'bulk-insert/error' },
+};
+
+/* */
+const pushTask = (store, transaction) => ([first, ...rest] = []) => {
+	if (first) {
+		const request = store.add(first);
+		request.onsuccess = () => {
+			if (rest.length) {
+				pushTask(store, transaction)(rest);
+			}
+		};
+
+		request.onerror = (e) => {
+			throw e;
+		};
+	}
+};
+
+/* */
+function split(entities, limit = BULK_LIMIT) {
+	return entities.reduce(
+		([f, ...r], n, i) => {
+			return (i + 1) % limit === 0 ? [[n], f, ...r] : [[n, ...f], ...r];
+		},
+		[[]]
+	);
+}
+
+/* */
+const bulkPush = (db, name, hook, resolve, reject, max) => (
+	lot,
+	i = 0,
+	treated = 0
+) => {
+	const [first, ...rest] = lot;
+	try {
+		if (first) {
+			const transaction = db.transaction(name, 'readwrite');
+			const store = transaction.objectStore(name);
+			pushTask(store, transaction)(first);
+			transaction.oncomplete = function () {
+				const nextTreated = treated + first.length;
+				hook({
+					message: {
+						...BULK_INSERT_MESSAGES.bulkInsertComplete,
+						nb: first.length,
+						treated: nextTreated,
+						step: i,
+						max,
+						percent: Math.round((nextTreated / max) * 100),
+					},
+				});
+				bulkPush(
+					db,
+					name,
+					hook,
+					resolve,
+					reject,
+					max
+				)(rest, i + 1, nextTreated);
+			};
+			transaction.onerror = function (e) {
+				hook({ message: BULK_INSERT_MESSAGES.bulkInsertError, error: e });
+				reject(e);
+			};
+		} else {
+			hook({ message: BULK_INSERT_MESSAGES.bulkInsertFinished });
+			resolve('success');
+		}
+	} catch (e) {
+		hook({ message: BULK_INSERT_MESSAGES.bulkInsertError });
+		reject(e);
+	}
+};
+
+/* */
+function bulkInsert(db, store, hook = () => null) {
+	return (entities = []) => {
+		const lots = split(entities, BULK_LIMIT);
+		return new Promise((resolve, reject) => {
+			try {
+				bulkPush(db, store, hook, resolve, reject, entities.length)(lots);
+			} catch (e) {
+				reject(e);
+			}
+		});
+	};
+}
+
+export default bulkInsert;

package/src/utils/idb-tools/index.js

@@ -0,0 +1,10 @@
+export { default as openDb } from './open-db';
+export { default as clearDb } from './clear-store';
+export { default as getIDB } from './get-idb';
+export { default as getEntity } from './get-entity';
+export { default as insertEntity } from './insert-entity';
+export { default as openOnCreateDb } from './open-or-create-db';
+export {
+	default as idbBulkInsert,
+	BULK_INSERT_MESSAGES,
+} from './idb-bulk-insert';

package/src/utils/idb-tools/insert-entity.js

@@ -0,0 +1,15 @@
+function insert(db, storeName, entity) {
+	return new Promise(function (resolve, reject) {
+		const transaction = db.transaction(storeName, 'readwrite');
+		const store = transaction.objectStore(storeName);
+		const request = store.add(entity);
+		request.onsuccess = () => {
+			resolve('success');
+		};
+		request.onerror = (e) => {
+			reject(e);
+		};
+	});
+}
+
+export default insert;

package/src/utils/idb-tools/open-db.js

@@ -0,0 +1,13 @@
+/* eslint-disable no-restricted-globals */
+import createDbOpener from './create-db-opener';
+
+function onUpgradeNeeded(e, _, reject) {
+	e.target.transaction.abort();
+	reject({ message: 'base seems to need an upgrade!' });
+}
+
+function onSuccess(e, resolve) {
+	resolve(e.target.result);
+}
+
+export default createDbOpener({ onUpgradeNeeded, onSuccess });

package/src/utils/idb-tools/open-or-create-db.js

@@ -0,0 +1,34 @@
+/* eslint-disable no-restricted-globals */
+import CONSTANTE from '../store-tools/constantes';
+import createDbOpener from './create-db-opener';
+
+function onUpgradeNeeded(e, resolve, reject) {
+	try {
+		e.target.onsuccess = function () {
+			// Block success event!
+		};
+		const db = e.target.result;
+		const store = db.createObjectStore(CONSTANTE.STORE_DATA_NAME, {
+			keyPath: 'id',
+		});
+		db.createObjectStore(CONSTANTE.STORE_INFO_NAME, {
+			keyPath: 'name',
+		});
+		store.createIndex(CONSTANTE.STORE_INDEX_NAME, 'tokens', {
+			multiEntry: true,
+		});
+
+		const txn = e.target.transaction;
+		txn.oncomplete = function () {
+			resolve(db);
+		};
+	} catch (e) {
+		reject(e);
+	}
+}
+
+function onSuccess(e, resolve) {
+	resolve(e.target.result);
+}
+
+export default createDbOpener({ onUpgradeNeeded, onSuccess });

package/src/utils/is-element.js

@@ -0,0 +1,7 @@
+import { isValidElement } from 'react';
+
+function isElement(element) {
+	return isValidElement(element);
+}
+
+export { isElement };

package/src/utils/store-tools/auto-load.js

@@ -0,0 +1,74 @@
+import openStorage from './open-or-create-store';
+import updateStoreInfo from './create/update-store-info';
+import cleanStorage from './clear-store-data';
+import cleanInfoStorage from './clear-store-info';
+import { createWorker } from '../suggester-workers/create-worker';
+
+const workerPath =
+	process.env.LUNATIC_LOADER_WORKER_PATH ||
+	process.env.REACT_APP_LUNATIC_LOADER_WORKER_PATH ||
+	'workers/lunatic-loader-worker-0.1.0.js';
+
+export const loadSuggesters = (suggesterFetcher) => async (suggesters) => {
+	Object.entries(suggesters).forEach(async ([name, attrs]) => {
+		const { version } = attrs;
+		const db = await openStorage(name, version);
+		if (db) {
+			await cleanStorage(db);
+			await cleanInfoStorage(db);
+			await updateStoreInfo(db, attrs);
+		}
+	});
+	Object.entries(suggesters).forEach(([name, attrs]) => {
+		const { url, version, fields, stopWords } = attrs;
+		const f = suggesterFetcher || fetch;
+		f(url).then(async (res) => {
+			const data = await res.json();
+			const [launch, terminate] = task({ name, fields, stopWords }, version);
+			await launch(data);
+			terminate();
+		});
+	});
+};
+
+/**
+ * Only with Worker
+ */
+function task({ name, fields, stopWords }, version, log = () => null) {
+	const worker = createWorker(workerPath);
+	let start = false;
+	let stop = false;
+
+	function launch(entities, post = () => null) {
+		return new Promise(function (resolve) {
+			start = true;
+			worker.postMessage({
+				name,
+				version,
+				fields,
+				entities,
+				stopWords,
+			});
+			worker.addEventListener('message', function (e) {
+				const { data } = e;
+				if (data === 'success') {
+					if (!stop) {
+						post();
+					}
+					resolve(data);
+				} else {
+					log(data);
+				}
+			});
+		});
+	}
+
+	function terminate() {
+		if (start) {
+			stop = true;
+			worker.terminate();
+		}
+	}
+
+	return [launch, terminate];
+}

package/src/utils/store-tools/clear-store-data.js

@@ -0,0 +1,8 @@
+import { clearDb } from '../idb-tools';
+import CONSTANTES from './constantes';
+
+async function clear(db) {
+	await clearDb(db, CONSTANTES.STORE_DATA_NAME);
+}
+
+export default clear;

package/src/utils/store-tools/clear-store-info.js

@@ -0,0 +1,8 @@
+import { clearDb } from '../idb-tools';
+import CONSTANTES from './constantes';
+
+async function clear(db) {
+	await clearDb(db, CONSTANTES.STORE_INFO_NAME);
+}
+
+export default clear;

package/src/utils/store-tools/constantes.js

@@ -0,0 +1,20 @@
+export const STORE_DATA_NAME = 'store/entities';
+export const STORE_INFO_NAME = 'store/info';
+export const STORE_INDEX_NAME = 'store/entities/index';
+export const STORE_INDEX_PATH = 'searching';
+export const MAX_STRING = String.fromCharCode(65535);
+export const SEARCH_TYPES = {
+	prefix: 'search/prefix',
+	tokens: 'search/tokens',
+};
+
+const CONSTANTES = {
+	STORE_DATA_NAME,
+	STORE_INDEX_NAME,
+	STORE_INDEX_PATH,
+	MAX_STRING,
+	SEARCH_TYPES,
+	STORE_INFO_NAME,
+};
+
+export default CONSTANTES;

package/src/utils/store-tools/create/create.js

@@ -0,0 +1,19 @@
+import { clearDb } from '../../idb-tools';
+import CONSTANTES from '../constantes';
+import openOrCreateStore from '../open-or-create-store';
+import updateStoreInfo from './update-store-info';
+
+async function create(storeInfo, idbVersion) {
+	const { name } = storeInfo;
+	try {
+		const db = await openOrCreateStore(name, idbVersion);
+		await clearDb(db, CONSTANTES.STORE_INFO_NAME);
+		await updateStoreInfo(db, storeInfo);
+
+		return db;
+	} catch (e) {
+		throw e;
+	}
+}
+
+export default create;

package/src/utils/store-tools/create/index.js

@@ -0,0 +1,2 @@
+export { default } from './create';
+export { default as updateStoreInfo } from './update-store-info';

package/src/utils/store-tools/create/update-store-info.js

@@ -0,0 +1,26 @@
+import CONSTANTES from '../constantes';
+
+function updateStoreInfo(db, storeInfo) {
+	return new Promise(function (resolve, reject) {
+		try {
+			const transaction = db.transaction(
+				CONSTANTES.STORE_INFO_NAME,
+				'readwrite'
+			);
+			const store = transaction.objectStore(CONSTANTES.STORE_INFO_NAME);
+			const request = store.add(storeInfo);
+
+			request.onsuccess = function () {
+				resolve('success');
+			};
+
+			request.onerror = function (e) {
+				reject(e);
+			};
+		} catch (e) {
+			reject(e);
+		}
+	});
+}
+
+export default updateStoreInfo;

package/src/utils/store-tools/get-store-count.js

@@ -0,0 +1,22 @@
+import CONSTANTES from './constantes';
+
+function get(db) {
+	return new Promise(function (resolve, reject) {
+		try {
+			const transaction = db.transaction(
+				[CONSTANTES.STORE_DATA_NAME],
+				'readonly'
+			);
+
+			const store = transaction.objectStore(CONSTANTES.STORE_DATA_NAME);
+			const countRequest = store.count();
+			countRequest.onsuccess = function () {
+				resolve(countRequest.result);
+			};
+		} catch (e) {
+			reject(e);
+		}
+	});
+}
+
+export default get;

package/src/utils/store-tools/index.js

@@ -0,0 +1,7 @@
+export * as CONSTANTES from './constantes';
+export { default as openOrCreateStore } from './open-or-create-store';
+export { default as clearStoreData } from './clear-store-data';
+export { default as clearStoreInfo } from './clear-store-info';
+export { default as useStoreIndex } from './use-store-index';
+export { default as getStoreCount } from './get-store-count';
+export { default as updateStoreInfo } from './create/update-store-info';

package/src/utils/store-tools/open-or-create-store.js

@@ -0,0 +1,47 @@
+/* eslint-disable no-restricted-globals */
+import CONSTANTE from './constantes';
+import { getIDB } from '../idb-tools';
+
+const IDB_REF = getIDB();
+
+function openStorage(name, idbVersion = 1) {
+	return new Promise(function (resolve, reject) {
+		if (!IDB_REF) {
+			reject('indexedDb not supported !');
+		}
+		const request = IDB_REF.open(name, idbVersion);
+		let db;
+		let doIt = true;
+
+		request.onupgradeneeded = function (e) {
+			doIt = false;
+			db = e.target.result;
+			const store = db.createObjectStore(CONSTANTE.STORE_DATA_NAME, {
+				keyPath: 'id',
+			});
+			db.createObjectStore(CONSTANTE.STORE_INFO_NAME, {
+				keyPath: 'name',
+			});
+			store.createIndex(CONSTANTE.STORE_INDEX_NAME, 'tokens', {
+				multiEntry: true,
+			});
+
+			const txn = e.target.transaction;
+			txn.oncomplete = function () {
+				resolve(db);
+			};
+		};
+
+		request.onsuccess = function () {
+			if (doIt) {
+				resolve(request.result);
+			}
+		};
+
+		request.onerror = function (e) {
+			reject(e);
+		};
+	});
+}
+
+export default openStorage;

package/src/utils/store-tools/use-store-index.js

@@ -0,0 +1,24 @@
+import { useEffect, useState } from 'react';
+import createStore from './create';
+
+function useStoreIndex(storeInfo, idbVersion) {
+	const [db, setDb] = useState(undefined);
+
+	useEffect(
+		function () {
+			async function init() {
+				if (storeInfo && idbVersion) {
+					const db = await createStore(storeInfo, idbVersion);
+					setDb(db);
+				}
+			}
+
+			init();
+			return function () {};
+		},
+		[storeInfo, idbVersion]
+	);
+	return db;
+}
+
+export default useStoreIndex;

package/src/utils/suggester-workers/append-to-index/append.js

@@ -0,0 +1,25 @@
+import { CONSTANTES } from '../../store-tools';
+import { openDb, idbBulkInsert } from '../../idb-tools';
+import MESSAGES from './store-messages';
+import prepareEntities from './prepare-entities';
+
+async function append(storeInfo, version, entities, log = () => null) {
+	try {
+		const { name, stopWords, fields } = storeInfo;
+		const prepared = prepareEntities(entities, { fields, stopWords }, log);
+		const db = await openDb(name, version);
+		log({ message: MESSAGES.startInsertBatch });
+		await idbBulkInsert(db, CONSTANTES.STORE_DATA_NAME, function (args) {
+			const { message } = args;
+			log({ message });
+		})(prepared);
+		log({ message: MESSAGES.insertBatchDone });
+		log({ message: MESSAGES.done });
+		return 'success';
+	} catch (e) {
+		log({ message: 'Errors occurred when trying to append data.' });
+		console.error(e);
+	}
+}
+
+export default append;

package/src/utils/suggester-workers/append-to-index/append.worker.js

@@ -0,0 +1,16 @@
+/* eslint-disable no-restricted-globals */
+import 'core-js/stable';
+import 'regenerator-runtime/runtime';
+import append from './append';
+
+self.onmessage = function (e) {
+	function log(message) {
+		self.postMessage(message);
+	}
+	const { name, version, stopWords, fields, entities } = e.data;
+	append({ name, version, stopWords, fields }, version, entities, log).then(
+		function () {
+			self.postMessage('success');
+		}
+	);
+};

package/src/utils/suggester-workers/append-to-index/create-append-task.js

@@ -0,0 +1,45 @@
+import { createWorker } from '../create-worker';
+
+const workerPath =
+	process.env.LUNATIC_LOADER_WORKER_PATH ||
+	process.env.REACT_APP_LUNATIC_LOADER_WORKER_PATH ||
+	'workers/lunatic-loader-worker-0.1.0.js';
+
+/**
+ * Only with Worker
+ */
+function task(info, version, log = () => null) {
+	const { name, fields, stopWords } = info;
+	const worker = createWorker(workerPath);
+	let start = false;
+	let stop = false;
+
+	function launch(entities, post = () => null) {
+		return new Promise(function (resolve) {
+			start = true;
+			worker.addEventListener('message', function (e) {
+				const { data } = e;
+				if (data === 'success') {
+					if (!stop) {
+						post();
+					}
+					resolve(data);
+				} else {
+					log(data);
+				}
+			});
+			worker.postMessage({ name, version, fields, stopWords, entities });
+		});
+	}
+
+	function terminate() {
+		if (start) {
+			stop = true;
+			worker.terminate();
+		}
+	}
+
+	return [launch, terminate];
+}
+
+export default task;

package/src/utils/suggester-workers/append-to-index/index.js

@@ -0,0 +1,2 @@
+export { default } from './append';
+export { default as createAppendTask } from './create-append-task';

package/src/utils/suggester-workers/append-to-index/prepare-entities.js

@@ -0,0 +1,61 @@
+import { createTokenizer } from '../commons-tokenizer';
+import MESSAGES from './store-messages';
+
+const DEFAULT_BATCH_SIZE = 1000;
+
+function appendField(fields, field) {
+	if (fields.indexOf(field) !== -1) {
+		return fields;
+	}
+	return [...fields, field];
+}
+
+function createTokensMap(tokensFields) {
+	return Object.entries(tokensFields).reduce(function (map, [field, tokens]) {
+		return tokens.reduce(function (map2, token) {
+			if (token in map2) {
+				const entry = map2[token];
+				const { fields, count } = entry;
+				return {
+					...map2,
+					[token]: {
+						count: count + 1,
+						fields: appendField(fields, field),
+					},
+				};
+			}
+			return { ...map2, [token]: { count: 1, fields: [field] } };
+		}, map);
+	}, {});
+}
+
+function prepareEntities(entities, { fields, stopWords }, log) {
+	const tokenizer = createTokenizer(fields, stopWords);
+
+	let done = 0;
+	const size = DEFAULT_BATCH_SIZE;
+	const { length: max } = entities || [];
+
+	return entities.map(function (suggestion) {
+		const { id } = suggestion;
+		if (id) {
+			const tokensFields = tokenizer(suggestion);
+			const tokensMap = createTokensMap(tokensFields);
+
+			done++;
+			if (done % size === 0 || done === max) {
+				log({
+					message: {
+						...MESSAGES.indexBatch,
+						max,
+						done,
+						percent: (done / max) * 100,
+					},
+				});
+			}
+			return { id, suggestion, tokens: Object.keys(tokensMap), tokensMap };
+		} else throw new Error(`Missing id on entity.`);
+	}, []);
+}
+
+export default prepareEntities;

package/src/utils/suggester-workers/append-to-index/store-messages.js

@@ -0,0 +1,21 @@
+import { BULK_INSERT_MESSAGES } from '../../idb-tools';
+
+const MESSAGES = {
+	...BULK_INSERT_MESSAGES,
+	startCreateIndex: { type: 'fill-store/start-create-index' },
+	indexBatch: {
+		type: 'fill-store/index-batch',
+		max: undefined,
+		done: undefined,
+		percent: undefined,
+	},
+	createIndexDone: { type: 'fill-store/create-index-done' },
+	//
+	storeClear: { type: 'fill-store/clear-store' },
+	startInsertBatch: { type: 'fill-store/start-insert-bacth' },
+	insertBatchDone: { type: 'fill-store/insert-done' },
+	done: { type: 'fill-store/done' },
+	error: { type: 'fill-store/error' },
+};
+
+export default MESSAGES;

package/src/utils/suggester-workers/commons-tokenizer/create-entity-tokenizer.js

@@ -0,0 +1,56 @@
+import createFieldsTokenizer from './create-fields-tokenizer';
+import { composeFilters, createFilterStopWords } from './filters';
+import filterStemmer from './filters/filter-stemmer';
+import filterLength from './filters/filter-length';
+import filterSynonyms from './filters/filter-synonyms';
+import filterAccents from './filters/filter-accents';
+import filterToLower from './filters/filter-to-lower';
+import filterDouble from './filters/filter-double';
+
+function createMapFieldsTokenizer(fields, filters) {
+	return fields.reduce(function (mapFieldTokenizers, field) {
+		const { name } = field;
+		return {
+			...mapFieldTokenizers,
+			[name]: createFieldsTokenizer(field, filters),
+		};
+	}, {});
+}
+
+function createFilterTokens(fields, stopWords) {
+	const filterStopWords = createFilterStopWords(stopWords);
+	const getFilters = composeFilters(
+		filterDouble,
+		filterAccents,
+		filterStemmer,
+		filterSynonyms,
+		filterToLower,
+		filterStopWords,
+		filterLength
+	);
+	const FIELDS_TOKENIZER_MAP = createMapFieldsTokenizer(
+		fields || [],
+		getFilters
+	);
+
+	return function (field, entity) {
+		const { name } = field;
+		const tokenizeIt = FIELDS_TOKENIZER_MAP[name];
+		if (name in entity) {
+			return tokenizeIt(`${entity[name]}`);
+		}
+		return [];
+	};
+}
+
+function createEntityTokenizer(fields, stopWords) {
+	const tokenizeAll = createFilterTokens(fields, stopWords);
+	return function (entity) {
+		return fields.reduce(function (a, field) {
+			const { name } = field;
+			return { ...a, [name]: tokenizeAll(field, entity) };
+		}, {});
+	};
+}
+
+export default createEntityTokenizer;