@inseefr/lunatic 0.2.0-experimental → 0.2.0-v2

package/src/utils/suggester-workers/commons-tokenizer/{filter-double.js → filters/filter-double.js}

@@ -1,4 +1,4 @@
-function filter(tokens = []) {
+function filterDouble(tokens = []) {
 	const map = {};
 	return tokens.reduce(function (a, token) {
 		if (token in map) {
@@ -9,4 +9,4 @@ function filter(tokens = []) {
 	}, []);
 }
 
-export default filter;
+export default filterDouble;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-double.spec.js

@@ -0,0 +1,20 @@
+import filterDouble from './filter-double';
+
+describe('filter-double', function () {
+	it('empty', function () {
+		const tokens = ['un', 'un'];
+		const results = filterDouble(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('un');
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'un', 'long'];
+		const results = filterDouble(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-length.js → filters/filter-length.js}

@@ -1,4 +1,4 @@
-function filterLength(tokens, min = 2) {
+function filterLength(tokens, { min = 2 }) {
 	return tokens.filter(function (token) {
 		return token.length >= min;
 	});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-length.spec.js

@@ -0,0 +1,18 @@
+import filterLength from './filter-length';
+
+describe('filter-length', function () {
+	it('empty', function () {
+		const tokens = ['un', 'no'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(0);
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'long'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.js

@@ -0,0 +1,13 @@
+import getStemmer from '../get-stemmer';
+
+function filterStemmer(tokens, { language = 'French', stemmer = true }) {
+	if (stemmer) {
+		const stemming = getStemmer(language);
+		return tokens.map(function (token) {
+			return stemming(token);
+		});
+	}
+	return tokens;
+}
+
+export default filterStemmer;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.spec.js

@@ -0,0 +1,12 @@
+import filterStemmer from './filter-stemmer';
+
+describe('filter-stemmer', function () {
+	it('commerce', function () {
+		const tokens = ['commerce', 'commercial'];
+		const results = filterStemmer(tokens, { language: 'French' });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('commerc');
+		expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.js

@@ -0,0 +1,36 @@
+const MAPS_FROM_ARRAY = new Map(); // Pour éviter de recalculer une map à chaque fois
+
+function filterFromObject(tokens, synonyms) {
+	return tokens.reduce(function (prec, token) {
+		if (token in synonyms) {
+			return [...prec, token, ...synonyms[token]];
+		}
+		return [...prec, token];
+	}, []);
+}
+
+function buildSynonymsMap(array) {
+	return array.reduce(function (map, { source, target }) {
+		return { ...map, [source]: target };
+	}, {});
+}
+
+function filterFromArray(tokens, synonyms) {
+	if (!MAPS_FROM_ARRAY.has(synonyms)) {
+		MAPS_FROM_ARRAY.set(synonyms, buildSynonymsMap(synonyms));
+	}
+	return filterFromObject(tokens, MAPS_FROM_ARRAY.get(synonyms));
+}
+
+function filterSynonyms(tokens, { synonyms }) {
+	if (Array.isArray(synonyms) && synonyms.length) {
+		return filterFromArray(tokens, synonyms);
+	}
+	if (typeof synonyms === 'object') {
+		return filterFromObject(tokens, synonyms);
+	}
+
+	return tokens;
+}
+
+export default filterSynonyms;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.spec.js

@@ -0,0 +1,12 @@
+import filterSynonyms from './filter-synonyms';
+
+describe('filter-synonyms', function () {
+	it('', function () {
+		// const tokens = ['saint', 'sainte'];
+		// const results = filterStemmer(tokens, { language: 'French' });
+		// expect(Array.isArray(results)).toBe(true);
+		// expect(results.length).toBe(2);
+		// expect(results[0]).toBe('commerc');
+		// expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.js

@@ -0,0 +1,10 @@
+function filterToLower(tokens) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return token.toLocaleLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterToLower;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.spec.js

@@ -0,0 +1,12 @@
+import filterToLower from './filter-to-lower';
+
+describe('filter-to-lower', function () {
+	it('UN', function () {
+		const tokens = ['UN', 'no'];
+		const results = filterToLower(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('no');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/index.js

@@ -0,0 +1,2 @@
+export { default as composeFilters } from './compose-filters';
+export { default as createFilterStopWords } from './create-filter-stop-words';

package/src/utils/suggester-workers/commons-tokenizer/index.js

@@ -1,8 +1,9 @@
 export { default as getStemmer } from './get-stemmer';
-export { default as createTokenizer } from './create-tokenizer';
+export { default as createTokenizer } from './create-entity-tokenizer';
 export { default as prepareStringIndexation } from './prepare-string-indexation';
-export { default as filterStemmer } from './filter-stemmer';
-export { default as filterLength } from './filter-length';
-export { default as filterDouble } from './filter-double';
-export { default as filterAccentsToLower } from './filter-accents-to-lower';
+export { default as filterStemmer } from './filters/filter-stemmer';
+export { default as filterLength } from './filters/filter-length';
+export { default as filterDouble } from './filters/filter-double';
+export { default as filterAccents } from './filters/filter-accents';
+export { default as filterToLower } from './filters/filter-to-lower';
 export { default as getRegExpFromPattern } from './get-regexp-from-pattern';

package/src/utils/suggester-workers/commons-tokenizer/prepare-string-indexation.js

@@ -1,9 +1,13 @@
 import removeAccents from 'remove-accents';
 
 function prepare(token = '', replacement = '') {
-	return typeof token === 'string'
-		? removeAccents(token.toLowerCase()).replace(/[- ']/g, replacement)
-		: token;
+	if (typeof token === 'string') {
+		return removeAccents(token.trim().toLowerCase()).replace(
+			/[- ']/g,
+			replacement
+		);
+	}
+	return token;
 }
 
 export default prepare;

package/src/utils/suggester-workers/commons-tokenizer/soft-tokenizer.js

@@ -1,4 +1,4 @@
-import prepareStringIndexation from './prepare-string-indexation';
+import { prepareStringIndexation } from '../commons-tokenizer';
 
 function softTokenizer(string) {
 	return [prepareStringIndexation(string, '-')];

package/src/utils/suggester-workers/create-worker.js

@@ -0,0 +1,55 @@
+// Enable cross domain workers
+
+const testSameOrigin = (url) => {
+	var loc = window.location;
+	var a = document.createElement('a');
+	a.href = url;
+	return (
+		a.hostname === loc.hostname &&
+		a.port === loc.port &&
+		a.protocol === loc.protocol
+	);
+};
+
+const createWorkerFallback = (workerUrl) => {
+	console.log('Create worker for MFE');
+	var worker = null;
+	try {
+		var blob;
+		try {
+			blob = new Blob([`importScripts('${workerUrl}');`], {
+				type: 'application/javascript',
+			});
+		} catch (e) {
+			var blobBuilder = new (window.BlobBuilder ||
+				window.WebKitBlobBuilder ||
+				window.MozBlobBuilder)();
+			blobBuilder.append(`importScripts('${workerUrl}');`);
+			blob = blobBuilder.getBlob('application/javascript');
+		}
+		var url = window.URL || window.webkitURL;
+		var blobUrl = url.createObjectURL(blob);
+		worker = new Worker(blobUrl);
+	} catch (e1) {
+		console.error(`Lunatic-worker : Failed to load web worker : ${workerUrl}`);
+	}
+	return worker;
+};
+
+export const createWorker = (workerUrl) => {
+	var worker = null;
+	try {
+		if (testSameOrigin(workerUrl)) {
+			worker = new Worker(workerUrl);
+			worker.onerror = (event) => {
+				event.preventDefault();
+				worker = createWorkerFallback(workerUrl);
+			};
+		} else {
+			worker = createWorkerFallback(workerUrl);
+		}
+	} catch (e) {
+		worker = createWorkerFallback(workerUrl);
+	}
+	return worker;
+};

package/src/utils/suggester-workers/find-best-label/find-best-label.js

@@ -0,0 +1,39 @@
+import distance from 'damerau-levenshtein';
+
+function reduceFields(map, fields, similarity) {
+	return fields.reduce(function (m, field) {
+		if (field in m) {
+			return { ...m, [field]: m[field] + similarity };
+		}
+		return { ...m, [field]: similarity };
+	}, map);
+}
+
+function reduceTokensMap(map, entries, word) {
+	return entries.reduce(function (inter, [token, { fields }]) {
+		const { similarity } = distance(token, word);
+		return reduceFields(inter, fields, similarity);
+	}, map);
+}
+
+function findBest(map) {
+	let best = -1;
+	return Object.entries(map).reduce(function (curr, [field, score]) {
+		if (score > best) {
+			best = score;
+			return field;
+		}
+		return curr;
+	}, undefined);
+}
+
+function findBestLabel(searchTokens = [], tokensMap = {}) {
+	const entries = Object.entries(tokensMap);
+	const map = searchTokens.reduce(function (map, word) {
+		return reduceTokensMap(map, entries, word);
+	}, {});
+	const best = findBest(map);
+	return best;
+}
+
+export default findBestLabel;

package/src/utils/suggester-workers/find-best-label/find-best-label.worker.js

@@ -0,0 +1,40 @@
+/* eslint-disable no-restricted-globals */
+import 'core-js/stable';
+import 'regenerator-runtime/runtime';
+import tokenize from './tokenize';
+import findBestLabel from './find-best-label';
+
+const TASKS = [];
+let RUN = false;
+
+function executeTask(task) {
+	const { option, search, idTask } = task;
+	const { tokensMap } = option;
+	const { search: st } = tokenize({ search });
+	const label = findBestLabel(st, tokensMap);
+
+	self.postMessage({ response: label, idTask });
+}
+
+function appendTask(task) {
+	TASKS.push(task);
+	if (!RUN) {
+		activate();
+	}
+}
+
+function activate() {
+	RUN = true;
+	while (TASKS.length) {
+		const task = TASKS.pop();
+		executeTask(task);
+	}
+	RUN = false;
+}
+
+self.onmessage = function (e) {
+	const { option, search, idTask } = e.data;
+	if (option && search && idTask) {
+		appendTask({ option, search, idTask });
+	}
+};

package/src/utils/suggester-workers/find-best-label/tokenize.js

@@ -0,0 +1,30 @@
+import tokenizer from 'string-tokenizer';
+
+function toArray(tokens) {
+	if (Array.isArray(tokens)) {
+		return tokens;
+	}
+	return [tokens];
+}
+
+function tokenize(object, pattern = /[\w]+/) {
+	if (typeof object === 'object') {
+		const map = {};
+		Object.entries(object).forEach(function ([key, string]) {
+			if (string.length) {
+				const { tokens } = tokenizer()
+					.input(string)
+					.tokens({ tokens: pattern })
+					.resolve();
+				map[key] = toArray(tokens);
+			} else {
+				map[key] = [];
+			}
+		});
+
+		return map;
+	}
+	return {};
+}
+
+export default tokenize;

package/src/utils/suggester-workers/find-best-label/tokenize.spec.js

@@ -0,0 +1,19 @@
+import tokenize from './tokenize';
+
+describe('tokenize', function () {
+	it('simple', function () {
+		const results = tokenize({ hello: 'hello world!' });
+		const { hello } = results;
+		expect(Array.isArray(hello)).toEqual(true);
+		expect(hello.length).toEqual(2);
+		expect(hello[0]).toEqual('hello');
+		expect(hello[1]).toEqual('world');
+	});
+
+	it('undefined', function () {
+		const results = tokenize({ hello: '' });
+		const { hello } = results;
+		expect(Array.isArray(hello)).toEqual(true);
+		expect(hello.length).toEqual(0);
+	});
+});

package/src/utils/suggester-workers/searching/query-parser/query-parser-soft.js

@@ -0,0 +1,7 @@
+import { prepareStringIndexation } from '../../commons-tokenizer';
+
+function queryParserSoft(string) {
+	return [prepareStringIndexation(string, '-')];
+}
+
+export default queryParserSoft;

package/src/utils/suggester-workers/searching/query-parser/query-parser-tokenized.js

@@ -0,0 +1,34 @@
+import tokenizer from 'string-tokenizer';
+import {
+	filterStemmer,
+	filterLength,
+	filterDouble,
+	getRegExpFromPattern,
+	filterToLower,
+} from '../../commons-tokenizer';
+import { composeFilters } from '../../commons-tokenizer/filters';
+
+const filterTokens = composeFilters(
+	filterDouble,
+	filterStemmer,
+	filterToLower,
+	filterLength
+);
+
+function toArray(tokens) {
+	if (tokens) {
+		return Array.isArray(tokens) ? tokens : [tokens];
+	}
+	return [];
+}
+
+function queryParserTokenized(query = '', { pattern, ...args } = {}) {
+	const patternForTokens = { tokens: getRegExpFromPattern(pattern) };
+	const { tokens } = tokenizer()
+		.input(query)
+		.tokens(patternForTokens)
+		.resolve();
+	return filterTokens(toArray(tokens), args);
+}
+
+export default queryParserTokenized;

package/src/utils/suggester-workers/searching/resolve-query-parser.js

@@ -1,22 +1,20 @@
-import { queryParserTokenized, queryParserSoft } from '../query-parser';
+import { queryParserTokenized, queryParserSoft } from './query-parser';
 
-let PARSERS = {};
-
-async function resolve(name, queryParser) {
-	if (name in PARSERS) {
-		return PARSERS[name];
-	}
+async function resolve(queryParser) {
 	try {
 		const { type } = queryParser;
 		switch (type) {
 			case 'tokenized':
-				PARSERS[name] = queryParserTokenized;
 				const { params } = queryParser;
-				const { language, pattern, min } = params;
+				const { language, pattern, min, stemmer } = params;
 				return (query) =>
-					queryParserTokenized(query, { language, pattern, min });
+					queryParserTokenized(query, {
+						language,
+						pattern,
+						min,
+						stemmer,
+					});
 			case 'soft':
-				PARSERS[name] = queryParserSoft;
 				return queryParserSoft;
 			default:
 				throw new Error(`Unknown parser type ${type}`);

package/src/utils/suggester-workers/searching/searching.js

@@ -7,7 +7,10 @@ import computeScore from './compute-score';
 import getOrderingFunction from './order';
 
 function prepare(response) {
-	return response.map(({ suggestion }) => suggestion);
+	return response.map(({ suggestion, tokensMap }) => ({
+		...suggestion,
+		tokensMap,
+	}));
 }
 
 async function searchTokens(tokens, index) {
@@ -40,7 +43,7 @@ async function searching(search, { name, version = '1' }) {
 			const db = await getDb(name, version);
 			const info = await getEntity(db, CONSTANTES.STORE_INFO_NAME, name);
 			const { queryParser, max, order } = info;
-			const parser = await resolveQueryParser(name, queryParser);
+			const parser = await resolveQueryParser(queryParser);
 			const transaction = db.transaction(
 				CONSTANTES.STORE_DATA_NAME,
 				'readonly'
@@ -50,7 +53,6 @@ async function searching(search, { name, version = '1' }) {
 			const tokens = parser(search);
 			const tokensSuggestions = await searchTokens(tokens, index);
 			const response = computeScore(tokensSuggestions);
-
 			return {
 				results: prepare(
 					getOrderingFunction(order)(filterSize(response, max), order)
@@ -60,7 +62,7 @@ async function searching(search, { name, version = '1' }) {
 		}
 		return { results: [], search };
 	} catch (e) {
-		console.err(e);
+		console.error(e);
 		throw e;
 	}
 }

package/src/utils/to-expose/index.js

@@ -1,11 +1,12 @@
-// updateQuestionnaire is useless since 2.0 optimization efforts
-// export { updateQuestionnaire } from './handler';
-export { mergeQuestionnaireAndData } from './init-questionnaire';
-export {
-	getState,
-	getCollectedState,
-	getCollectedStateByValueType,
-	getBindings,
-} from './state';
-export { interpret } from './interpret';
-export { default as useLunatic } from './hooks';
+// updateQuestionnaire is useless since 2.0 optimization efforts
+// export { updateQuestionnaire } from './handler';
+// export { mergeQuestionnaireAndData } from './init-questionnaire';
+// export {
+// 	getState,
+// 	getCollectedState,
+// 	getCollectedStateByValueType,
+// 	getBindings,
+// } from './state';
+// export { interpret } from './interpret';
+// export * from './hooks';
+export { default as useLunatic } from '../../use-lunatic';

package/src/components/breadcrumb/breadcrumb.scss

@@ -1,20 +0,0 @@
-.breadcrumb-lunatic {
-	width: 100%;
-	height: 40px;
-	box-sizing: border-box;
-	border-radius: 0px;
-	background-color: var(--color-primary-light);
-	color: white;
-	font-weight: bold;
-	font-size: 130%;
-	padding-left: 10px;
-	display: flex;
-	align-items: center;
-}
-
-.breadcrumb-element-lunatic:before {
-	content: '>';
-	color: white;
-	margin-left: 0.4em;
-	margin-right: 0.4em;
-}

package/src/components/breadcrumb/component.js

@@ -1,29 +0,0 @@
-import React from 'react';
-import PropTypes from 'prop-types';
-import { buildStyleObject } from '../../utils/lib';
-import './breadcrumb.scss';
-
-const Breadcrumb = ({ elements, style }) => (
-	<div
-		aria-label={`breadcrumb`}
-		className="breadcrumb-lunatic"
-		style={buildStyleObject(style)}
-	>
-		{elements.map((e, i) => (
-			<React.Fragment key={`breadcrumb-${e.toLowerCase()}`}>
-				{e && (
-					<span className={i !== 0 ? 'breadcrumb-element-lunatic' : undefined}>
-						{e}
-					</span>
-				)}
-			</React.Fragment>
-		))}
-	</div>
-);
-
-Breadcrumb.propTypes = {
-	elements: PropTypes.arrayOf(PropTypes.string).isRequired,
-	style: PropTypes.object,
-};
-
-export default Breadcrumb;

package/src/components/breadcrumb/index.js

@@ -1 +0,0 @@
-export { default } from './component';

package/src/components/button/component.js

@@ -1,53 +0,0 @@
-import React from 'react';
-import PropTypes from 'prop-types';
-import {
-	buildStyleObject,
-	createObjectEvent,
-	isFunction,
-} from '../../utils/lib';
-import './button.scss';
-import { BUTTON_CATEGORY, EVENT_CLICK } from '../../constants';
-
-const Button = ({
-	label,
-	value,
-	onClick,
-	disabled,
-	style,
-	logFunction,
-	id,
-}) => {
-	const handleClick = (e) => {
-		onClick(e);
-		if (isFunction(logFunction))
-			logFunction(
-				createObjectEvent(
-					`button-lunatic-${id}`,
-					BUTTON_CATEGORY,
-					EVENT_CLICK,
-					null,
-					label
-				)
-			);
-	};
-	return (
-		<button
-			type="button"
-			aria-label={label || 'button'}
-			className="button-lunatic"
-			style={buildStyleObject(style)}
-			disabled={disabled}
-			onClick={handleClick}
-		>
-			{value}
-		</button>
-	);
-};
-
-Button.propTypes = {
-	value: PropTypes.oneOfType([PropTypes.string, PropTypes.element]).isRequired,
-	disabled: PropTypes.bool,
-	onClick: PropTypes.func.isRequired,
-};
-
-export default Button;