@inseefr/lunatic 0.1.0-hackathon → 0.1.0-v2

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-double.spec.js

@@ -0,0 +1,20 @@
+import filterDouble from './filter-double';
+
+describe('filter-double', function () {
+	it('empty', function () {
+		const tokens = ['un', 'un'];
+		const results = filterDouble(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('un');
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'un', 'long'];
+		const results = filterDouble(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-length.spec.js

@@ -0,0 +1,18 @@
+import filterLength from './filter-length';
+
+describe('filter-length', function () {
+	it('empty', function () {
+		const tokens = ['un', 'no'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(0);
+	});
+
+	it('remove one', function () {
+		const tokens = ['un', 'long'];
+		const results = filterLength(tokens, { min: 3 });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(1);
+		expect(results[0]).toBe('long');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/{filter-stemmer.js → filters/filter-stemmer.js}

@@ -1,6 +1,6 @@
-import getStemmer from './get-stemmer';
+import getStemmer from '../get-stemmer';
 
-function filterStemmer(tokens, { language, stemmer = true }) {
+function filterStemmer(tokens, { language = 'French', stemmer = true }) {
 	if (stemmer) {
 		const stemming = getStemmer(language);
 		return tokens.map(function (token) {

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-stemmer.spec.js

@@ -0,0 +1,12 @@
+import filterStemmer from './filter-stemmer';
+
+describe('filter-stemmer', function () {
+	it('commerce', function () {
+		const tokens = ['commerce', 'commercial'];
+		const results = filterStemmer(tokens, { language: 'French' });
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('commerc');
+		expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.js

@@ -0,0 +1,36 @@
+const MAPS_FROM_ARRAY = new Map(); // Pour éviter de recalculer une map à chaque fois
+
+function filterFromObject(tokens, synonyms) {
+	return tokens.reduce(function (prec, token) {
+		if (token in synonyms) {
+			return [...prec, token, ...synonyms[token]];
+		}
+		return [...prec, token];
+	}, []);
+}
+
+function buildSynonymsMap(array) {
+	return array.reduce(function (map, { source, target }) {
+		return { ...map, [source]: target };
+	}, {});
+}
+
+function filterFromArray(tokens, synonyms) {
+	if (!MAPS_FROM_ARRAY.has(synonyms)) {
+		MAPS_FROM_ARRAY.set(synonyms, buildSynonymsMap(synonyms));
+	}
+	return filterFromObject(tokens, MAPS_FROM_ARRAY.get(synonyms));
+}
+
+function filterSynonyms(tokens, { synonyms }) {
+	if (Array.isArray(synonyms) && synonyms.length) {
+		return filterFromArray(tokens, synonyms);
+	}
+	if (typeof synonyms === 'object') {
+		return filterFromObject(tokens, synonyms);
+	}
+
+	return tokens;
+}
+
+export default filterSynonyms;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-synonyms.spec.js

@@ -0,0 +1,12 @@
+import filterSynonyms from './filter-synonyms';
+
+describe('filter-synonyms', function () {
+	it('', function () {
+		// const tokens = ['saint', 'sainte'];
+		// const results = filterStemmer(tokens, { language: 'French' });
+		// expect(Array.isArray(results)).toBe(true);
+		// expect(results.length).toBe(2);
+		// expect(results[0]).toBe('commerc');
+		// expect(results[1]).toBe('commercial');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.js

@@ -0,0 +1,10 @@
+function filterToLower(tokens) {
+	return tokens.map(function (token) {
+		if (typeof token === 'string') {
+			return token.toLocaleLowerCase();
+		}
+		return token;
+	});
+}
+
+export default filterToLower;

package/src/utils/suggester-workers/commons-tokenizer/filters/filter-to-lower.spec.js

@@ -0,0 +1,12 @@
+import filterToLower from './filter-to-lower';
+
+describe('filter-to-lower', function () {
+	it('UN', function () {
+		const tokens = ['UN', 'no'];
+		const results = filterToLower(tokens);
+		expect(Array.isArray(results)).toBe(true);
+		expect(results.length).toBe(2);
+		expect(results[0]).toBe('un');
+		expect(results[1]).toBe('no');
+	});
+});

package/src/utils/suggester-workers/commons-tokenizer/filters/index.js

@@ -0,0 +1,2 @@
+export { default as composeFilters } from './compose-filters';
+export { default as createFilterStopWords } from './create-filter-stop-words';

package/src/utils/suggester-workers/commons-tokenizer/index.js

@@ -1,8 +1,9 @@
 export { default as getStemmer } from './get-stemmer';
-export { default as createTokenizer } from './create-tokenizer';
+export { default as createTokenizer } from './create-entity-tokenizer';
 export { default as prepareStringIndexation } from './prepare-string-indexation';
-export { default as filterStemmer } from './filter-stemmer';
-export { default as filterLength } from './filter-length';
-export { default as filterDouble } from './filter-double';
-export { default as filterAccentsToLower } from './filter-accents-to-lower';
+export { default as filterStemmer } from './filters/filter-stemmer';
+export { default as filterLength } from './filters/filter-length';
+export { default as filterDouble } from './filters/filter-double';
+export { default as filterAccents } from './filters/filter-accents';
+export { default as filterToLower } from './filters/filter-to-lower';
 export { default as getRegExpFromPattern } from './get-regexp-from-pattern';

package/src/utils/suggester-workers/commons-tokenizer/prepare-string-indexation.js

@@ -1,9 +1,13 @@
 import removeAccents from 'remove-accents';
 
 function prepare(token = '', replacement = '') {
-	return typeof token === 'string'
-		? removeAccents(token.toLowerCase()).replace(/[- ']/g, replacement)
-		: token;
+	if (typeof token === 'string') {
+		return removeAccents(token.trim().toLowerCase()).replace(
+			/[- ']/g,
+			replacement
+		);
+	}
+	return token;
 }
 
 export default prepare;

package/src/utils/suggester-workers/commons-tokenizer/soft-tokenizer.js

@@ -1,4 +1,4 @@
-import prepareStringIndexation from './prepare-string-indexation';
+import { prepareStringIndexation } from '../commons-tokenizer';
 
 function softTokenizer(string) {
 	return [prepareStringIndexation(string, '-')];

package/src/utils/suggester-workers/create-worker.js

@@ -0,0 +1,55 @@
+// Enable cross domain workers
+
+const testSameOrigin = (url) => {
+	var loc = window.location;
+	var a = document.createElement('a');
+	a.href = url;
+	return (
+		a.hostname === loc.hostname &&
+		a.port === loc.port &&
+		a.protocol === loc.protocol
+	);
+};
+
+const createWorkerFallback = (workerUrl) => {
+	console.log('Create worker for MFE');
+	var worker = null;
+	try {
+		var blob;
+		try {
+			blob = new Blob([`importScripts('${workerUrl}');`], {
+				type: 'application/javascript',
+			});
+		} catch (e) {
+			var blobBuilder = new (window.BlobBuilder ||
+				window.WebKitBlobBuilder ||
+				window.MozBlobBuilder)();
+			blobBuilder.append(`importScripts('${workerUrl}');`);
+			blob = blobBuilder.getBlob('application/javascript');
+		}
+		var url = window.URL || window.webkitURL;
+		var blobUrl = url.createObjectURL(blob);
+		worker = new Worker(blobUrl);
+	} catch (e1) {
+		console.error(`Lunatic-worker : Failed to load web worker : ${workerUrl}`);
+	}
+	return worker;
+};
+
+export const createWorker = (workerUrl) => {
+	var worker = null;
+	try {
+		if (testSameOrigin(workerUrl)) {
+			worker = new Worker(workerUrl);
+			worker.onerror = (event) => {
+				event.preventDefault();
+				worker = createWorkerFallback(workerUrl);
+			};
+		} else {
+			worker = createWorkerFallback(workerUrl);
+		}
+	} catch (e) {
+		worker = createWorkerFallback(workerUrl);
+	}
+	return worker;
+};

package/src/utils/suggester-workers/find-best-label/tokenize.js

@@ -1,5 +1,4 @@
 import tokenizer from 'string-tokenizer';
-import removeAccents from 'remove-accents';
 
 function toArray(tokens) {
 	if (Array.isArray(tokens)) {
@@ -12,13 +11,11 @@ function tokenize(object, pattern = /[\w]+/) {
 	if (typeof object === 'object') {
 		const map = {};
 		Object.entries(object).forEach(function ([key, string]) {
-			const input = removeAccents(string).toLowerCase().trim();
-			if (input.length) {
+			if (string.length) {
 				const { tokens } = tokenizer()
-					.input(removeAccents(string).toLowerCase())
+					.input(string)
 					.tokens({ tokens: pattern })
 					.resolve();
-
 				map[key] = toArray(tokens);
 			} else {
 				map[key] = [];

package/src/utils/suggester-workers/searching/query-parser/query-parser-soft.js

@@ -0,0 +1,7 @@
+import { prepareStringIndexation } from '../../commons-tokenizer';
+
+function queryParserSoft(string) {
+	return [prepareStringIndexation(string, '-')];
+}
+
+export default queryParserSoft;

package/src/utils/suggester-workers/searching/query-parser/query-parser-tokenized.js

@@ -0,0 +1,34 @@
+import tokenizer from 'string-tokenizer';
+import {
+	filterStemmer,
+	filterLength,
+	filterDouble,
+	getRegExpFromPattern,
+	filterToLower,
+} from '../../commons-tokenizer';
+import { composeFilters } from '../../commons-tokenizer/filters';
+
+const filterTokens = composeFilters(
+	filterDouble,
+	filterStemmer,
+	filterToLower,
+	filterLength
+);
+
+function toArray(tokens) {
+	if (tokens) {
+		return Array.isArray(tokens) ? tokens : [tokens];
+	}
+	return [];
+}
+
+function queryParserTokenized(query = '', { pattern, ...args } = {}) {
+	const patternForTokens = { tokens: getRegExpFromPattern(pattern) };
+	const { tokens } = tokenizer()
+		.input(query)
+		.tokens(patternForTokens)
+		.resolve();
+	return filterTokens(toArray(tokens), args);
+}
+
+export default queryParserTokenized;

package/src/utils/suggester-workers/searching/resolve-query-parser.js

@@ -1,6 +1,6 @@
-import { queryParserTokenized, queryParserSoft } from '../query-parser';
+import { queryParserTokenized, queryParserSoft } from './query-parser';
 
-async function resolve(name, queryParser) {
+async function resolve(queryParser) {
 	try {
 		const { type } = queryParser;
 		switch (type) {

package/src/utils/suggester-workers/searching/searching.js

@@ -43,7 +43,7 @@ async function searching(search, { name, version = '1' }) {
 			const db = await getDb(name, version);
 			const info = await getEntity(db, CONSTANTES.STORE_INFO_NAME, name);
 			const { queryParser, max, order } = info;
-			const parser = await resolveQueryParser(name, queryParser);
+			const parser = await resolveQueryParser(queryParser);
 			const transaction = db.transaction(
 				CONSTANTES.STORE_DATA_NAME,
 				'readonly'
@@ -62,7 +62,7 @@ async function searching(search, { name, version = '1' }) {
 		}
 		return { results: [], search };
 	} catch (e) {
-		console.err(e);
+		console.error(e);
 		throw e;
 	}
 }

package/src/utils/to-expose/{handler.js → _old/handler.js}

@@ -15,7 +15,9 @@ export const updateQuestionnaire =
 			!updatedValues
 		)
 			return questionnaire;
+
 		const { variables, components, ...other } = questionnaire;
+
 		if (!components || components.length === 0) return questionnaire;
 		const varsAndRefs = Object.entries(updatedValues).reduce(
 			(acc, [key, value]) => {
@@ -59,6 +61,26 @@ export const updateQuestionnaire =
 		};
 	};
 
+export const updateExternals =
+	(questionnaire) => (logFunction) => (updatedValues) => {
+		const { variables, ...other } = questionnaire;
+		const { EXTERNAL } = variables;
+		const newVariables = {
+			...variables,
+			EXTERNAL: { ...EXTERNAL, ...updatedValues },
+		};
+
+		const newVariablesWithCalculated = addCalculatedVars(
+			newVariables,
+			updatedValues
+		)(logFunction);
+
+		return {
+			...other,
+			variables: newVariablesWithCalculated,
+		};
+	};
+
 export const buildNewValue =
 	(preferences) => (valueType) => (oldValues) => (value) => {
 		if (preferences.length === 1) return value;

package/src/utils/to-expose/{state.js → _old/state.js}

@@ -33,23 +33,22 @@ const getCalculatedFromVariables = (variables) =>
 const getExternalFromVariables = (variables) =>
 	(variables && variables[C.EXTERNAL]) || {};
 
-export const getCollectedStateByValueType = (questionnaire) => (
-	valueType,
-	displayNull
-) =>
-	['PREVIOUS', 'COLLECTED', 'FORCED', 'EDITED', 'INPUTED'].includes(valueType)
-		? Object.entries(getCollectedState(questionnaire)).reduce((_, v) => {
-				if (displayNull || v[1][valueType] !== null)
-					return {
-						..._,
-						[v[0]]: v[1][valueType],
-					};
-				return _;
-		  }, {})
-		: {};
+export const getCollectedStateByValueType =
+	(questionnaire) => (valueType, displayNull) =>
+		['PREVIOUS', 'COLLECTED', 'FORCED', 'EDITED', 'INPUTED'].includes(valueType)
+			? Object.entries(getCollectedState(questionnaire)).reduce((_, v) => {
+					if (displayNull || v[1][valueType] !== null)
+						return {
+							..._,
+							[v[0]]: v[1][valueType],
+						};
+					return _;
+			  }, {})
+			: {};
 
 export const getBindings = (questionnaire) => {
 	const { variables } = questionnaire;
+
 	return {
 		...getCollectedStateByValueType(questionnaire)('COLLECTED', true),
 		...getCalculatedFromVariables(variables),

package/src/utils/to-expose/hooks/filter-components.js

@@ -1,106 +1,106 @@
-import { interpret } from '../interpret';
-import { isDev, buildVectorialBindings } from '../../lib';
-
-let cache = {};
-
-const customFilterPagination = ({ page }, pagination, currentPage) => {
-	return pagination ? currentPage.split('.')[0] === page : true;
-};
-
-const filterComponents = ({ components, updatedVars, features, bindings }) => {
-	const localCache = {};
-	const filtered = components.filter(({ conditionFilter }) => {
-		if (!conditionFilter || !conditionFilter.value) return true;
-		const { bindingDependencies, value } = conditionFilter;
-		if (localCache[value] !== undefined) return localCache[value];
-		if (
-			(!bindingDependencies ||
-				!updatedVars.some((t) => bindingDependencies.includes(t))) &&
-			cache[value] !== undefined
-		)
-			return cache[value];
-		const vectorialBindings = buildVectorialBindings(bindings);
-		const inter = interpret(features)(vectorialBindings)(value);
-		localCache[value] = inter;
-		return inter;
-	});
-	cache = { ...cache, ...localCache };
-	return filtered;
-};
-
-const buildComponents = ({
-	components,
-	management,
-	bindings,
-	features,
-	page,
-	pagination,
-	todo,
-}) => {
-	if (management && !pagination) return components;
-
-	if (management && pagination)
-		return components.filter((c) =>
-			customFilterPagination(c, pagination, page)
-		);
-
-	if (isDev) {
-		console.log('Start filter');
-		var start = new Date().getTime();
-	}
-
-	const updatedVars = Object.keys(todo);
-
-	if (!pagination) {
-		const filtered = filterComponents({
-			components,
-			updatedVars,
-			features,
-			bindings,
-		});
-		if (isDev) console.log(`End filter: ${new Date().getTime() - start} ms`);
-		return filtered;
-	}
-
-	const pageComponents = components.filter((c) =>
-		customFilterPagination(c, pagination, page)
-	);
-
-	const pageComponentsFiltered = filterComponents({
-		components: pageComponents,
-		updatedVars,
-		features,
-		bindings,
-	});
-	if (isDev) console.log(`End filter: ${new Date().getTime() - start}`);
-	return pageComponentsFiltered;
-};
-
-let oldComponents = [];
-let memoryTodo = {};
-
-export const useFilterComponents = ({
-	questionnaire,
-	management,
-	bindings,
-	features,
-	page,
-	pagination,
-	todo,
-}) => {
-	if (Object.keys(todo).length > 0) {
-		memoryTodo = todo;
-		return oldComponents;
-	}
-	const components = buildComponents({
-		components: questionnaire.components,
-		management,
-		bindings,
-		features,
-		page,
-		pagination,
-		todo: memoryTodo,
-	});
-	oldComponents = components;
-	return components;
-};
+import { interpret } from '../interpret';
+import { isDev, buildVectorialBindings } from '../../lib';
+
+let cache = {};
+
+const customFilterPagination = ({ page }, pagination, currentPage) => {
+	return pagination ? currentPage.split('.')[0] === page : true;
+};
+
+const filterComponents = ({ components, updatedVars, features, bindings }) => {
+	const localCache = {};
+	const filtered = components.filter(({ conditionFilter }) => {
+		if (!conditionFilter || !conditionFilter.value) return true;
+		const { bindingDependencies, value } = conditionFilter;
+		if (localCache[value] !== undefined) return localCache[value];
+		if (
+			(!bindingDependencies ||
+				!updatedVars.some((t) => bindingDependencies.includes(t))) &&
+			cache[value] !== undefined
+		)
+			return cache[value];
+		const vectorialBindings = buildVectorialBindings(bindings);
+		const inter = interpret(features)(vectorialBindings)(value);
+		localCache[value] = inter;
+		return inter;
+	});
+	cache = { ...cache, ...localCache };
+	return filtered;
+};
+
+const buildComponents = ({
+	components,
+	management,
+	bindings,
+	features,
+	page,
+	pagination,
+	todo,
+}) => {
+	if (management && !pagination) return components;
+
+	if (management && pagination)
+		return components.filter((c) =>
+			customFilterPagination(c, pagination, page)
+		);
+
+	if (isDev) {
+		console.log('Start filter');
+		var start = new Date().getTime();
+	}
+
+	const updatedVars = Object.keys(todo);
+
+	if (!pagination) {
+		const filtered = filterComponents({
+			components,
+			updatedVars,
+			features,
+			bindings,
+		});
+		if (isDev) console.log(`End filter: ${new Date().getTime() - start} ms`);
+		return filtered;
+	}
+
+	const pageComponents = components.filter((c) =>
+		customFilterPagination(c, pagination, page)
+	);
+
+	const pageComponentsFiltered = filterComponents({
+		components: pageComponents,
+		updatedVars,
+		features,
+		bindings,
+	});
+	if (isDev) console.log(`End filter: ${new Date().getTime() - start}`);
+	return pageComponentsFiltered;
+};
+
+let oldComponents = [];
+let memoryTodo = {};
+
+export const useFilterComponents = ({
+	questionnaire,
+	management,
+	bindings,
+	features,
+	page,
+	pagination,
+	todo,
+}) => {
+	if (Object.keys(todo).length > 0) {
+		memoryTodo = todo;
+		return oldComponents;
+	}
+	const components = buildComponents({
+		components: questionnaire.components,
+		management,
+		bindings,
+		features,
+		page,
+		pagination,
+		todo: memoryTodo,
+	});
+	oldComponents = components;
+	return components;
+};

package/src/utils/to-expose/hooks/index.js

@@ -1 +1 @@
-export { default } from './lunatic';
+export { default } from './use-lunatic';