@inseefr/lunatic 0.2.3-experimental → 0.2.3-prisme

package/src/utils/suggester-workers/commons-tokenizer/create-tokenizer.js

@@ -1,103 +0,0 @@
-import tokenizer from 'string-tokenizer';
-import removeAccents from 'remove-accents';
-import prepareStringIndexation from './prepare-string-indexation';
-import softTokenizer from './soft-tokenizer';
-import createFilterStopWords from './create-filter-stop-words';
-import filterStemmer from './filter-stemmer';
-import filterLength from './filter-length';
-import filterSynonyms from './filter-synonyms';
-import getRegExpFromPattern from './get-regexp-from-pattern';
-
-function createStemmedFiltersStack({ filterStopWords }) {
-	return [filterStemmer, filterSynonyms, filterStopWords, filterLength].reduce(
-		function (next, current) {
-			return (tokens, args) => next(current(tokens, args), args);
-		},
-		(t) => t
-	);
-}
-
-function defaultTokenizeIt(string) {
-	return [prepareStringIndexation(string)];
-}
-
-export function tokensToArray(tokenized) {
-	return Object.entries(tokenized).reduce(function (a, [k, values]) {
-		if (k.startsWith('pattern')) {
-			if (typeof values === 'string') {
-				return [...a, values];
-			}
-			return [...a, ...values];
-		}
-		return a;
-	}, []);
-}
-
-function createMapFieldsTokenizer(fields, filtersStack) {
-	return fields.reduce(function (mapFieldTokenizers, field) {
-		const {
-			name,
-			rules = [],
-			min,
-			language = 'French',
-			stemmer = true,
-			synonyms = {},
-		} = field;
-		if (rules === 'soft') {
-			return { ...mapFieldTokenizers, [name]: softTokenizer };
-		}
-		if (rules.length) {
-			const tokenRules = rules.reduce(function (rulesMap, pattern, index) {
-				return {
-					...rulesMap,
-					[`pattern${name}${index}`]: getRegExpFromPattern(pattern),
-				};
-			}, {});
-
-			return {
-				...mapFieldTokenizers,
-				[name]: function (string) {
-					const what = tokenizer().input(string).tokens(tokenRules).resolve();
-					return filtersStack(tokensToArray(what), {
-						min,
-						language,
-						stemmer,
-						synonyms,
-					});
-				},
-			};
-		}
-		return { ...mapFieldTokenizers, [name]: defaultTokenizeIt };
-	}, {});
-}
-
-function createTokenizer(fields, stopWords) {
-	const filterStopWords = createFilterStopWords(stopWords);
-	const filtersStack = createStemmedFiltersStack({ filterStopWords });
-	const FIELDS_TOKENIZER_MAP = createMapFieldsTokenizer(
-		fields || [],
-		filtersStack
-	);
-
-	return function (field, entity) {
-		const { name } = field;
-		const tokenizeIt = FIELDS_TOKENIZER_MAP[name];
-		if (name in entity) {
-			const value = `${entity[name]}`;
-			return tokenizeIt(removeAccents(`${value}`).toLowerCase());
-		}
-		return [];
-	};
-}
-
-function createEntityTokenizer(fields, stopWords) {
-	const tokenizeAll = createTokenizer(fields, stopWords);
-	return function (entity) {
-		return fields.reduce(function (a, field) {
-			const { name } = field;
-			return { ...a, [name]: tokenizeAll(field, entity) };
-		}, {});
-	};
-}
-
-export default createEntityTokenizer;

package/src/utils/suggester-workers/commons-tokenizer/filter-accents-to-lower.js

@@ -1,9 +0,0 @@
-import removeAccents from 'remove-accents';
-
-function filterAccentToLower(tokens = []) {
-	return tokens.map(function (token) {
-		return removeAccents(token.toLowerCase());
-	});
-}
-
-export default filterAccentToLower;

package/src/utils/suggester-workers/commons-tokenizer/filter-synonyms.js

@@ -1,10 +0,0 @@
-function filterSynonyms(tokens, { synonyms }) {
-	return tokens.reduce(function (prec, token) {
-		if (token in synonyms) {
-			return [...prec, token, ...synonyms[token]];
-		}
-		return [...prec, token];
-	}, []);
-}
-
-export default filterSynonyms;

package/src/utils/suggester-workers/query-parser/query-parser-soft.js

@@ -1,7 +0,0 @@
-import { prepareStringIndexation } from '../commons-tokenizer';
-
-function parser(string) {
-	return [prepareStringIndexation(string, '-')];
-}
-
-export default parser;

package/src/utils/suggester-workers/query-parser/query-parser-tokenized.js

@@ -1,31 +0,0 @@
-import tokenizer from 'string-tokenizer';
-import removeAccents from 'remove-accents';
-import {
-	filterStemmer,
-	filterLength,
-	filterDouble,
-} from '../commons-tokenizer';
-import { getRegExpFromPattern } from '../commons-tokenizer';
-
-function toArray(tokens) {
-	if (tokens) {
-		return Array.isArray(tokens) ? tokens : [tokens];
-	}
-	return [];
-}
-
-function parser(
-	query = '',
-	{ language = 'French', pattern = /[\w]+/, min = 2, stemmer = true } = {}
-) {
-	const patternForTokens = { tokens: getRegExpFromPattern(pattern) };
-	const { tokens } = tokenizer()
-		.input(removeAccents(query).toLowerCase())
-		.tokens(patternForTokens)
-		.resolve();
-	return stemmer
-		? filterDouble(filterStemmer(filterLength(toArray(tokens), min), language))
-		: filterDouble(filterLength(toArray(tokens), min));
-}
-
-export default parser;

package/src/utils/suggester-workers/query-parser/query-parser-tokenized.spec.js

@@ -1,32 +0,0 @@
-import parse from './query-parser-tokenized';
-describe('query-parser-tokenized', function () {
-	it('tokenized', function () {
-		const result = parse('bonjour monde!');
-		expect(result.length).toBe(2);
-		expect(result).toContain('bonjour');
-		expect(result).toContain('mond');
-	});
-
-	it('remove double', function () {
-		const result = parse('industrielle industriel');
-		expect(result.length).toBe(1);
-		expect(result).toContain('industriel');
-	});
-
-	it('language', function () {
-		const result = parse('unlimited expression', {
-			language: 'French',
-		});
-		expect(result.length).toBe(2);
-		expect(result).toContain('unlimited');
-		expect(result).toContain('express');
-	});
-
-	it('no stemmer', function () {
-		const result = parse('habi', {
-			stemmer: false,
-		});
-		expect(result.length).toBe(1);
-		expect(result).toContain('habi');
-	});
-});