@inseefr/lunatic 0.1.0-hackathon → 0.1.0-v2

package/src/utils/suggester-workers/query-parser/query-parser-tokenized.js

@@ -1,31 +0,0 @@
-import tokenizer from 'string-tokenizer';
-import removeAccents from 'remove-accents';
-import {
-	filterStemmer,
-	filterLength,
-	filterDouble,
-} from '../commons-tokenizer';
-import { getRegExpFromPattern } from '../commons-tokenizer';
-
-function toArray(tokens) {
-	if (tokens) {
-		return Array.isArray(tokens) ? tokens : [tokens];
-	}
-	return [];
-}
-
-function parser(
-	query = '',
-	{ language = 'French', pattern = /[\w]+/, min = 2, stemmer = true } = {}
-) {
-	const patternForTokens = { tokens: getRegExpFromPattern(pattern) };
-	const { tokens } = tokenizer()
-		.input(removeAccents(query).toLowerCase())
-		.tokens(patternForTokens)
-		.resolve();
-	return stemmer
-		? filterDouble(filterStemmer(filterLength(toArray(tokens), min), language))
-		: filterDouble(filterLength(toArray(tokens), min));
-}
-
-export default parser;

package/src/utils/suggester-workers/query-parser/query-parser-tokenized.spec.js

@@ -1,32 +0,0 @@
-import parse from './query-parser-tokenized';
-describe('query-parser-tokenized', function () {
-	it('tokenized', function () {
-		const result = parse('bonjour monde!');
-		expect(result.length).toBe(2);
-		expect(result).toContain('bonjour');
-		expect(result).toContain('mond');
-	});
-
-	it('remove double', function () {
-		const result = parse('industrielle industriel');
-		expect(result.length).toBe(1);
-		expect(result).toContain('industriel');
-	});
-
-	it('language', function () {
-		const result = parse('unlimited expression', {
-			language: 'French',
-		});
-		expect(result.length).toBe(2);
-		expect(result).toContain('unlimited');
-		expect(result).toContain('express');
-	});
-
-	it('no stemmer', function () {
-		const result = parse('habi', {
-			stemmer: false,
-		});
-		expect(result.length).toBe(1);
-		expect(result).toContain('habi');
-	});
-});