natural 0.2.1 → 0.4.0

package/spec/porter_stemmer_pt_spec.js

@@ -1,50 +0,0 @@
-/*
-Copyright (c) 2014, Luís Rodrigues
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var PorterStemmer = require('../lib/natural/stemmers/porter_stemmer_pt'),
-	fs = require('fs');
-
-describe('porter_stemmer_pt', function() {
-
- 	it('should perform stemming on a lot of words', function() {
- 		var errors = [];
-
- 		fs.readFileSync('spec/test_data/snowball_pt.txt').toString().split('\n').forEach(function(line) {
- 			if (line) {
- 				var fields = line.replace(/\s+/g, ' ').split(' '),
- 					stemmed = PorterStemmer.stem(fields[0]);
-
- 				if (stemmed !== fields[1]) {
-					console.log('Error:', fields[0], 'Expected:', fields[1], 'Got:', stemmed);
- 					errors.push({
- 						word:     fields[0],
- 						expected: fields[1],
- 						actual:   stemmed
- 					});
- 				}
- 			}
- 		});
-
- 		expect(errors.length).toBe(0);
- 	});
-
-});

package/spec/porter_stemmer_ru_spec.js

@@ -1,63 +0,0 @@
-/*
-Copyright (c) 2011, Polyakov Vladimir, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-var stemmer = require('../lib/natural/stemmers/porter_stemmer_ru');
-
-var test = [
-	'в', 'вавиловка', 'вагнера', 'вагон', 'вагона', 'вагоне', 'вагонов', 'вагоном', 'вагоны',
-	'важная', 'важнее', 'важнейшие', 'важнейшими', 'важничал', 'важно', 'важного', 'важное',
-	'важной', 'важном', 'важному', 'важности', 'важностию', 'важность', 'важностью', 'важную',
-	'важны', 'важные', 'важный', 'важным', 'важных', 'вазах', 'вазы', 'вакса', 'вакханка', 'вал',
-	'валандался', 'валентина', 'валериановых', 'валерию', 'валетами', 'вали', 'валил', 'валился',
-	'валится', 'валов', 'вальдшнепа', 'вальс', 'вальса', 'вальсе', 'вальсишку', 'вальтера', 'валяется',
-	'валялась', 'валялись', 'валялось', 'валялся', 'валять', 'валяются', 'вам', 'вами', 'п', 'па', 'пава',
-	'павел', 'павильон', 'павильонам', 'павла', 'павлиний', 'павлиньи', 'павлиньим', 'павлович', 'павловна',
-	'павловне', 'павловной', 'павловну', 'павловны', 'павловцы', 'павлыч', 'павлыча', 'пагубная', 'падает',
-	'падай', 'падал', 'падала', 'падаль', 'падать', 'падаю', 'падают', 'падающего', 'падающие', 'падеж', 'падение',
-	'падением', 'падении', 'падений', 'падения', 'паденье', 'паденья', 'падет', 'падут', 'падучая', 'падчерицей',
-	'падчерицы', 'падшая', 'падшей', 'падшему', 'падший', 'падшим', 'падших', 'падшую', 'паек', 'пазухи', 'пазуху',
-	'пай', 'пакет', 'пакетом', 'пакеты', 'пакостей', 'пакостно', 'пал'];
-
-var testResult = [
-	'в', 'вавиловк', 'вагнер', 'вагон', 'вагон', 'вагон', 'вагон', 'вагон', 'вагон', 'важн', 'важн', 'важн',
-	'важн', 'важнича', 'важн', 'важн', 'важн', 'важн', 'важн', 'важн', 'важност', 'важност', 'важност',
-	'важност', 'важн', 'важн', 'важн', 'важн', 'важн', 'важн', 'ваз', 'ваз', 'вакс', 'вакханк', 'вал',
-	'валанда', 'валентин', 'валерианов', 'валер', 'валет', 'вал', 'вал', 'вал', 'вал', 'вал', 'вальдшнеп',
-	'вальс', 'вальс', 'вальс', 'вальсишк', 'вальтер', 'валя', 'валя', 'валя', 'валя', 'валя', 'валя', 'валя',
-	'вам', 'вам', 'п', 'па', 'пав', 'павел', 'павильон', 'павильон', 'павл', 'павлин', 'павлин', 'павлин',
-	'павлович', 'павловн', 'павловн', 'павловн', 'павловн', 'павловн', 'павловц', 'павлыч', 'павлыч', 'пагубн',
-	'пада', 'пада', 'пада', 'пада', 'падал', 'пада', 'пада', 'пада', 'пада', 'пада', 'падеж', 'паден', 'паден',
-	'паден', 'паден', 'паден', 'паден', 'паден', 'падет', 'падут', 'падуч', 'падчериц', 'падчериц', 'падш', 'падш',
-	'падш', 'падш', 'падш', 'падш', 'падш', 'паек', 'пазух', 'пазух', 'па', 'пакет', 'пакет', 'пакет', 'пакост',
-	'пакостн', 'пал'];
-
-describe('porter_stemmer', function() {
-	it('should permof stem', function() {
-		for (var i = 0; i < test.length; i++) {
-			expect(stemmer.stem(test[i])).toBe(testResult[i]);
-		}
-	}),
-	it('should tokenize and stem attached', function() {
-		stemmer.attach();
-		expect('мама мыла раму'.tokenizeAndStem()).toEqual(['мам', 'мыл', 'рам']);
-		expect('МАМА МЫЛА РАМУ'.tokenizeAndStem()).toEqual(['мам', 'мыл', 'рам']);
-	});
-});

package/spec/porter_stemmer_spec.js

@@ -1,182 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var stemmer = require('../lib/natural/stemmers/porter_stemmer');
-
-describe('porter_stemmer', function() {
-	it('should categorizeGroups', function() {
-		expect(stemmer.categorizeGroups('syllog')).toBe('CVCVC');
-		expect(stemmer.categorizeGroups('gypsy')).toBe('CVCV');		
-	});
-
-	it('should measure', function() {
-		expect(stemmer.measure('syllog')).toBe(2);
-	});
-
-	it('should perform step 1a', function() {
-		expect(stemmer.step1a('caresses')).toBe('caress');
-		expect(stemmer.step1a('ponies')).toBe('poni');
-		expect(stemmer.step1a('ties')).toBe('ti');
-		expect(stemmer.step1a('caress')).toBe('caress');
-		expect(stemmer.step1a('cats')).toBe('cat');
-	 });
-
-	it('should perform step 1b', function() {
-		expect(stemmer.step1b('feed')).toBe('feed');
-		expect(stemmer.step1b('agreed')).toBe('agree');
-		expect(stemmer.step1b('plastered')).toBe('plaster');
-		expect(stemmer.step1b('bled')).toBe('bled');
-		expect(stemmer.step1b('motoring')).toBe('motor');
-		expect(stemmer.step1b('sing')).toBe('sing');
-	});
-
-	it('should perform step 1c', function() {
-		expect(stemmer.step1c('happy')).toBe('happi');
-		expect(stemmer.step1c('sky')).toBe('sky');
-	});
-
-	it('should perform step 2', function() {
-		expect(stemmer.step2('relational')).toBe('relate');
-		expect(stemmer.step2('conditional')).toBe('condition');
-		expect(stemmer.step2('rational')).toBe('rational');
-		expect(stemmer.step2('valenci')).toBe('valence');
-		expect(stemmer.step2('hesitanci')).toBe('hesitance');
-		expect(stemmer.step2('digitizer')).toBe('digitize');
-		expect(stemmer.step2('conformabli')).toBe('conformable');
-		expect(stemmer.step2('radicalli')).toBe('radical');
-		expect(stemmer.step2('differentli')).toBe('different');
-		expect(stemmer.step2('vileli')).toBe('vile');
-		expect(stemmer.step2('analogousli')).toBe('analogous');
-		expect(stemmer.step2('vietnamization')).toBe('vietnamize');
-		expect(stemmer.step2('predication')).toBe('predicate');
-		expect(stemmer.step2('operator')).toBe('operate');
-		expect(stemmer.step2('feudalism')).toBe('feudal');
-		expect(stemmer.step2('decisiveness')).toBe('decisive');
-		expect(stemmer.step2('hopefulness')).toBe('hopeful');
-		expect(stemmer.step2('callousness')).toBe('callous');
-		expect(stemmer.step2('formaliti')).toBe('formal');
-		expect(stemmer.step2('sensitiviti')).toBe('sensitive');
-		expect(stemmer.step2('sensibiliti')).toBe('sensible');
-	});
-
-	it('should perform step 3', function() {
-		expect(stemmer.step3('triplicate')).toBe('triplic');
-		expect(stemmer.step3('formative')).toBe('form');
-		expect(stemmer.step3('formalize')).toBe('formal');
-		expect(stemmer.step3('electriciti')).toBe('electric');
-		expect(stemmer.step3('electrical')).toBe('electric');
-		expect(stemmer.step3('hopeful')).toBe('hope');
-		expect(stemmer.step3('goodness')).toBe('good');
-	});
-
-	it('should perform step 4', function() {
-		expect(stemmer.step4('revival')).toBe('reviv');
-		expect(stemmer.step4('allowance')).toBe('allow');
-		expect(stemmer.step4('inference')).toBe('infer');
-		expect(stemmer.step4('airliner')).toBe('airlin');
-		expect(stemmer.step4('gyroscopic')).toBe('gyroscop');
-		expect(stemmer.step4('adjustable')).toBe('adjust');
-		expect(stemmer.step4('defensible')).toBe('defens');
-		expect(stemmer.step4('irritant')).toBe('irrit');
-		expect(stemmer.step4('replacement')).toBe('replac');
-		expect(stemmer.step4('adjustment')).toBe('adjust');
-		expect(stemmer.step4('dependent')).toBe('depend');
-		expect(stemmer.step4('adoption')).toBe('adopt');
-		expect(stemmer.step4('homologou')).toBe('homolog');
-		expect(stemmer.step4('communism')).toBe('commun');
-		expect(stemmer.step4('activate')).toBe('activ');
-		expect(stemmer.step4('angulariti')).toBe('angular');
-		expect(stemmer.step4('homologous')).toBe('homolog');
-		expect(stemmer.step4('effective')).toBe('effect');
-		expect(stemmer.step4('bowdlerize')).toBe('bowdler');
-	});
-
-	it('should perform step 5a', function() {
-		expect(stemmer.step5a('probate')).toBe('probat');
-		expect(stemmer.step5a('rate')).toBe('rate');
-		expect(stemmer.step5a('cease')).toBe('ceas');
-	});
-
-	it('should perform step5b', function() {
-		expect(stemmer.step5b('controll')).toBe('control');
-		expect(stemmer.step5b('roll')).toBe('roll');
-	});
-
-	it('should perform complete stemming', function() {
-		expect(stemmer.stem('scoring')).toBe('score');
-		expect(stemmer.stem('scored')).toBe('score');
-		expect(stemmer.stem('scores')).toBe('score');
-		expect(stemmer.stem('score')).toBe('score');
-		expect(stemmer.stem('SCORING')).toBe('score');
-		expect(stemmer.stem('SCORED')).toBe('score');
-		expect(stemmer.stem('SCORES')).toBe('score');
-		expect(stemmer.stem('SCORE')).toBe('score');
-		expect(stemmer.stem('nationals')).toBe('nation');
-		expect(stemmer.stem('doing')).toBe('do');
-	});
-
-	it('should perform stem animated to anim', function() {
-		expect(stemmer.stem('animated')).toBe('anim');
-	});
-
-	it('issue 176 - attemptReplace ed|ing', function() {
-		expect(stemmer.stem('aedile')).toBe('aedil');
-		expect(stemmer.stem('adoptedly')).toBe('adoptedli');
-	});
-
-	it('issue 176 - minimum length for final e', function() {
-		expect(stemmer.stem('ace')).toBe('ac');
-	});
-
-	it('issue 176 - minimum length for final s', function() {
-		expect(stemmer.stem('yes')).toBe('ye');
-	});
-
-	it('issue 176 - s/tion measure', function() {
-		expect(stemmer.stem('invasion')).toBe('invas');
-	});
-
-	it('issue 176 - vehement - step 4', function() {
-		expect(stemmer.stem('vehement')).toBe('vehement');
-		expect(stemmer.stem('syllogism')).toBe('syllog');
-		expect(stemmer.stem('gypsy')).toBe('gypsi');
-	});
-
-	it('issue 176 step 2', function() {
-		expect(stemmer.stem('terribly')).toBe('terribl');				
-		expect(stemmer.stem('apology')).toBe('apolog');	
-	});
-
-	it('issue 176 - step 5a', function() {
-		expect(stemmer.stem('type')).toBe('type');
-	});	
-
-	it('issue 176 - corruptiblity', function() {
-		expect(stemmer.stem('corruptibility')).toBe('corrupt');
-	});	
-
-	it('should tokenize and stem attached', function() {
-		stemmer.attach();
-		expect('scoring stinks'.tokenizeAndStem()).toEqual(['score', 'stink']);
-		expect('SCORING STINKS'.tokenizeAndStem()).toEqual(['score', 'stink']);
-	});
-});

package/spec/present_verb_inflector_spec.js

@@ -1,128 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var VerbInflector = require('../lib/natural/inflectors/present_verb_inflector'),
-    inflector = new VerbInflector ();
-
-describe('present verb inflector', function() {
-    describe('singularization', function() {
-        it('should singularize regular ES forms', function() {
-            expect(inflector.singularize('catch')).toBe('catches');
-            expect(inflector.singularize('do')).toBe('does');
-            expect(inflector.singularize('go')).toBe('goes');
-        });
-
-        it('should handle [CS]HES forms', function () {
-            expect(inflector.singularize('cash')).toBe('cashes');
-            expect(inflector.singularize('ach')).toBe('aches');
-        });
-
-        it('should ignore XES forms', function() {
-            expect(inflector.singularize('annex')).toBe('annexes');
-        });
-
-        it('should handle SSES forms', function() {
-            expect(inflector.singularize('access')).toBe('accesses');
-        });
-
-        it('should ignore ZZES forms', function() {
-            expect(inflector.singularize('buzz')).toBe('buzzes');
-        });
-
-        it('should singularize regular S forms', function() {
-            expect(inflector.singularize('claim')).toBe('claims');
-            expect(inflector.singularize('drink')).toBe('drinks');
-            expect(inflector.singularize('become')).toBe('becomes');
-        });
-
-        it('should singularize irregular forms', function() {
-            expect(inflector.singularize('are')).toBe('is');
-            expect(inflector.singularize('were')).toBe('was');
-            expect(inflector.singularize('have')).toBe('has');
-        });
-
-        it('should singularize ies forms', function() {
-            expect(inflector.singularize('fly')).toBe('flies');
-            expect(inflector.singularize('try')).toBe('tries');
-        });
-        
-        it('should handle ambiguous forms', function() {
-            expect(inflector.singularize('will')).toBe('will');
-        });                
-    });
-    
-    describe('pluralization', function() {
-        it('should pluralize regular ES forms', function() {
-            expect(inflector.pluralize('catches')).toBe('catch');
-            expect(inflector.pluralize('does')).toBe('do');
-            expect(inflector.pluralize('goes')).toBe('go');
-        });
-        
-        it('should handle [CS]HES forms', function () {
-            expect(inflector.pluralize('cashes')).toBe('cash');
-            expect(inflector.pluralize('aches')).toBe('ach');            
-        });        
-        
-        it('should handle XES forms', function() {
-            expect(inflector.pluralize('annexes')).toBe('annex');
-        });
-
-        it('should handle SSES forms', function() {
-            expect(inflector.pluralize('accesses')).toBe('access');
-        });
-
-        it('should handle ZZES forms', function() {
-            expect(inflector.pluralize('buzzes')).toBe('buzz');
-        });
-
-        it('should pluralize regular S forms that done drop e', function() {
-            expect(inflector.pluralize('becomes')).toBe('become');
-        });
-
-        it('should pluralize regular S forms', function() {
-            expect(inflector.pluralize('drinks')).toBe('drink');
-            expect(inflector.pluralize('claims')).toBe('claim');            
-        });
-        
-        it('should pluralize irregular forms', function() {
-            expect(inflector.pluralize('was')).toBe('were');                        
-            expect(inflector.pluralize('is')).toBe('are');
-            expect(inflector.pluralize('am')).toBe('are');
-            expect(inflector.pluralize('has')).toBe('have');
-        });
-
-        it('should pluralize ies forms', function() {
-            expect(inflector.pluralize('flies')).toBe('fly');
-            expect(inflector.pluralize('tries')).toBe('try');
-        });
-        
-        it('should handle ambiguous forms', function() {
-            expect(inflector.pluralize('will')).toBe('will');                        
-        });
-    });
-    
-    describe('should pluralize and singularize string from patch', function() {
-        inflector.attach();
-        expect('becomes'.pluralizePresentVerb()).toBe('become');
-        expect('become'.singularizePresentVerb()).toBe('becomes');
-    });
-});

package/spec/remove_diacritics_spec.js

@@ -1,31 +0,0 @@
-/*
-Copyright (c) 2013, Ken Koch
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var removeDiacritics = require("../lib/natural/normalizers/remove_diacritics.js");
-
-describe('remove_diacritics', function() {
-
-    it('should correctly remove diacritics', function () {
-        var original_phrase = 'piñon ça va über résumé';
-        expect(removeDiacritics(original_phrase)).toEqual('pinon ca va uber resume');
-    });
-});