natural 0.2.0 → 0.2.1

package/spec/metaphone_spec.js

@@ -211,6 +211,11 @@ describe('metaphone', function() {
         expect(metaphone.compare('PHONETICS', 'garbonzo')).toBeFalsy();
     });
     
+    it('#173', function() {
+        //expect(metaphone.process('caesar')).toBe('KSR');   
+        //expect(metaphone.process('chemical')).toBe('KMKL');   
+    });
+
     it('should compare strings with string patch', function() {
         metaphone.attach();
         expect('phonetics'.soundsLike('fonetix')).toBeTruthy();

package/spec/porter_stemmer_pt_spec.js

@@ -0,0 +1,50 @@
+/*
+Copyright (c) 2014, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+var PorterStemmer = require('../lib/natural/stemmers/porter_stemmer_pt'),
+	fs = require('fs');
+
+describe('porter_stemmer_pt', function() {
+
+ 	it('should perform stemming on a lot of words', function() {
+ 		var errors = [];
+
+ 		fs.readFileSync('spec/test_data/snowball_pt.txt').toString().split('\n').forEach(function(line) {
+ 			if (line) {
+ 				var fields = line.replace(/\s+/g, ' ').split(' '),
+ 					stemmed = PorterStemmer.stem(fields[0]);
+
+ 				if (stemmed !== fields[1]) {
+					console.log('Error:', fields[0], 'Expected:', fields[1], 'Got:', stemmed);
+ 					errors.push({
+ 						word:     fields[0],
+ 						expected: fields[1],
+ 						actual:   stemmed
+ 					});
+ 				}
+ 			}
+ 		});
+
+ 		expect(errors.length).toBe(0);
+ 	});
+
+});

package/spec/porter_stemmer_spec.js

@@ -23,6 +23,15 @@ THE SOFTWARE.
 var stemmer = require('../lib/natural/stemmers/porter_stemmer');
 
 describe('porter_stemmer', function() {
+	it('should categorizeGroups', function() {
+		expect(stemmer.categorizeGroups('syllog')).toBe('CVCVC');
+		expect(stemmer.categorizeGroups('gypsy')).toBe('CVCV');		
+	});
+
+	it('should measure', function() {
+		expect(stemmer.measure('syllog')).toBe(2);
+	});
+
 	it('should perform step 1a', function() {
 		expect(stemmer.step1a('caresses')).toBe('caress');
 		expect(stemmer.step1a('ponies')).toBe('poni');
@@ -129,6 +138,42 @@ describe('porter_stemmer', function() {
 		expect(stemmer.stem('animated')).toBe('anim');
 	});
 
+	it('issue 176 - attemptReplace ed|ing', function() {
+		expect(stemmer.stem('aedile')).toBe('aedil');
+		expect(stemmer.stem('adoptedly')).toBe('adoptedli');
+	});
+
+	it('issue 176 - minimum length for final e', function() {
+		expect(stemmer.stem('ace')).toBe('ac');
+	});
+
+	it('issue 176 - minimum length for final s', function() {
+		expect(stemmer.stem('yes')).toBe('ye');
+	});
+
+	it('issue 176 - s/tion measure', function() {
+		expect(stemmer.stem('invasion')).toBe('invas');
+	});
+
+	it('issue 176 - vehement - step 4', function() {
+		expect(stemmer.stem('vehement')).toBe('vehement');
+		expect(stemmer.stem('syllogism')).toBe('syllog');
+		expect(stemmer.stem('gypsy')).toBe('gypsi');
+	});
+
+	it('issue 176 step 2', function() {
+		expect(stemmer.stem('terribly')).toBe('terribl');				
+		expect(stemmer.stem('apology')).toBe('apolog');	
+	});
+
+	it('issue 176 - step 5a', function() {
+		expect(stemmer.stem('type')).toBe('type');
+	});	
+
+	it('issue 176 - corruptiblity', function() {
+		expect(stemmer.stem('corruptibility')).toBe('corrupt');
+	});	
+
 	it('should tokenize and stem attached', function() {
 		stemmer.attach();
 		expect('scoring stinks'.tokenizeAndStem()).toEqual(['score', 'stink']);

package/spec/present_verb_inflector_spec.js

@@ -57,6 +57,7 @@ describe('present verb inflector', function() {
         it('should singularize irregular forms', function() {
             expect(inflector.singularize('are')).toBe('is');
             expect(inflector.singularize('were')).toBe('was');
+            expect(inflector.singularize('have')).toBe('has');
         });
 
         it('should singularize ies forms', function() {
@@ -105,6 +106,8 @@ describe('present verb inflector', function() {
         it('should pluralize irregular forms', function() {
             expect(inflector.pluralize('was')).toBe('were');                        
             expect(inflector.pluralize('is')).toBe('are');
+            expect(inflector.pluralize('am')).toBe('are');
+            expect(inflector.pluralize('has')).toBe('have');
         });
 
         it('should pluralize ies forms', function() {

package/spec/soundex_spec.js

@@ -65,15 +65,6 @@ describe('soundex', function() {
         it('sound condense sequences', function() {
             expect(soundex.condense('11222556')).toBe('1256');
         });
-
-        it('sound condense sequences around H and W', function() {
-            expect(soundex.condense('113w355')).toBe('135');
-        });
-        
-        it('sound drop irrelevant H and W', function() {
-            expect(soundex.condense('phonetics')).toBe('ponetics');
-            expect(soundex.condense('towering')).toBe('toering');            
-        });
     });
     
     describe('padRight0', function() {
@@ -132,6 +123,19 @@ describe('soundex', function() {
         expect('phonetics'.phonetics()).toBe('P532');
         expect('PHONETICS'.phonetics()).toBe('P532');        
     });
+
+    it('issue 221 -- inital vowels that duplicate cons codes', function() {
+        soundex.attach();
+        expect('Lloyd'.phonetics()).toBe('L300');
+        expect('Pfister'.phonetics()).toBe('P236');
+
+        expect('manhattan'.phonetics()).toBe('M535');
+        expect('Lukasiewicz'.phonetics()).toBe('L222');        
+
+        expect('Gauss'.phonetics()).toBe('G200');        
+        expect('Tymczak'.phonetics()).toBe('T522');        
+        expect('Jackson'.phonetics()).toBe('J250');                                
+    });    
     
     it('should tokenize and return string phonetics with string patch', function() {
         soundex.attach();

package/spec/stemmer_token_spec.js

@@ -0,0 +1,161 @@
+/*
+Copyright (c) 2014, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+var StemmerToken = require('../lib/natural/stemmers/token');
+
+describe('stemmer token', function () {
+
+  it('should receive a string', function () {
+    var string = 'test',
+      token    = new StemmerToken(string);
+
+    expect(token.string).toBe(string);
+  });
+
+  it('should hold the original token string', function () {
+    var string = 'test',
+      token    = new StemmerToken(string);
+
+    expect(token.original).toBe(string);
+  });
+
+  it('should replace all instances of a string', function () {
+    var string = 'tester',
+      token    = new StemmerToken(string);
+
+    token.replaceAll('e', 'a');
+    expect(token.string).toBe('tastar');
+
+    token.replaceAll('ar', 'er');
+    expect(token.string).toBe('taster');
+
+    token.replaceAll('r', '');
+    expect(token.string).toBe('taste');
+  });
+
+  it('should allow vowels to be set', function () {
+    var vowels    = 'aeiou',
+      vowelsArray = vowels.split(''),
+      token       = new StemmerToken('');
+
+    token.usingVowels(vowels);
+    expect(token.vowels).toBe(vowels);
+
+    token.usingVowels(vowelsArray);
+    expect(token.vowels).toBe(vowelsArray);
+  });
+
+  it('should check for vowels', function () {
+    var vowels = 'aeiou',
+      token    = new StemmerToken('test');
+
+    token.usingVowels(vowels);
+
+    expect(token.hasVowelAtIndex(0)).toBe(false);
+    expect(token.hasVowelAtIndex(1)).toBe(true);
+    expect(token.hasVowelAtIndex(99)).toBe(false);
+
+    token.usingVowels(vowels.split(''));
+
+    expect(token.hasVowelAtIndex(0)).toBe(false);
+    expect(token.hasVowelAtIndex(1)).toBe(true);
+    expect(token.hasVowelAtIndex(99)).toBe(false);
+  });
+
+  it('should find the next vowel', function () {
+    var token = new StemmerToken('tester').usingVowels('aeiou');
+
+    expect(token.nextVowelIndex(0)).toBe(1);
+    expect(token.nextVowelIndex(1)).toBe(1);
+    expect(token.nextVowelIndex(2)).toBe(4);
+    expect(token.nextVowelIndex(5)).toBe(6);
+    expect(token.nextVowelIndex(99)).toBe(6);
+    expect(token.nextVowelIndex(-1)).toBe(6);
+  });
+
+  it('should find the next consonant', function () {
+    var token = new StemmerToken('testee').usingVowels('aeiou');
+
+    expect(token.nextConsonantIndex(0)).toBe(0);
+    expect(token.nextConsonantIndex(1)).toBe(2);
+    expect(token.nextConsonantIndex(5)).toBe(6);
+    expect(token.nextConsonantIndex(99)).toBe(6);
+    expect(token.nextConsonantIndex(-1)).toBe(6);
+  });
+
+  it('should mark regions', function() {
+    var token = new StemmerToken('tester');
+
+    token.markRegion('test', 1);
+    expect(token.regions.test).toBe(1);
+  });
+
+  it('should mark regions with a callback', function() {
+    var token = new StemmerToken('tester'),
+      context = { value: 99 };
+
+    token.markRegion('test', 1, function (arg) { return arg; });
+    expect(token.regions.test).toBe(1);
+
+    token.markRegion('test', [1], function (arg) { return arg; });
+    expect(token.regions.test).toBe(1);
+
+    token.markRegion('test', [1, 1], function (a1, a2) { return a1 + a2; });
+    expect(token.regions.test).toBe(2);
+
+    token.markRegion('test', null, function () { return this.string.length; });
+    expect(token.regions.test).toBe(6);
+
+    token.markRegion('test', null, function () { return this.value; }, context);
+    expect(token.regions.test).toBe(99);
+  });
+
+  it('should check for suffixes', function() {
+    var token = new StemmerToken('tester');
+
+    expect(token.hasSuffix('er')).toBe(true);
+    expect(token.hasSuffix('st')).toBe(false);
+  });
+
+  it('should check for suffixes within a region', function() {
+    var token = new StemmerToken('tester').markRegion('region', 2);
+
+    expect(token.hasSuffixInRegion('st', 'region')).toBe(false);
+    expect(token.hasSuffixInRegion('ster', 'region')).toBe(true);
+    expect(token.hasSuffixInRegion('ester', 'region')).toBe(false);
+  });
+
+  it('should replace the suffix within a region', function() {
+    var t1 = new StemmerToken('tester').markRegion('region', 4),
+      t2   = new StemmerToken('tester').markRegion('region', 0);
+
+    t1.replaceSuffixInRegion('ter', '<s>', 'region');
+    expect(t1.string).toBe('tester');
+
+    t1.replaceSuffixInRegion('er', '<s>', 'region');
+    expect(t1.string).toBe('test<s>');
+
+    t2.replaceSuffixInRegion('protester', '<s>', 'region');
+    expect(t2.string).toBe('tester');
+  });
+
+});