natural 0.1.27 → 0.2.1

package/spec/sentence_analyzer_spec.js

@@ -26,4 +26,252 @@ describe('sentence analyzer', function() {
     it('should load', function() {
         var analyzer = new SentenceAnalyzer(null, function() { });
     });
+
+    it('should determine PP and SP, given a POS', function () {
+        var sentenceTags = [
+            {token: 'The', pos: 'DT'},
+            {token: 'angry', pos: 'JJ'},
+            {token: 'bear', pos: 'NN'},
+            {token: 'chased', pos: 'VB'},
+            {token: 'the', pos: 'DT'},
+            {token: 'frightened', pos: 'JJ'},
+            {token: 'little', pos: 'JJ'},
+            {token: 'squirrel', pos: 'NN'}
+        ];
+        new SentenceAnalyzer({tags: sentenceTags}, function (analyzer) {
+            analyzer.part(function (part) {
+                var posTags = part.posObj.tags;
+                for (var tagNum = 0; tagNum < posTags.length; tagNum++) {
+                    var posTag = posTags[tagNum];
+                    if (posTag.token === 'angry') {
+                        expect(posTag.spos).toEqual('SP');
+                    } else if (posTag.token === 'squirrel') {
+                        expect(posTag.spos).toEqual('PP');
+                    }
+                }
+                expect(analyzer.subjectToString().trim()).toEqual('The angry bear');
+                expect(analyzer.predicateToString().trim()).toEqual('chased the frightened little squirrel');
+                expect(analyzer.toString().trim()).toEqual('The angry bear chased the frightened little squirrel');
+                expect(analyzer.implicitYou()).toEqual(false);
+            });
+        });
+    });
+
+    it('should determine PP and SP given a POS that begins with a verb', function () {
+        var sentenceTags = [
+            {token: 'Vote', pos: 'VB'},
+            {token: 'for', pos: 'IN'},
+            {token: 'me', pos: 'PRP'}
+        ];
+        new SentenceAnalyzer({tags: sentenceTags}, function (analyzer) {
+            analyzer.part(function (part) {
+                var posTags = part.posObj.tags;
+                for (var tagNum = 0; tagNum < posTags.length; tagNum++) {
+                    var posTag = posTags[tagNum];
+                    if (posTag.token === 'Vote') {
+                        expect(posTag.spos).toEqual('PP');
+                    } else if (posTag.token === 'me') {
+                        expect(posTag.pp).toEqual(true);
+                    }
+                }
+                // Adds implicit you.
+                var lastTagNum = posTags.length - 1;
+                expect(posTags[lastTagNum].token).toEqual('You');
+                expect(posTags[lastTagNum].pos).toEqual('PRP');
+                expect(posTags[lastTagNum].added).toEqual(true);
+                expect(analyzer.implicitYou()).toEqual(true);
+            });
+        });
+    });
+
+    it('should look for EX before VB', function () {
+        var sentenceTags = [
+            {token: 'There', pos: 'EX'},
+            {token: 'is', pos: 'VB'},
+            {token: 'a', pos: 'DT'},
+            {token: 'house', pos: 'NN'},
+            {token: 'in', pos: 'IN'},
+            {token: 'the', pos: 'DT'},
+            {token: 'valley', pos: 'DT'}
+        ];
+        new SentenceAnalyzer({tags: sentenceTags}, function (analyzer) {
+            analyzer.part(function (part) {
+                var posTags = part.posObj.tags;
+                for (var tagNum = 0; tagNum < posTags.length; tagNum++) {
+                    var posTag = posTags[tagNum];
+                    if (posTag.token === 'There') {
+                        expect(posTag.spos).toEqual('SP');
+                    } else if (posTag.token === 'is') {
+                        expect(posTag.spos).toEqual('SP');
+                    }
+                }
+                expect(analyzer.subjectToString().trim()).toEqual('There is a house');
+                expect(analyzer.predicateToString().trim()).toEqual('');
+                expect(analyzer.toString().trim()).toEqual('There is a house in the valley');
+                expect(analyzer.implicitYou()).toEqual(false);
+            });
+        });
+    });
+
+    describe('#type', function () {
+
+        it('should determine a command without punctuation', function () {
+            var sentenceTags = [
+                {token: 'Vote', pos: 'VB'},
+                {token: 'for', pos: 'IN'},
+                {token: 'me', pos: 'PRP'}
+            ];
+            var punct = function () {
+                return [];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('COMMAND');
+                    });
+                });
+            });
+        });
+
+        it('should determine an interrogative beginning with who', function () {
+            var sentenceTags = [
+                {token: 'Who', pos: 'WP'},
+                {token: 'voted', pos: 'VB'}
+            ];
+            var punct = function () {
+                return [];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('INTERROGATIVE');
+                    });
+                });
+            });
+        });
+
+        it('should determine an interrogative ending with a personal pronoun', function () {
+            var sentenceTags = [
+                {token: 'Should', pos: 'MD'},
+                {token: 'we', pos: 'PRP'}
+            ];
+            var punct = function () {
+                return '';
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('INTERROGATIVE');
+                    });
+                });
+            });
+        });
+
+        it('should classify other sentences as unknown', function () {
+            var sentenceTags = [
+                {token: 'I', pos: 'PRP'},
+                {token: 'am', pos: 'VB'},
+                {token: 'unsure', pos: 'JJ'}
+            ];
+            var punct = function () {
+                return '';
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('UNKNOWN');
+                    });
+                });
+            });
+        });
+
+        it('should determine an interrogative ending with a question mark', function () {
+            var sentenceTags = [
+                {token: 'Do', pos: 'VB'},
+                {token: 'I', pos: 'PRP'},
+                {token: 'care', pos: 'VB'}
+            ];
+            var punct = function () {
+                return [{token: '?', pos: '.'}];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('INTERROGATIVE');
+                    });
+                });
+            });
+        });
+
+        it('should determine a command ending in an exclamation point', function () {
+            var sentenceTags = [
+                {token: 'Vote', pos: 'VB'},
+                {token: 'for', pos: 'IN'},
+                {token: 'me', pos: 'PRP'}
+            ];
+            var punct = function () {
+                return [{token: '!', pos: '.'}];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('COMMAND');
+                    });
+                });
+            });
+        });
+
+        it('should determine an exclamation ending in an exclamation point', function () {
+            var sentenceTags = [
+                {token: 'We', pos: 'PRP'},
+                {token: 'like', pos: 'VB'},
+                {token: 'sheep', pos: 'NN'}
+            ];
+            var punct = function () {
+                return [{token: '!', pos: '.'}];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('EXCLAMATORY');
+                    });
+                });
+            });
+        });
+
+        it('should determine a command ending with .', function () {
+            var sentenceTags = [
+                {token: 'Vote', pos: 'VB'},
+                {token: 'for', pos: 'IN'},
+                {token: 'me', pos: 'PRP'}
+            ];
+            var punct = function () {
+                return [{token: '.', pos: '.'}];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    analyzer.type(function (type) {
+                        expect(analyzer.senType).toEqual('COMMAND');
+                    });
+                });
+            });
+        });
+
+        it('should determine a declaration ending with a .', function () {
+            var sentenceTags = [
+                {token: 'We', pos: 'PRP'},
+                {token: 'like', pos: 'VB'},
+                {token: 'sheep', pos: 'NN'}
+            ];
+            var punct = function () {
+                return [{token: '.', pos: '.'}];
+            };
+            new SentenceAnalyzer({tags: sentenceTags, punct: punct}, function (analyzer) {
+                analyzer.part(function () {
+                    type = analyzer.type();
+                    expect(type).toEqual('DECLARATIVE');
+                });
+            });
+        });
+    });
 });

package/spec/shortest_path_tree_spec.js

@@ -0,0 +1,132 @@
+/*
+ Copyright (c) 2014, Lee Wenzhu
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+'use strict';
+
+var SPT = require('../lib/natural/util/shortest_path_tree'),
+    Topological = require('../lib/natural/util/topological'),
+    EdgeWeightedDigraph = require('../lib/natural/util/edge_weighted_digraph');
+
+describe("shortest path tree", function() {
+    var digraph = new EdgeWeightedDigraph();
+    digraph.add(5,4,0.35);
+    digraph.add(4,7,0.37);
+    digraph.add(5,7,0.28);
+    digraph.add(5,1,0.32);
+    digraph.add(4,0,0.38);
+    digraph.add(0,2,0.26);
+    digraph.add(3,7,0.39);
+    digraph.add(1,3,0.29);
+    digraph.add(7,2,0.34);
+    digraph.add(6,2,0.40);
+    digraph.add(3,6,0.52);
+    digraph.add(6,0,0.58);
+    digraph.add(6,4,0.93);
+    describe("edge weighted digraph normal operations", function() {
+        it("should get the numbers of vertexs", function() {
+            expect(digraph.v()).toBe(8);
+        });
+        it("should get the numbers of edges", function() {
+            expect(digraph.e()).toBe(13);
+        });
+        it("should print all item in digraph", function() {
+            expect(digraph.toString()).toBe('0 -> 2, 0.26\n' +
+                                            '1 -> 3, 0.29\n' +
+                                            '3 -> 7, 0.39\n' +
+                                            '3 -> 6, 0.52\n' +
+                                            '4 -> 7, 0.37\n' +
+                                            '4 -> 0, 0.38\n' +
+                                            '5 -> 4, 0.35\n' +
+                                            '5 -> 7, 0.28\n' +
+                                            '5 -> 1, 0.32\n' +
+                                            '6 -> 2, 0.4\n' +
+                                            '6 -> 0, 0.58\n' +
+                                            '6 -> 4, 0.93\n' +
+                                            '7 -> 2, 0.34');
+        });
+    });
+
+    describe("topo sort for digraph", function() {
+        it("should sort all the vertexs", function() {
+            var topoSort = new Topological(digraph);
+            expect(topoSort.isDAG()).toBe(true);
+            expect(topoSort.order()).toEqual([ 5, 1, 3, 6, 4, 7, 0, 2 ]);
+        });
+    });
+
+    describe("shortest path tree normal operations", function() {
+        var lpt = new SPT(digraph, 5);
+        expect(lpt.hasPathTo(0)).toBe(true);
+        expect(lpt.hasPathTo(1)).toBe(true);
+        expect(lpt.hasPathTo(2)).toBe(true);
+        expect(lpt.hasPathTo(3)).toBe(true);
+        expect(lpt.hasPathTo(4)).toBe(true);
+        expect(lpt.hasPathTo(5)).toBe(false);
+        expect(lpt.hasPathTo(6)).toBe(true);
+        expect(lpt.hasPathTo(7)).toBe(true);
+        expect(lpt.pathTo(0)).toEqual([ 5, 4, 0 ]);
+        expect(lpt.pathTo(1)).toEqual([ 5, 1 ]);
+        expect(lpt.pathTo(2)).toEqual([ 5, 7, 2 ]);
+        expect(lpt.pathTo(3)).toEqual([ 5, 1, 3 ]);
+        expect(lpt.pathTo(4)).toEqual([ 5, 4 ]);
+        expect(lpt.pathTo(5)).toEqual([ ]);
+        expect(lpt.pathTo(6)).toEqual([ 5, 1, 3, 6 ]);
+        expect(lpt.pathTo(7)).toEqual([ 5, 7 ]);
+        expect(lpt.getDistTo(0)).toBe(0.73);
+        expect(lpt.getDistTo(1)).toBe(0.32);
+        expect(lpt.getDistTo(2)).toBe(0.62);
+        expect(lpt.getDistTo(3)).toBe(0.61);
+        expect(lpt.getDistTo(4)).toBe(0.35);
+        expect(lpt.getDistTo(5)).toBe(0);
+        expect(lpt.getDistTo(6)).toBe(1.13);
+        expect(lpt.getDistTo(7)).toBe(0.28);
+    });
+
+    /**
+     * Check the issue #150
+     **/
+     describe("lightest weight path", function(){
+        it("should find the lightest weight path", function(){
+            var digraph = new EdgeWeightedDigraph();
+
+            digraph.add(1,3,0.29);
+            digraph.add(1,6,0);
+            digraph.add(3,6,0);
+
+            var spt = new SPT(digraph, 1);
+            var path = spt.pathTo(6);
+            expect(path).toEqual([1,6]);
+        });
+        it("should find the lightest weight path", function(){
+            var digraph = new EdgeWeightedDigraph();
+
+            digraph.add(1,3,-1);
+            digraph.add(1,6,0);
+            digraph.add(3,6,0);
+
+            var spt = new SPT(digraph, 1);
+            var path = spt.pathTo(6);
+            var hasPath = spt.hasPathTo(6);
+            expect(hasPath).toBe(true);
+            expect(path).toEqual([1,3,6]);
+        });
+     });
+});

package/spec/soundex_spec.js

@@ -65,15 +65,6 @@ describe('soundex', function() {
         it('sound condense sequences', function() {
             expect(soundex.condense('11222556')).toBe('1256');
         });
-
-        it('sound condense sequences around H and W', function() {
-            expect(soundex.condense('113w355')).toBe('135');
-        });
-        
-        it('sound drop irrelevant H and W', function() {
-            expect(soundex.condense('phonetics')).toBe('ponetics');
-            expect(soundex.condense('towering')).toBe('toering');            
-        });
     });
     
     describe('padRight0', function() {
@@ -132,6 +123,19 @@ describe('soundex', function() {
         expect('phonetics'.phonetics()).toBe('P532');
         expect('PHONETICS'.phonetics()).toBe('P532');        
     });
+
+    it('issue 221 -- inital vowels that duplicate cons codes', function() {
+        soundex.attach();
+        expect('Lloyd'.phonetics()).toBe('L300');
+        expect('Pfister'.phonetics()).toBe('P236');
+
+        expect('manhattan'.phonetics()).toBe('M535');
+        expect('Lukasiewicz'.phonetics()).toBe('L222');        
+
+        expect('Gauss'.phonetics()).toBe('G200');        
+        expect('Tymczak'.phonetics()).toBe('T522');        
+        expect('Jackson'.phonetics()).toBe('J250');                                
+    });    
     
     it('should tokenize and return string phonetics with string patch', function() {
         soundex.attach();

package/spec/spellcheck_spec.js

@@ -0,0 +1,92 @@
+
+
+var Spellcheck = require('../lib/natural/spellcheck/spellcheck.js');
+
+describe('spellcheck', function() {
+    
+    describe('distance 1 corrections', function() {
+        var spellcheck = new Spellcheck(['something']);
+        
+        it('should fix deletions', function() {
+            expect(spellcheck.getCorrections('smething')).toContain('something');
+        });
+        it('should fix insertions', function() {
+            expect(spellcheck.getCorrections('somethhing')).toContain('something');
+        });
+        it('should fix transpositions', function() {
+            expect(spellcheck.getCorrections('somehting')).toContain('something');
+        });
+        it('should fix replacements', function() {
+            expect(spellcheck.getCorrections('sometzing')).toContain('something');
+        });
+    });
+
+    describe('distance 2 corrections', function() {
+        var spellcheck = new Spellcheck(['something']);
+
+        it('should fix a deletion then a transposition', function() {
+            expect(spellcheck.getCorrections('smtehing', 2)).toContain('something');
+        });
+    });
+
+    describe('edits', function() {
+        var spellcheck = new Spellcheck([]);
+        var edits = spellcheck.edits('ab');
+
+        it('should compute insertions', function() {
+            var insertions = ['xab', 'axb', 'abx'];
+            for(var i in insertions) {
+                expect(edits.indexOf(insertions[i])).toBeGreaterThan(-1);
+            }
+        });
+        it('should compute deletions', function() {
+            var deletions = ['a', 'b']; 
+            for(var i in deletions) {
+                expect(edits.indexOf(deletions[i])).toBeGreaterThan(-1);
+            }
+        });
+        it('should compute transpositions', function() {
+            var transpositions = ['ba']
+            for(var i in transpositions) {
+                expect(edits.indexOf(transpositions[i])).toBeGreaterThan(-1);
+            }
+        });
+        it('should compute replacements', function() {
+            var replacements = ['zb', 'af']
+            for(var i in replacements) {
+                expect(edits.indexOf(replacements[i])).toBeGreaterThan(-1);
+            }
+        });
+    });
+
+    describe('edits up to distance', function() {
+        var spellcheck = new Spellcheck([]);
+        
+        it('should correctly compute edits at distance', function() {
+            var edits = spellcheck.editsWithMaxDistance('abc', 2);
+            expect(edits[1].indexOf('abzc')).toBeGreaterThan(-1); // 1 insertion
+            expect(edits[2].indexOf('a')).toBeGreaterThan(-1); // 2 deletions 
+        });
+    });
+
+    describe('boolean spellcheck', function() {
+        var spellcheck = new Spellcheck(['cat']);
+
+        it('should consider cat a word', function() {
+            expect(spellcheck.isCorrect('cat')).toBe(true);
+        });
+
+        it('should not consider dog a word', function() {
+            expect(spellcheck.isCorrect('dog')).toBe(false);
+        });
+    });
+
+    describe('special cases', function() {
+        var spellcheck = new Spellcheck(['cat']);
+
+        it('should return the input word as the most probable correction if it is already correct', function() {
+            expect(spellcheck.getCorrections('cat').indexOf('cat')).toBe(0);
+        });
+
+    });
+});

package/spec/stemmer_token_spec.js

@@ -0,0 +1,161 @@
+/*
+Copyright (c) 2014, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+var StemmerToken = require('../lib/natural/stemmers/token');
+
+describe('stemmer token', function () {
+
+  it('should receive a string', function () {
+    var string = 'test',
+      token    = new StemmerToken(string);
+
+    expect(token.string).toBe(string);
+  });
+
+  it('should hold the original token string', function () {
+    var string = 'test',
+      token    = new StemmerToken(string);
+
+    expect(token.original).toBe(string);
+  });
+
+  it('should replace all instances of a string', function () {
+    var string = 'tester',
+      token    = new StemmerToken(string);
+
+    token.replaceAll('e', 'a');
+    expect(token.string).toBe('tastar');
+
+    token.replaceAll('ar', 'er');
+    expect(token.string).toBe('taster');
+
+    token.replaceAll('r', '');
+    expect(token.string).toBe('taste');
+  });
+
+  it('should allow vowels to be set', function () {
+    var vowels    = 'aeiou',
+      vowelsArray = vowels.split(''),
+      token       = new StemmerToken('');
+
+    token.usingVowels(vowels);
+    expect(token.vowels).toBe(vowels);
+
+    token.usingVowels(vowelsArray);
+    expect(token.vowels).toBe(vowelsArray);
+  });
+
+  it('should check for vowels', function () {
+    var vowels = 'aeiou',
+      token    = new StemmerToken('test');
+
+    token.usingVowels(vowels);
+
+    expect(token.hasVowelAtIndex(0)).toBe(false);
+    expect(token.hasVowelAtIndex(1)).toBe(true);
+    expect(token.hasVowelAtIndex(99)).toBe(false);
+
+    token.usingVowels(vowels.split(''));
+
+    expect(token.hasVowelAtIndex(0)).toBe(false);
+    expect(token.hasVowelAtIndex(1)).toBe(true);
+    expect(token.hasVowelAtIndex(99)).toBe(false);
+  });
+
+  it('should find the next vowel', function () {
+    var token = new StemmerToken('tester').usingVowels('aeiou');
+
+    expect(token.nextVowelIndex(0)).toBe(1);
+    expect(token.nextVowelIndex(1)).toBe(1);
+    expect(token.nextVowelIndex(2)).toBe(4);
+    expect(token.nextVowelIndex(5)).toBe(6);
+    expect(token.nextVowelIndex(99)).toBe(6);
+    expect(token.nextVowelIndex(-1)).toBe(6);
+  });
+
+  it('should find the next consonant', function () {
+    var token = new StemmerToken('testee').usingVowels('aeiou');
+
+    expect(token.nextConsonantIndex(0)).toBe(0);
+    expect(token.nextConsonantIndex(1)).toBe(2);
+    expect(token.nextConsonantIndex(5)).toBe(6);
+    expect(token.nextConsonantIndex(99)).toBe(6);
+    expect(token.nextConsonantIndex(-1)).toBe(6);
+  });
+
+  it('should mark regions', function() {
+    var token = new StemmerToken('tester');
+
+    token.markRegion('test', 1);
+    expect(token.regions.test).toBe(1);
+  });
+
+  it('should mark regions with a callback', function() {
+    var token = new StemmerToken('tester'),
+      context = { value: 99 };
+
+    token.markRegion('test', 1, function (arg) { return arg; });
+    expect(token.regions.test).toBe(1);
+
+    token.markRegion('test', [1], function (arg) { return arg; });
+    expect(token.regions.test).toBe(1);
+
+    token.markRegion('test', [1, 1], function (a1, a2) { return a1 + a2; });
+    expect(token.regions.test).toBe(2);
+
+    token.markRegion('test', null, function () { return this.string.length; });
+    expect(token.regions.test).toBe(6);
+
+    token.markRegion('test', null, function () { return this.value; }, context);
+    expect(token.regions.test).toBe(99);
+  });
+
+  it('should check for suffixes', function() {
+    var token = new StemmerToken('tester');
+
+    expect(token.hasSuffix('er')).toBe(true);
+    expect(token.hasSuffix('st')).toBe(false);
+  });
+
+  it('should check for suffixes within a region', function() {
+    var token = new StemmerToken('tester').markRegion('region', 2);
+
+    expect(token.hasSuffixInRegion('st', 'region')).toBe(false);
+    expect(token.hasSuffixInRegion('ster', 'region')).toBe(true);
+    expect(token.hasSuffixInRegion('ester', 'region')).toBe(false);
+  });
+
+  it('should replace the suffix within a region', function() {
+    var t1 = new StemmerToken('tester').markRegion('region', 4),
+      t2   = new StemmerToken('tester').markRegion('region', 0);
+
+    t1.replaceSuffixInRegion('ter', '<s>', 'region');
+    expect(t1.string).toBe('tester');
+
+    t1.replaceSuffixInRegion('er', '<s>', 'region');
+    expect(t1.string).toBe('test<s>');
+
+    t2.replaceSuffixInRegion('protester', '<s>', 'region');
+    expect(t2.string).toBe('tester');
+  });
+
+});