natural 0.1.27 → 0.2.1

package/spec/tfidf_spec.js

@@ -22,7 +22,7 @@ THE SOFTWARE.
 
 var TfIdf = require('../lib/natural/tfidf/tfidf');
 var tfidf;
-        
+
 describe('tfidf', function() {
     describe('stateless operations', function() {
         it('should tf', function() {
@@ -36,15 +36,55 @@ describe('tfidf', function() {
         it('should store and recall keys', function() {
             tfidf = new TfIdf();
             tfidf.addDocument('document one', 'un');
-	        tfidf.addDocument('document Two', 'deux');
-            
+            tfidf.addDocument('document Two', 'deux');
+
+            tfidf.tfidfs('two', function(i, tfidf, key) {
+                if(i === 0)
+                    expect(key).toBe('un');
+                else
+                    expect(key).toBe('deux');
+            });
+        });
+
+        it('should handle a deserialized object passed to the constructor', function () {
+            tfidf = new TfIdf({
+                documents: [
+                    {__key: 'un', document: 1, one: 1},
+                    {__key: 'deux', document: 1, two: 1}
+                ]
+            });
+            tfidf.tfidfs('two', function(i, tfidf, key) {
+                if(i === 0)
+                    expect(key).toBe('un');
+                else
+                    expect(key).toBe('deux');
+            });
+        });
+
+        it('should work when called without a callback', function () {
+            tfidf = new TfIdf({
+                documents: [
+                    {__key: 'un', document: 1, one: 1},
+                    {__key: 'deux', document: 1, two: 1}
+                ]
+            });
+            tfidfs = tfidf.tfidfs('two');
+            expect(tfidfs[1]).toBe(1 + Math.log( 2.0 / 2.0 ));
+        });
+
+        it('should work with the restoreCache flag set to true', function() {
+            tfidf = new TfIdf();
+            tfidf.addDocument('document one', 'un');
+            expect(tfidf.idf("one")).toBe(1 + Math.log( 1.0 / 2.0 ));
+            tfidf.addDocument('document Two', 'deux', true);
+
             tfidf.tfidfs('two', function(i, tfidf, key) {
-                if(i == 0)
+                if(i === 0)
                     expect(key).toBe('un');
                 else
                     expect(key).toBe('deux');
             });
-        });    
+        });
     });
 
     describe('stateful operations', function() {
@@ -54,15 +94,15 @@ describe('tfidf', function() {
             tfidf.addDocument('document Two');
         });
 
-    	it('should list important terms', function() {
+        it('should list important terms', function() {
             var terms = tfidf.listTerms(0);
             expect(terms[0].tfidf).toBeGreaterThan(terms[1].tfidf);
-    	});
+        });
     });
 
     describe("special cases", function(){
 
-        // In response to 
+        // In response to
         it("should handle reserved function names correctly in documents", function(){
             var reservedWords = [
                 'toString',
@@ -75,11 +115,18 @@ describe('tfidf', function() {
             ];
             tfidf = new TfIdf();
             tfidf.addDocument(reservedWords.join(" "));
-            
+
             for(var i in reservedWords) {
-                expect(tfidf.tfidf(reservedWords[i], 0)).toBe(0);
+                expect(tfidf.tfidf(reservedWords[i], 0)).toBe(1*(1+Math.log(1/2)));
             }
         });
+
+        it('should handle an array passed to tfidf()', function() {
+            tfidf = new TfIdf();
+            var terms = ['this', 'document', 'is', 'about', 'poetry'];
+            tfidf.addDocument(terms.join(" "));
+            expect(tfidf.tfidf(terms, 0)).toBe( 2 * (1 + Math.log( 1.0 / 2.0 )) );
+        });
     });
 
     describe("correct calculations", function(){
@@ -92,7 +139,20 @@ describe('tfidf', function() {
             tfidf.addDocument('this document is about ruby and node.');
             tfidf.addDocument('this document is about node. it has node examples');
 
-            expect(tfidf.idf("node")).toBe(Math.log( 4.0 / 3.0 ));
+            expect(tfidf.idf("node")).toBe(1 + Math.log( 4.0 / 4.0 ));
+        });
+
+        it("should compute idf correctly with non-string documents", function(){
+
+            tfidf = new TfIdf();
+            tfidf.addDocument('this document is about node.');
+            tfidf.addDocument('this document is about ruby.');
+            tfidf.addDocument('this document is about ruby and node.');
+            tfidf.addDocument('this document is about node. it has node examples');
+            tfidf.addDocument({text: 'this document is about python'});
+            tfidf.addDocument(['this', 'document', 'is', 'about', 'node', 'and', 'JavaScript']);
+
+            expect(tfidf.idf("node")).toBe(1 + Math.log( 6.0 / 5.0 ));
         });
 
         it("should compute tf correctly", function(){
@@ -106,10 +166,10 @@ describe('tfidf', function() {
         it("should compute tf-idf correctly", function(){
 
             var correctCalculations = [
-                1 * Math.log( 4.0 / 3.0 ),
+                1 * (1 + Math.log( 4.0 / 4.0 )),
                 0,
-                2 * Math.log( 4.0 / 3.0 ),
-                1 * Math.log( 4.0 / 2.0 )
+                2 * (1 + Math.log( 4.0 / 4.0 )),
+                1 * (1 + Math.log( 4.0 / 3.0 ))
             ];
 
             tfidf = new TfIdf();
@@ -119,15 +179,22 @@ describe('tfidf', function() {
             tfidf.addDocument('this document is about node. it has node examples', {node:2, ruby:1});
 
             tfidf.tfidfs('node', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.node])
+                expect(measure).toBe(correctCalculations[k.node]);
             });
 
             tfidf.tfidfs('ruby', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.ruby])
+                expect(measure).toBe(correctCalculations[k.ruby]);
             });
 
         });
 
+        it("should not return NaN if a term is not present in any documents", function() {
+            tfidf = new TfIdf();
+            tfidf.addDocument('this document is about node.');
+
+            expect(tfidf.tfidf('ruby', 0)).toBe(0);
+        });
+
         // This test assures that tf-idf is computed correctly before and after a document is added
         // Computes and tests a few tf-idfs, then adds a document and ensures that those terms tf-idf value
         // is updated accordingly.
@@ -140,14 +207,14 @@ describe('tfidf', function() {
             tfidf.addDocument('this document is about ruby.', 1);
 
             // check the tf-idf for 'node'
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * Math.log( 2.0 / 1.0 ) );
+            expect( tfidf.tfidf("node", 0) ).toBe( 1 * ( 1 + Math.log( 2.0 / 2.0 ) ));
 
             // Add 2 more documents
             tfidf.addDocument('this document is about ruby and node.');
             tfidf.addDocument('this document is about node. it has node examples');
 
             // Ensure that the tf-idf in the same document has changed to reflect the new idf.
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * Math.log( 4.0 / 3.0 ) );
+            expect( tfidf.tfidf("node", 0) ).toBe( 1 * ( 1 + Math.log( 4.0 / 4.0 ) ));
         });
 
         // Test tf-idf computation on files loaded using readFileSync
@@ -161,38 +228,96 @@ describe('tfidf', function() {
             tfidf.addFileSync("spec/test_data/tfidf_document4.txt", null, {node:2, ruby:1});
 
             var correctCalculations = [
-                1 * Math.log( 4.0 / 3.0 ),
+                1 * (1 + Math.log( 4.0 / 4.0 )),
                 0,
-                2 * Math.log( 4.0 / 3.0 ),
-                1 * Math.log( 4.0 / 2.0 )
+                2 * (1 + Math.log( 4.0 / 4.0 )),
+                1 * (1 + Math.log( 4.0 / 3.0 ))
             ];
 
             tfidf.tfidfs('node', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.node])
+                expect(measure).toBe(correctCalculations[k.node]);
             });
 
             tfidf.tfidfs('ruby', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.ruby])
+                expect(measure).toBe(correctCalculations[k.ruby]);
             });
+
+            // addFileSync with restoreCache flag set to true.
+            tfidf.addFileSync("spec/test_data/tfidf_document4.txt", null, {node:2, ruby:1}, true);
         });
 
         // Test idf caching when adding documents from addFileSync
-        if("should update a terms tf-idf score after adding documents from addFileSync", function(){
+        it("should update a terms tf-idf score after adding documents from addFileSync", function(){
             tfidf = new TfIdf();
 
             // Add 2 documents
-            tfidf.addFileSync("spec/test_data/tfidf_document1.txt", 0);
-            tfidf.addFileSync("spec/test_data/tfidf_document2.txt", 1);
+            tfidf.addFileSync("spec/test_data/tfidf_document1.txt", null, 0);
+            tfidf.addFileSync("spec/test_data/tfidf_document2.txt", null, 1);
 
             // check the tf-idf for 'node'
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * Math.log( 2.0 / 1.0 ) );
+            expect( tfidf.tfidf("node", 0) ).toBe( 1 * (1 + Math.log( 2.0 / 2.0 ) ));
 
             // Add 2 more documents
             tfidf.addFileSync("spec/test_data/tfidf_document3.txt");
             tfidf.addFileSync("spec/test_data/tfidf_document4.txt");
 
             // Ensure that the tf-idf in the same document has changed to reflect the new idf.
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * Math.log( 4.0 / 3.0 ) );
+            expect( tfidf.tfidf("node", 0) ).toBe( 1 * (1 + Math.log( 4.0 / 4.0 ) ));
+        });
+
+        // Test idf.setTokenizer
+        it('should allow for specific types of Tokenizers', function(){
+            tfidf = new TfIdf();
+
+            tfidf.addDocument('this document isn\'t about node.', 0);
+            tfidf.addDocument('that doc is about node.', 1);
+            expect( tfidf.tfidf('n\'t', 0) ).toBe(0);
+            expect( tfidf.tfidf('isn', 0) ).toBe( 1 * (1 + Math.log( 2 / 2 ) ));
+
+            tfidf = new TfIdf();
+
+            tfidf.addDocument('this document isn\'t about node.', 0);
+            tfidf.addDocument('this document isn\'t about node.', 1);
+
+            expect( tfidf.tfidf('isn', 0) ).toBe(1 * (1 + Math.log (2/3)));
+
+            tfidf = new TfIdf();
+            var TreebankWordTokenizer = require('../lib/natural/tokenizers/treebank_word_tokenizer');
+            var tokenizer = new TreebankWordTokenizer();
+
+            tfidf.addDocument('this document isn\'t about node.', 0);
+            tfidf.setTokenizer(tokenizer);
+            tfidf.addDocument('this doc isn\'t about node.', 1);
+
+            expect( tfidf.tfidf('isn', 0) ).toBe( 1 * ( 1 + Math.log( 2 / 2 ) ));
+            expect( tfidf.tfidf('n\'t', 1) ).toBe( 1 * ( 1 + Math.log( 2 / 2 ) ));
+            expect( tfidf.tfidf('isn', 1) ).toBe(0);
+        });
+
+        it('should require a valid tokenizer when using setTokenizer', function(){
+            tfidf = new TfIdf();
+
+            expect( function() { tfidf.setTokenizer(1); } ).toThrow(new Error('Expected a valid Tokenizer'));
+            expect( function() { tfidf.setTokenizer({}); } ).toThrow(new Error('Expected a valid Tokenizer'));
+        });
+
+        // Test encoding for addFileSync
+        it('should use the specified encoding for addFileSync', function(){
+
+            tfidf = new TfIdf();
+
+            tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'base64');
+            tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'utf8');
+
+            expect( tfidf.tfidf('dghpcybkb2n1bwvudcbpcybhym91dcbub2rllg', 0) ).toBe( 1 * ( 1 + Math.log( 2.0 / 2.0 ) ));
+        });
+
+        // Test encoding check for addFileSync
+        it('should require a valid encoding for addFileSync', function(){
+
+            tfidf = new TfIdf();
+
+            expect( function() { tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'foobar'); } ).toThrow(new Error('Invalid encoding: foobar'));
         });
     });
 });

package/spec/trie_spec.js

@@ -123,7 +123,24 @@ describe('trie', function() {
 			expect(results).not.toContain("cd");
 		});
 
-		it("should be able to execute the find_prefix search with a match", function(){
+		it("should be able to guess all full prefix matched words.", function(){
+			var trie = new Trie();
+			trie.addStrings(["a","ab","bc","cd","abc"]);
+			var results = trie.keysWithPrefix("a");
+			expect(results).toContain("a");
+			expect(results).toContain("ab");
+			expect(results).toContain("abc");
+			expect(results).not.toContain("bc");
+			expect(results).not.toContain("cd");
+			results = trie.keysWithPrefix("ab");
+			expect(results).toContain("ab");
+			expect(results).toContain("abc");
+			expect(results).not.toContain("a");
+			expect(results).not.toContain("bc");
+			expect(results).not.toContain("cd");
+		});
+
+        it("should be able to execute the find_prefix search with a match", function(){
 			var trie = new Trie();
 			trie.addStrings(["their", "and", "they"]);
 			var results = trie.findPrefix("theyre");
@@ -131,7 +148,14 @@ describe('trie', function() {
 			expect(results[1]).toBe("re");
 		});
 
-		it("should be able to execute the find_prefix search without a match", function(){
+        it("should return empty array if no full prefix matches found.", function(){
+            var trie = new Trie();
+            trie.addStrings(["a","ab","bc","cd","abc"]);
+            var results = trie.keysWithPrefix("not-found");
+            expect(results.length).toEqual(0);
+        });
+
+        it("should be able to execute the find_prefix search without a match", function(){
 			var trie = new Trie();
 			trie.addStrings(["their", "and"]);
 			var results = trie.findPrefix("theyre");
@@ -179,4 +203,4 @@ describe('trie', function() {
 			expect(results[1]).toBe("re");
 		});
 	});
-});
+});