natural 0.2.1 → 0.4.0

package/spec/test_data/tfdif_testcorpus.txt

@@ -1,7 +0,0 @@
-50
-the:23
-a:17
-girl:1
-moon:1
-said:5
-phone:2

package/spec/test_data/tfidf_document1.txt

@@ -1 +0,0 @@
-this document is about node.

package/spec/test_data/tfidf_document2.txt

@@ -1 +0,0 @@
-this document is about ruby.

package/spec/test_data/tfidf_document3.txt

@@ -1 +0,0 @@
-this document is about ruby and node.

package/spec/test_data/tfidf_document4.txt

@@ -1 +0,0 @@
-this document is about node. it has node examples

package/spec/test_data/tfidf_teststopwords.txt

@@ -1 +0,0 @@
-moon

package/spec/test_data/wordnet_mini.sql

@@ -1,95 +0,0 @@
-PRAGMA foreign_keys=OFF;
-BEGIN TRANSACTION;
-CREATE TABLE adjpositions (synsetid integer not null, wordid integer not null, position varchar(2) not null, primary key (synsetid, wordid));
-CREATE TABLE adjpositiontypes (position varchar(2) not null, positionname varchar(24) not null, primary key (position));
-CREATE TABLE casedwords (casedwordid integer not null, wordid integer not null, cased varchar(80) not null unique, primary key (casedwordid));
-CREATE TABLE lexdomains (lexdomainid smallint not null, lexdomainname varchar(32), pos char, primary key (lexdomainid));
-CREATE TABLE lexlinks (word1id integer not null, synset1id integer not null, word2id integer not null, synset2id integer not null, linkid smallint not null, primary key (word1id, synset1id, word2id, synset2id, linkid));
-CREATE TABLE linktypes (linkid smallint not null, link varchar(50), recurses integer not null, primary key (linkid));
-CREATE TABLE morphmaps (morphid integer not null, pos char not null, wordid integer not null, primary key (morphid, pos, wordid));
-CREATE TABLE morphs (morphid integer not null, morph varchar(70) not null unique, primary key (morphid));
-CREATE TABLE postypes (pos char not null, posname varchar(20) not null, primary key (pos));
-CREATE TABLE samples (synsetid integer not null, sampleid smallint not null, sample longvarchar not null, primary key (synsetid, sampleid));
-CREATE TABLE semlinks (synset1id integer not null, synset2id integer not null, linkid smallint not null, primary key (synset1id, synset2id, linkid));
-CREATE TABLE senses (wordid integer not null, synsetid integer not null, casedwordid integer, senseid integer unique, sensenum smallint not null, lexid smallint not null, tagcount integer, sensekey varchar(100) unique, primary key (wordid, synsetid));
-INSERT INTO "senses" VALUES(138336,300491511,NULL,194444,4,0,0,'unique%5:00:00:unusual:00');
-INSERT INTO "senses" VALUES(138336,300505853,NULL,194441,1,0,10,'unique%5:00:00:incomparable:00');
-INSERT INTO "senses" VALUES(138336,301105042,NULL,194443,2,0,2,'unique%5:00:00:specific:00');
-INSERT INTO "senses" VALUES(138336,302215087,NULL,194442,3,0,0,'unique%5:00:00:single:05');
-INSERT INTO "senses" VALUES(138337,400175490,NULL,194445,1,0,4,'uniquely%4:02:00::');
-INSERT INTO "senses" VALUES(138338,104763650,NULL,194446,1,0,3,'uniqueness%1:07:00::');
-INSERT INTO "senses" VALUES(58843,103076708,NULL,81518,4,0,0,'good%1:06:00::');
-INSERT INTO "senses" VALUES(58843,104849241,NULL,81520,2,1,9,'good%1:07:01::');
-INSERT INTO "senses" VALUES(58843,105142180,NULL,81521,3,2,6,'good%1:07:02::');
-INSERT INTO "senses" VALUES(58843,105159725,NULL,81519,1,0,11,'good%1:07:00::');
-INSERT INTO "senses" VALUES(58843,300064787,NULL,81526,5,0,5,'good%5:00:00:advantageous:00');
-INSERT INTO "senses" VALUES(58843,300106020,NULL,81527,2,0,22,'good%5:00:00:ample:00');
-INSERT INTO "senses" VALUES(58843,300452883,NULL,81528,10,0,2,'good%5:00:00:close:02');
-INSERT INTO "senses" VALUES(58843,300523364,NULL,81529,9,0,2,'good%5:00:00:complete:00');
-INSERT INTO "senses" VALUES(58843,300775611,NULL,81530,21,0,0,'good%5:00:00:discriminating:00');
-INSERT INTO "senses" VALUES(58843,300832784,NULL,81538,14,0,1,'good%5:00:00:operative:00');
-INSERT INTO "senses" VALUES(58843,301048762,NULL,81531,13,0,1,'good%5:00:00:fortunate:00');
-INSERT INTO "senses" VALUES(58843,301068306,NULL,81532,20,0,0,'good%5:00:00:fresh:01');
-INSERT INTO "senses" VALUES(58843,301116026,NULL,81533,19,0,0,'good%5:00:00:genuine:00');
-INSERT INTO "senses" VALUES(58843,301123148,NULL,81522,1,1,190,'good%3:00:01::');
-INSERT INTO "senses" VALUES(58843,301129977,NULL,81523,3,2,12,'good%3:00:02::');
-INSERT INTO "senses" VALUES(58843,301166413,NULL,81534,18,0,0,'good%5:00:00:healthful:00');
-INSERT INTO "senses" VALUES(58843,301171213,NULL,81535,17,0,0,'good%5:00:00:healthy:00');
-INSERT INTO "senses" VALUES(58843,301333477,NULL,81536,16,0,0,'good%5:00:00:intellectual:00');
-INSERT INTO "senses" VALUES(58843,301586752,NULL,81537,6,0,4,'good%5:00:00:nice:00');
-INSERT INTO "senses" VALUES(58843,301661289,NULL,81539,12,0,1,'good%5:00:00:opportune:00');
-INSERT INTO "senses" VALUES(58843,301808329,NULL,81540,15,0,0,'good%5:00:00:pleasing:00');
-INSERT INTO "senses" VALUES(58843,301983162,NULL,81541,4,0,10,'good%5:00:00:reputable:00');
-INSERT INTO "senses" VALUES(58843,302036934,NULL,81542,7,0,3,'good%5:00:00:righteous:00');
-INSERT INTO "senses" VALUES(58843,302226162,NULL,81543,8,0,2,'good%5:00:00:skilled:00');
-INSERT INTO "senses" VALUES(58843,302273643,NULL,81544,11,1,1,'good%5:00:01:sound:00');
-INSERT INTO "senses" VALUES(58843,400011093,NULL,81524,1,0,9,'good%4:02:00::');
-INSERT INTO "senses" VALUES(58843,400057388,NULL,81525,2,1,4,'good%4:02:01::');
-INSERT INTO "senses" VALUES(27771,103076708,NULL,38135,1,0,4,'commodity%1:06:00::');
-INSERT INTO "senses" VALUES(134408,103076708,NULL,188782,1,0,0,'trade_good%1:06:00::');
-CREATE TABLE synsets (synsetid integer not null, pos char, lexdomainid smallint not null, definition longvarchar, primary key (synsetid));
-INSERT INTO "synsets" VALUES(103076708,'n',6,'articles of commerce');
-INSERT INTO "synsets" VALUES(104763650,'n',7,'the quality of being one of a kind');
-INSERT INTO "synsets" VALUES(104849241,'n',7,'moral excellence or admirableness');
-INSERT INTO "synsets" VALUES(105142180,'n',7,'that which is pleasing or valuable or useful');
-INSERT INTO "synsets" VALUES(105159725,'n',7,'benefit');
-INSERT INTO "synsets" VALUES(300064787,'s',0,'promoting or enhancing well-being');
-INSERT INTO "synsets" VALUES(300106020,'s',0,'having the normally expected amount');
-INSERT INTO "synsets" VALUES(300452883,'s',0,'with or in a close or intimate relationship');
-INSERT INTO "synsets" VALUES(300491511,'s',0,'highly unusual or rare but not the single instance');
-INSERT INTO "synsets" VALUES(300505853,'s',0,'radically distinctive and without equal');
-INSERT INTO "synsets" VALUES(300523364,'s',0,'thorough');
-INSERT INTO "synsets" VALUES(300775611,'s',0,'generally admired');
-INSERT INTO "synsets" VALUES(300832784,'s',0,'exerting force or influence');
-INSERT INTO "synsets" VALUES(301048762,'s',0,'resulting favorably');
-INSERT INTO "synsets" VALUES(301068306,'s',0,'not left to spoil');
-INSERT INTO "synsets" VALUES(301105042,'s',0,'(followed by `to'') applying exclusively to a given category or condition or locality');
-INSERT INTO "synsets" VALUES(301116026,'s',0,'not forged');
-INSERT INTO "synsets" VALUES(301123148,'a',0,'having desirable or positive qualities especially those suitable for a thing specified');
-INSERT INTO "synsets" VALUES(301129977,'a',0,'morally admirable');
-INSERT INTO "synsets" VALUES(301166413,'s',0,'tending to promote physical well-being; beneficial to health');
-INSERT INTO "synsets" VALUES(301171213,'s',0,'in excellent physical condition');
-INSERT INTO "synsets" VALUES(301333477,'s',0,'appealing to the mind');
-INSERT INTO "synsets" VALUES(301586752,'s',0,'agreeable or pleasing');
-INSERT INTO "synsets" VALUES(301661289,'s',0,'most suitable or right for a particular purpose');
-INSERT INTO "synsets" VALUES(301808329,'s',0,'capable of pleasing');
-INSERT INTO "synsets" VALUES(301983162,'s',0,'deserving of esteem and respect');
-INSERT INTO "synsets" VALUES(302036934,'s',0,'of moral excellence');
-INSERT INTO "synsets" VALUES(302215087,'s',0,'the single one of its kind');
-INSERT INTO "synsets" VALUES(302226162,'s',0,'having or showing knowledge and skill and aptitude');
-INSERT INTO "synsets" VALUES(302273643,'s',0,'financially sound');
-INSERT INTO "synsets" VALUES(400011093,'r',2,'(often used as a combining form) in a good or proper or satisfactory manner or to a high standard (`good'' is a nonstandard dialectal variant for `well'')');
-INSERT INTO "synsets" VALUES(400057388,'r',2,'completely and absolutely (`good'' is sometimes used informally for `thoroughly'')');
-INSERT INTO "synsets" VALUES(400175490,'r',2,'so as to be unique');
-CREATE TABLE vframemaps (synsetid integer not null, wordid integer not null, frameid smallint not null, primary key (synsetid, wordid, frameid));
-CREATE TABLE vframes (frameid smallint not null, frame varchar(50), primary key (frameid));
-CREATE TABLE vframesentencemaps (synsetid integer not null, wordid integer not null, sentenceid smallint not null, primary key (synsetid, wordid, sentenceid));
-CREATE TABLE vframesentences (sentenceid smallint not null, sentence longvarchar, primary key (sentenceid));
-CREATE TABLE words (wordid integer not null, lemma varchar(80) not null unique, primary key (wordid));
-INSERT INTO "words" VALUES(27771,'commodity');
-INSERT INTO "words" VALUES(58843,'good');
-INSERT INTO "words" VALUES(134408,'trade good');
-INSERT INTO "words" VALUES(138336,'unique');
-INSERT INTO "words" VALUES(138337,'uniquely');
-INSERT INTO "words" VALUES(138338,'uniqueness');
-COMMIT;

package/spec/tfidf_spec.js

@@ -1,323 +0,0 @@
-/*
-Copyright (c) 2011, Rob Ellis, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var TfIdf = require('../lib/natural/tfidf/tfidf');
-var tfidf;
-
-describe('tfidf', function() {
-    describe('stateless operations', function() {
-        it('should tf', function() {
-            expect(TfIdf.tf('document', { document : 2, one : 1 })).toBe(2);
-            expect(TfIdf.tf('document', { greetings : 1, program : 1 })).toBe(0);
-            expect(TfIdf.tf('program', { greetings : 1, program : 1 })).toBe(1);
-        });
-    });
-
-    describe('keys', function() {
-        it('should store and recall keys', function() {
-            tfidf = new TfIdf();
-            tfidf.addDocument('document one', 'un');
-            tfidf.addDocument('document Two', 'deux');
-
-            tfidf.tfidfs('two', function(i, tfidf, key) {
-                if(i === 0)
-                    expect(key).toBe('un');
-                else
-                    expect(key).toBe('deux');
-            });
-        });
-
-        it('should handle a deserialized object passed to the constructor', function () {
-            tfidf = new TfIdf({
-                documents: [
-                    {__key: 'un', document: 1, one: 1},
-                    {__key: 'deux', document: 1, two: 1}
-                ]
-            });
-            tfidf.tfidfs('two', function(i, tfidf, key) {
-                if(i === 0)
-                    expect(key).toBe('un');
-                else
-                    expect(key).toBe('deux');
-            });
-        });
-
-        it('should work when called without a callback', function () {
-            tfidf = new TfIdf({
-                documents: [
-                    {__key: 'un', document: 1, one: 1},
-                    {__key: 'deux', document: 1, two: 1}
-                ]
-            });
-            tfidfs = tfidf.tfidfs('two');
-            expect(tfidfs[1]).toBe(1 + Math.log( 2.0 / 2.0 ));
-        });
-
-        it('should work with the restoreCache flag set to true', function() {
-            tfidf = new TfIdf();
-            tfidf.addDocument('document one', 'un');
-            expect(tfidf.idf("one")).toBe(1 + Math.log( 1.0 / 2.0 ));
-            tfidf.addDocument('document Two', 'deux', true);
-
-            tfidf.tfidfs('two', function(i, tfidf, key) {
-                if(i === 0)
-                    expect(key).toBe('un');
-                else
-                    expect(key).toBe('deux');
-            });
-        });
-    });
-
-    describe('stateful operations', function() {
-        beforeEach(function() {
-            tfidf = new TfIdf();
-            tfidf.addDocument('document one');
-            tfidf.addDocument('document Two');
-        });
-
-        it('should list important terms', function() {
-            var terms = tfidf.listTerms(0);
-            expect(terms[0].tfidf).toBeGreaterThan(terms[1].tfidf);
-        });
-    });
-
-    describe("special cases", function(){
-
-        // In response to
-        it("should handle reserved function names correctly in documents", function(){
-            var reservedWords = [
-                'toString',
-                'toLocaleString',
-                'valueOf',
-                'hasOwnProperty',
-                'isPrototypeOf',
-                'propertyIsEnumerable',
-                'constructor'
-            ];
-            tfidf = new TfIdf();
-            tfidf.addDocument(reservedWords.join(" "));
-
-            for(var i in reservedWords) {
-                expect(tfidf.tfidf(reservedWords[i], 0)).toBe(1*(1+Math.log(1/2)));
-            }
-        });
-
-        it('should handle an array passed to tfidf()', function() {
-            tfidf = new TfIdf();
-            var terms = ['this', 'document', 'is', 'about', 'poetry'];
-            tfidf.addDocument(terms.join(" "));
-            expect(tfidf.tfidf(terms, 0)).toBe( 2 * (1 + Math.log( 1.0 / 2.0 )) );
-        });
-    });
-
-    describe("correct calculations", function(){
-
-        it("should compute idf correctly", function(){
-
-            tfidf = new TfIdf();
-            tfidf.addDocument('this document is about node.');
-            tfidf.addDocument('this document is about ruby.');
-            tfidf.addDocument('this document is about ruby and node.');
-            tfidf.addDocument('this document is about node. it has node examples');
-
-            expect(tfidf.idf("node")).toBe(1 + Math.log( 4.0 / 4.0 ));
-        });
-
-        it("should compute idf correctly with non-string documents", function(){
-
-            tfidf = new TfIdf();
-            tfidf.addDocument('this document is about node.');
-            tfidf.addDocument('this document is about ruby.');
-            tfidf.addDocument('this document is about ruby and node.');
-            tfidf.addDocument('this document is about node. it has node examples');
-            tfidf.addDocument({text: 'this document is about python'});
-            tfidf.addDocument(['this', 'document', 'is', 'about', 'node', 'and', 'JavaScript']);
-
-            expect(tfidf.idf("node")).toBe(1 + Math.log( 6.0 / 5.0 ));
-        });
-
-        it("should compute tf correctly", function(){
-            expect(TfIdf.tf("node", {this:1, document:1, is:1, about:1, node:1})).toBe(1);
-            expect(TfIdf.tf("node", {this:1, document:1, is:1, about:1, ruby:1})).toBe(0);
-            expect(TfIdf.tf("node", {this:1, document:1, is:1, about:1, ruby:1, and:1, node:1})).toBe(1);
-            expect(TfIdf.tf("node", {this:1, document:1, is:1, about:1, node:2, it:1, has:1, examples:1})).toBe(2);
-        });
-
-        // This is a test of the use case outlined in the readme.
-        it("should compute tf-idf correctly", function(){
-
-            var correctCalculations = [
-                1 * (1 + Math.log( 4.0 / 4.0 )),
-                0,
-                2 * (1 + Math.log( 4.0 / 4.0 )),
-                1 * (1 + Math.log( 4.0 / 3.0 ))
-            ];
-
-            tfidf = new TfIdf();
-            tfidf.addDocument('this document is about node.', {node: 0, ruby:1});
-            tfidf.addDocument('this document is about ruby.', {node:1, ruby:3});
-            tfidf.addDocument('this document is about ruby and node.', {node:0, ruby:3});
-            tfidf.addDocument('this document is about node. it has node examples', {node:2, ruby:1});
-
-            tfidf.tfidfs('node', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.node]);
-            });
-
-            tfidf.tfidfs('ruby', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.ruby]);
-            });
-
-        });
-
-        it("should not return NaN if a term is not present in any documents", function() {
-            tfidf = new TfIdf();
-            tfidf.addDocument('this document is about node.');
-
-            expect(tfidf.tfidf('ruby', 0)).toBe(0);
-        });
-
-        // This test assures that tf-idf is computed correctly before and after a document is added
-        // Computes and tests a few tf-idfs, then adds a document and ensures that those terms tf-idf value
-        // is updated accordingly.
-        it("should update a terms tf-idf score after adding documents", function(){
-
-            tfidf = new TfIdf();
-
-            // Add 2 documents
-            tfidf.addDocument('this document is about node.', 0);
-            tfidf.addDocument('this document is about ruby.', 1);
-
-            // check the tf-idf for 'node'
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * ( 1 + Math.log( 2.0 / 2.0 ) ));
-
-            // Add 2 more documents
-            tfidf.addDocument('this document is about ruby and node.');
-            tfidf.addDocument('this document is about node. it has node examples');
-
-            // Ensure that the tf-idf in the same document has changed to reflect the new idf.
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * ( 1 + Math.log( 4.0 / 4.0 ) ));
-        });
-
-        // Test tf-idf computation on files loaded using readFileSync
-        it("should load documents from files", function(){
-
-            tfidf = new TfIdf();
-
-            tfidf.addFileSync("spec/test_data/tfidf_document1.txt", null, {node: 0, ruby:1});
-            tfidf.addFileSync("spec/test_data/tfidf_document2.txt", null, {node:1, ruby:3});
-            tfidf.addFileSync("spec/test_data/tfidf_document3.txt", null, {node:0, ruby:3});
-            tfidf.addFileSync("spec/test_data/tfidf_document4.txt", null, {node:2, ruby:1});
-
-            var correctCalculations = [
-                1 * (1 + Math.log( 4.0 / 4.0 )),
-                0,
-                2 * (1 + Math.log( 4.0 / 4.0 )),
-                1 * (1 + Math.log( 4.0 / 3.0 ))
-            ];
-
-            tfidf.tfidfs('node', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.node]);
-            });
-
-            tfidf.tfidfs('ruby', function(i, measure, k) {
-                expect(measure).toBe(correctCalculations[k.ruby]);
-            });
-
-            // addFileSync with restoreCache flag set to true.
-            tfidf.addFileSync("spec/test_data/tfidf_document4.txt", null, {node:2, ruby:1}, true);
-        });
-
-        // Test idf caching when adding documents from addFileSync
-        it("should update a terms tf-idf score after adding documents from addFileSync", function(){
-            tfidf = new TfIdf();
-
-            // Add 2 documents
-            tfidf.addFileSync("spec/test_data/tfidf_document1.txt", null, 0);
-            tfidf.addFileSync("spec/test_data/tfidf_document2.txt", null, 1);
-
-            // check the tf-idf for 'node'
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * (1 + Math.log( 2.0 / 2.0 ) ));
-
-            // Add 2 more documents
-            tfidf.addFileSync("spec/test_data/tfidf_document3.txt");
-            tfidf.addFileSync("spec/test_data/tfidf_document4.txt");
-
-            // Ensure that the tf-idf in the same document has changed to reflect the new idf.
-            expect( tfidf.tfidf("node", 0) ).toBe( 1 * (1 + Math.log( 4.0 / 4.0 ) ));
-        });
-
-        // Test idf.setTokenizer
-        it('should allow for specific types of Tokenizers', function(){
-            tfidf = new TfIdf();
-
-            tfidf.addDocument('this document isn\'t about node.', 0);
-            tfidf.addDocument('that doc is about node.', 1);
-            expect( tfidf.tfidf('n\'t', 0) ).toBe(0);
-            expect( tfidf.tfidf('isn', 0) ).toBe( 1 * (1 + Math.log( 2 / 2 ) ));
-
-            tfidf = new TfIdf();
-
-            tfidf.addDocument('this document isn\'t about node.', 0);
-            tfidf.addDocument('this document isn\'t about node.', 1);
-
-            expect( tfidf.tfidf('isn', 0) ).toBe(1 * (1 + Math.log (2/3)));
-
-            tfidf = new TfIdf();
-            var TreebankWordTokenizer = require('../lib/natural/tokenizers/treebank_word_tokenizer');
-            var tokenizer = new TreebankWordTokenizer();
-
-            tfidf.addDocument('this document isn\'t about node.', 0);
-            tfidf.setTokenizer(tokenizer);
-            tfidf.addDocument('this doc isn\'t about node.', 1);
-
-            expect( tfidf.tfidf('isn', 0) ).toBe( 1 * ( 1 + Math.log( 2 / 2 ) ));
-            expect( tfidf.tfidf('n\'t', 1) ).toBe( 1 * ( 1 + Math.log( 2 / 2 ) ));
-            expect( tfidf.tfidf('isn', 1) ).toBe(0);
-        });
-
-        it('should require a valid tokenizer when using setTokenizer', function(){
-            tfidf = new TfIdf();
-
-            expect( function() { tfidf.setTokenizer(1); } ).toThrow(new Error('Expected a valid Tokenizer'));
-            expect( function() { tfidf.setTokenizer({}); } ).toThrow(new Error('Expected a valid Tokenizer'));
-        });
-
-        // Test encoding for addFileSync
-        it('should use the specified encoding for addFileSync', function(){
-
-            tfidf = new TfIdf();
-
-            tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'base64');
-            tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'utf8');
-
-            expect( tfidf.tfidf('dghpcybkb2n1bwvudcbpcybhym91dcbub2rllg', 0) ).toBe( 1 * ( 1 + Math.log( 2.0 / 2.0 ) ));
-        });
-
-        // Test encoding check for addFileSync
-        it('should require a valid encoding for addFileSync', function(){
-
-            tfidf = new TfIdf();
-
-            expect( function() { tfidf.addFileSync('spec/test_data/tfidf_document1.txt', 'foobar'); } ).toThrow(new Error('Invalid encoding: foobar'));
-        });
-    });
-});

package/spec/tokenizer_ja_spec.js

@@ -1,58 +0,0 @@
-/*
-Copyright (c) 2012, Guillaume Marty
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var Tokenizer = require('../lib/natural/tokenizers/tokenizer_ja'),
-    tokenizer = new Tokenizer();
-
-var text = '計算機科学における字句解析 (じくかいせき、英: Lexical Analysis) とは、ソースコードを構成する文字の並びを、トークン (token) の並びに変換することをいう。\n' +
-    'ここでいう「トークン」とは、意味を持つコードの最小単位のこと。字句解析を行うプログラムは、字句解析器 (lexical analyzer, 略称：lexer) と呼ばれる。\n' +
-    '字句解析器はスキャナ (scanner) とトークナイザ (tokenizer) から構成される。\n';
-var result = ['計算', '機科', '学', 'に', 'おける', '字句', '解析',
-  'じくかい', 'せき', '英', 'Lexical', 'Analysis', 'と', 'は', 'ソースコード',
-  'を', '構成', 'する', '文字', 'の', '並び', 'を', 'トークン', 'token', 'の',
-  '並び', 'に', '変換', 'する', 'こと', 'を', 'いう', 'ここ', 'でいう', 'トークン',
-  'と', 'は', '意味', 'を', '持つ', 'コード', 'の', '最小', '単位', 'の', 'こと',
-  '字句', '解析', 'を', '行う', 'プログラム', 'は', '字句', '解析', '器', 'lexical',
-  'analyzer', '略称', 'lexer', 'と', '呼ば', 'れる', '字句', '解析', '器', 'は',
-  'スキャナ', 'scanner', 'と', 'トークナイザ', 'tokenizer', 'から', '構成', 'さ',
-  'れる'];
-
-describe('TokenizerJa', function() {
-  it('should tokenize', function() {
-    var tokens = tokenizer.tokenize(text);
-    expect(tokens).toEqual(result);
-
-    // This test is very hard to pass through, so we comment for now.
-    //tokens = tokenizer.tokenize('すもももももももものうち。');
-    //expect(tokens).toEqual(['すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち', '。']);
-  });
-
-  it('should normalize input', function() {
-    var converters = require('../lib/natural/normalizers/normalizer_ja').converters;
-    var tokens = tokenizer.tokenize(
-        converters.halfwidthToFullwidth.alphabet(
-        converters.halfwidthToFullwidth.numbers(
-        converters.fullwidthToHalfwidth.punctuation(
-        converters.fullwidthToHalfwidth.katakana(text)))));
-    expect(tokens).toEqual(result);
-  });
-});

package/spec/transliterator_ja_spec.js

@@ -1,103 +0,0 @@
-/*
- Copyright (c) 2012, Guillaume Marty
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
- */
-
-var transliterate_ja = require('../lib/natural/transliterators/ja');
-
-describe('transliterate_ja', function() {
-  it('should transliterate hiragana', function() {
-    expect(transliterate_ja('あいうえお かきくけこ')).toEqual('aiueo kakikukeko');
-    expect(transliterate_ja('さしすせそ たちつてと')).toEqual('sashisuseso tachitsuteto');
-  });
-
-  it('should transliterate katakana', function() {
-    expect(transliterate_ja('アイウエオ カキクケコ')).toEqual('aiueo kakikukeko');
-    expect(transliterate_ja('サシスセソ タチツテト')).toEqual('sashisuseso tachitsuteto');
-  });
-
-  it('should transliterate small tsu differently depending on context', function() {
-    expect(transliterate_ja('まっか ざっし たった はっぱ')).toEqual('makka zasshi tatta happa');
-    expect(transliterate_ja('マッカ ザッシ タッタ ハッパ')).toEqual('makka zasshi tatta happa');
-  });
-
-  it('should transliterate final small tsu', function() {
-    expect(transliterate_ja('ああっ')).toEqual('aat');
-    expect(transliterate_ja('アアッ')).toEqual('aat');
-  });
-
-  it('should transliterate n differently depending on context', function() {
-    expect(transliterate_ja('まんと ばんび ほんや')).toEqual("manto bambi hon'ya");
-    expect(transliterate_ja('マント バンビ ホンヤ')).toEqual("manto bambi hon'ya");
-  });
-
-  it('should transliterate long vowels', function() {
-    expect(transliterate_ja('ああ　いい　うう　ええ　おお　おう')).toEqual('aa　ii　ū　ee　oo　ō');
-    expect(transliterate_ja('あー　いー　うー　えー　おー')).toEqual('ā　ī　ū　ē　ō');
-    expect(transliterate_ja('アア　イイ　ウウ　エエ　オオ　オウ')).toEqual('aa　ii　ū　ee　oo　ō');
-    expect(transliterate_ja('アー　イー　ウー　エー　オー')).toEqual('ā　ī　ū　ē　ō');
-  });
-
-  it('should leave non fullwidth kana unchanged', function() {
-    expect(transliterate_ja('abc ABC 漢字 (.)')).toEqual('abc ABC 漢字 (.)');
-    expect(transliterate_ja('ｱｲｳｴｵ ｶｷｸｹｺ')).toEqual('ｱｲｳｴｵ ｶｷｸｹｺ');
-  });
-
-  it('should transliterate misc. words correctly', function() {
-    expect(transliterate_ja('アヴァンギャルド')).toEqual('avangyarudo'); // Avant-garde
-    expect(transliterate_ja('アクサン・スィルコンフレックス')).toEqual('akusan sirukonfurekkusu'); // Accent circonflexe
-    expect(transliterate_ja('アドレシッング')).toEqual('adoreshinngu'); // Addressing
-    expect(transliterate_ja('イェス')).toEqual('yesu'); // Yes
-    expect(transliterate_ja('インテリジェンス')).toEqual('interijensu'); // Intelligence
-    expect(transliterate_ja('インテルメッツォ')).toEqual('interumettso'); // Intermezzo
-    expect(transliterate_ja('グァテマラ')).toEqual('gwatemara'); // Guatemala
-    expect(transliterate_ja('クァルテット')).toEqual('kwarutetto'); // Quartet
-    expect(transliterate_ja('クィンテット')).toEqual('kwintetto'); // Quintet
-    expect(transliterate_ja('クォーター')).toEqual('kwōtā'); // Quarter
-    expect(transliterate_ja('サブウーファー')).toEqual('sabuūfā'); // Sub woofer
-    expect(transliterate_ja('ソフトウェア')).toEqual('sofutowea'); // Software
-    expect(transliterate_ja('ツィーター')).toEqual('tsītā'); // Tweeter
-    expect(transliterate_ja('デューティー')).toEqual('dyūtī'); // Duty
-    expect(transliterate_ja('ドキュメントィンドウ')).toEqual('dokyumentwindō'); // Document window
-    expect(transliterate_ja('ヌーヴェルヴァーグ')).toEqual('nūveruvāgu'); // Nouvelle vague
-    expect(transliterate_ja('ハイジャッンプ')).toEqual('haijanmpu'); // High jump
-    expect(transliterate_ja('バッファ')).toEqual('baffa'); // Buffer
-    expect(transliterate_ja('フーホェア')).toEqual('fūhwea'); // WhoWhere?
-    expect(transliterate_ja('フェイズィング')).toEqual('feizingu'); // Phasing
-    expect(transliterate_ja('フッロピー')).toEqual('furropī'); // Floppy (alternative transcription)
-    expect(transliterate_ja('ブュー')).toEqual('byū'); // View
-    expect(transliterate_ja('フューチャー')).toEqual('fyūchā'); // Future
-    expect(transliterate_ja('フロッピィ')).toEqual('furoppī'); // Floppy
-    expect(transliterate_ja('ボージョレー・ヌーヴォー')).toEqual('bōjorē nūvō'); // Beaujolais nouveau
-    expect(transliterate_ja('ボスニア・ヘルツェゴビナ')).toEqual('bosunia herutsegobina'); // Bosnia and Herzegovina
-    expect(transliterate_ja('マッハ')).toEqual('mahha'); // Mach
-    expect(transliterate_ja('レヴュー')).toEqual('revyū'); // Review
-    expect(transliterate_ja('レクリェーション')).toEqual('rekuryēshon'); // Recreation
-  });
-
-  it('should use fallback for small vowels on special case', function() {
-    // These words use rare combination of small vowels.
-    expect(transliterate_ja('アゲィンスト')).toEqual('ageinsuto'); // Against (alternative transcription)
-    expect(transliterate_ja('エッセィ')).toEqual('essei'); // Essay (alternative transcription)
-    expect(transliterate_ja('ゾゥアラジィ')).toEqual('zouarajī'); // Zoology (alternative transcription)
-    expect(transliterate_ja('ゾゥァラジカル')).toEqual('zouarajikaru'); // Zoological (alternative transcription)
-    expect(transliterate_ja('ボランテァ')).toEqual('borantea'); // Volunteer (alternative transcription)
-  });
-});

package/spec/treebank_word_tokenizer_spec.js

@@ -1,36 +0,0 @@
-/*
-Copyright (c) 2011, Chris Umbel
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-var Tokenizer = require('../lib/natural/tokenizers/treebank_word_tokenizer'),
-  tokenizer = new Tokenizer();
-
-describe('treebank', function() {
-    it('should tokenize', function() {
-        var tokens = tokenizer.tokenize("If we 'all' can't go. I'll stay home.");
-        expect(tokens).toEqual(['If', 'we', "'all", "'", 'ca', "n't", 'go.',
-            'I', "'ll", 'stay', 'home', '.']);
-        
-        tokens = tokenizer.tokenize("If we 'all' can't go. I'll stay home. If we 'all' can't go. I'll stay home.");
-        expect(tokens).toEqual(['If', 'we', "'all", "'", 'ca', "n't", 'go.',
-            'I', "'ll", 'stay', 'home.', 'If', 'we', "'all", "'", 'ca', "n't", 'go.', 'I', "'ll", 'stay', 'home', '.']);        
-    });    
-});