natural 0.1.27 → 0.2.1

package/lib/natural/util/stopwords_fr.js

@@ -0,0 +1,194 @@
+/*
+ Copyright (c) 2014, Ismaël Héry
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+
+// A list of commonly used french words that have little meaning and can be excluded
+// from analysis.
+
+var words = ['être', 'avoir', 'faire',
+    'a',
+    'au',
+    'aux',
+    'avec',
+    'ce',
+    'ces',
+    'dans',
+    'de',
+    'des',
+    'du',
+    'elle',
+    'en',
+    'et',
+    'eux',
+    'il',
+    'je',
+    'la',
+    'le',
+    'leur',
+    'lui',
+    'ma',
+    'mais',
+    'me',
+    'même',
+    'mes',
+    'moi',
+    'mon',
+    'ne',
+    'nos',
+    'notre',
+    'nous',
+    'on',
+    'ou',
+    'où',
+    'par',
+    'pas',
+    'pour',
+    'qu',
+    'que',
+    'qui',
+    'sa',
+    'se',
+    'ses',
+    'son',
+    'sur',
+    'ta',
+    'te',
+    'tes',
+    'toi',
+    'ton',
+    'tu',
+    'un',
+    'une',
+    'vos',
+    'votre',
+    'vous',
+    'c',
+    'd',
+    'j',
+    'l',
+    'à',
+    'm',
+    'n',
+    's',
+    't',
+    'y',
+    'été',
+    'étée',
+    'étées',
+    'étés',
+    'étant',
+    'suis',
+    'es',
+    'est',
+    'sommes',
+    'êtes',
+    'sont',
+    'serai',
+    'seras',
+    'sera',
+    'serons',
+    'serez',
+    'seront',
+    'serais',
+    'serait',
+    'serions',
+    'seriez',
+    'seraient',
+    'étais',
+    'était',
+    'étions',
+    'étiez',
+    'étaient',
+    'fus',
+    'fut',
+    'fûmes',
+    'fûtes',
+    'furent',
+    'sois',
+    'soit',
+    'soyons',
+    'soyez',
+    'soient',
+    'fusse',
+    'fusses',
+    'fût',
+    'fussions',
+    'fussiez',
+    'fussent',
+    'ayant',
+    'eu',
+    'eue',
+    'eues',
+    'eus',
+    'ai',
+    'as',
+    'avons',
+    'avez',
+    'ont',
+    'aurai',
+    'auras',
+    'aura',
+    'aurons',
+    'aurez',
+    'auront',
+    'aurais',
+    'aurait',
+    'aurions',
+    'auriez',
+    'auraient',
+    'avais',
+    'avait',
+    'avions',
+    'aviez',
+    'avaient',
+    'eut',
+    'eûmes',
+    'eûtes',
+    'eurent',
+    'aie',
+    'aies',
+    'ait',
+    'ayons',
+    'ayez',
+    'aient',
+    'eusse',
+    'eusses',
+    'eût',
+    'eussions',
+    'eussiez',
+    'eussent',
+    'ceci',
+    'cela',
+    'cet',
+    'cette',
+    'ici',
+    'ils',
+    'les',
+    'leurs',
+    'quel',
+    'quels',
+    'quelle',
+    'quelles',
+    'sans',
+    'soi'
+];
+
+exports.words = words;

package/lib/natural/util/stopwords_nl.js

@@ -0,0 +1,43 @@
+/*
+Copyright (c) 2011, Chris Umbel, Martijn de Boer, Damien van Holten
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// a list of commonly used words that have little meaning and can be excluded
+// from analysis.
+// This dutch wordlist has been parsed from a list created by Damien van Holten
+// source: http://www.damienvanholten.com/blog/dutch-stop-words/
+var words = [
+    'aan', 'af', 'al', 'alles', 'als', 'altijd', 'andere', 'ben', 'bij', 'daar',
+    'dan', 'dat', 'de', 'der', 'deze', 'die', 'dit', 'doch', 'doen', 'door', 'dus',
+    'een', 'eens', 'en', 'er', 'ge', 'geen', 'geweest', 'haar', 'had', 'heb',
+    'hebben', 'heeft', 'hem', 'het', 'hier', 'hij', 'hoe', 'hun', 'iemand', 'iets',
+    'ik', 'in', 'is', 'ja', 'je ', 'kan', 'kon', 'kunnen', 'maar', 'me', 'meer',
+    'men', 'met', 'mij', 'mijn', 'moet', 'na', 'naar', 'niet', 'niets', 'nog', 'nu',
+    'of', 'om', 'omdat', 'ons', 'ook', 'op', 'over', 'reeds', 'te', 'tegen', 'toch',
+    'toen', 'tot', 'u', 'uit', 'uw', 'van', 'veel', 'voor', 'want', 'waren', 'was',
+    'wat', 'we', 'wel', 'werd', 'wezen', 'wie', 'wij', 'wil', 'worden', 'zal', 'ze',
+    'zei', 'zelf', 'zich', 'zij', 'zijn', 'zo', 'zonder', 'zou',
+    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+    'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '$', '1',
+    '2', '3', '4', '5', '6', '7', '8', '9', '0', '_', '-'];
+
+// tell the world about the noise words.
+exports.words = words;

package/lib/natural/util/stopwords_no.js

@@ -0,0 +1,42 @@
+/*
+Copyright (c) 2014, Kristoffer Brabrand
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// a list of commonly used words that have little meaning and can be excluded
+// from analysis.
+var words = [
+    'og','i','jeg','det','at','en','et','den','til','er','som',
+    'på','de','med','han','av','ikke','der','så','var','meg',
+    'seg','men','ett','har','om','vi','min','mitt','ha','hadde',
+    'hun','nå','over','da','ved','fra','du','ut','sin','dem',
+    'oss','opp','man','kan','hans','hvor','eller','hva','skal',
+    'selv','sjøl','her','alle','vil','bli','ble','blitt','kunne',
+    'inn','når','være','kom','noen','noe','ville','dere','som',
+    'deres','kun','ja','etter','ned','skulle','denne','for','deg',
+    'si','sine','sitt','mot','å','meget','hvorfor','dette','disse',
+    'uten','hvordan','ingen','din','ditt','blir','samme','hvilken',
+    'hvilke','sånn','inni','mellom','vår','hver','hvem','vors',
+    'hvis','både','bare','enn','fordi','før','mange','også','slik',
+    'vært','være','begge','siden','henne','hennar','hennes',
+    '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '_'];
+
+// tell the world about the noise words.
+exports.words = words;

package/lib/natural/util/stopwords_pt.js

@@ -0,0 +1,136 @@
+/*
+Copyright (c) 2011, Luís Rodrigues
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// a list of commonly used words that have little meaning and can be excluded
+// from analysis.
+var words = [
+  'a',
+  'à',
+  'ao',
+  'aos',
+  'aquela',
+  'aquelas',
+  'aquele',
+  'aqueles',
+  'aquilo',
+  'as',
+  'às',
+  'até',
+  'com',
+  'como',
+  'da',
+  'das',
+  'de',
+  'dela',
+  'delas',
+  'dele',
+  'deles',
+  'depois',
+  'do',
+  'dos',
+  'e',
+  'ela',
+  'elas',
+  'ele',
+  'eles',
+  'em',
+  'entre',
+  'essa',
+  'essas',
+  'esse',
+  'esses',
+  'esta',
+  'estas',
+  'este',
+  'estes',
+  'eu',
+  'isso',
+  'isto',
+  'já',
+  'lhe',
+  'lhes',
+  'mais',
+  'mas',
+  'me',
+  'mesmo',
+  'meu',
+  'meus',
+  'minha',
+  'minhas',
+  'muito',
+  'muitos',
+  'na',
+  'não',
+  'nas',
+  'nem',
+  'no',
+  'nos',
+  'nós',
+  'nossa',
+  'nossas',
+  'nosso',
+  'nossos',
+  'num',
+  'nuns',
+  'numa',
+  'numas',
+  'o',
+  'os',
+  'ou',
+  'para',
+  'pela',
+  'pelas',
+  'pelo',
+  'pelos',
+  'por',
+  'quais',
+  'qual',
+  'quando',
+  'que',
+  'quem',
+  'se',
+  'sem',
+  'seu',
+  'seus',
+  'só',
+  'sua',
+  'suas',
+  'também',
+  'te',
+  'teu',
+  'teus',
+  'tu',
+  'tua',
+  'tuas',
+  'um',
+  'uma',
+  'umas',
+  'você',
+  'vocês',
+  'vos',
+  'vosso',
+  'vossos',
+  '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '_'
+];
+
+// tell the world about the noise words.
+exports.words = words;

package/lib/natural/util/topological.js

@@ -0,0 +1,98 @@
+/*
+ Copyright (c) 2014, Lee Wenzhu
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+'use strict';
+
+/**
+ * a topo sort for a digraph
+ * @param {Digraph}
+ */
+var Topological = function(g) {
+    this.isDag = true;
+    this.sorted = topoSort(uniqueVertexs(g.edges()), g.edges());
+};
+
+Topological.prototype.isDAG = function() {
+    return this.isDag;
+};
+
+/**
+ * get ordered vertexs of digraph
+ */
+Topological.prototype.order = function() {
+    return this.sorted.slice();
+};
+
+/**
+ * @param {Array} all vertex in digraph
+ * @param {Object} all edges in the digraph
+ */
+function topoSort(vertexs, edges) {
+    var sorted = [];
+    var cursor = vertexs.length,
+        visited = {},
+        i = cursor;
+    while (i--) {
+        if (!visited[i]) visit(vertexs[i], i, []);
+    }
+
+    return sorted.reverse();
+
+    function visit(vertex, i, predecessors) {
+        if(predecessors.indexOf(vertex) >= 0) {
+            throw new Error('Cyclic dependency:' + JSON.stringify(vertex));
+        }
+
+        if(visited[i]) return;
+        visited[i] = true;
+
+        var outgoing = edges.filter(function(edge) {
+            return edge.to() === vertex;
+        });
+
+        var preds = [];
+        if(outgoing.length > 0) {
+            preds = predecessors.concat(vertex);
+        }
+        var from;
+        outgoing.forEach(function(edge) {
+            from = edge.from();
+            visit(from, vertexs.indexOf(from), preds);
+        });
+
+        sorted[--cursor] = vertex;
+    };
+};
+
+
+function uniqueVertexs(edges) {
+    var vertexs = [];
+    var from, to;
+    edges.forEach(function(edge) {
+        from = edge.from();
+        to = edge.to();
+        if (vertexs.indexOf(from) < 0) vertexs.push(from);
+        if (vertexs.indexOf(to) < 0) vertexs.push(to);
+    });
+    return vertexs;
+};
+
+module.exports = Topological;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "0.1.27",
+  "version": "0.2.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "sylvester": ">= 0.0.12",
-    "apparatus": ">= 0.0.6",
+    "apparatus": ">= 0.0.9",
     "underscore": ">=1.3.1"
   },
   "devDependencies": {

package/spec/aggressive_tokenizer_fr_spec.js

@@ -0,0 +1,101 @@
+/*
+Copyright (c) 2011, Chris Umbel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this softwé and associated documentation files (the "Softwé"), to deal
+in the Softwé without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Softwé, and to permit persons to whom the Softwé is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Softwé.
+
+THE SOFTWé IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWé OR THE USE OR OTHER DEALINGS IN
+THE SOFTWé.
+*/
+
+var Tokenizer = require('../lib/natural/tokenizers/aggressive_tokenizer_fr'),
+    tokenizer = new Tokenizer();
+
+var text = "Affectueusement surnommé « Gabo » dans toute l'Amérique latine, le Colombien Gabriel Garcia Marquez, prix Nobel de littérature 1982, l'un des plus grands écrivains du XXe siècle, est mort À son domicile de Mexico jeudi 17 avril. Il était âgé de 87 ans. Son Œuvre a été traduite dans toutes les langues ou presque, et vendue à quelque 50 millions d'exemplaires.";
+
+var tokenized = [ 'Affectueusement',
+  'surnommé',
+  'Gabo',
+  'dans',
+  'toute',
+  'l',
+  'Amérique',
+  'latine',
+  'le',
+  'Colombien',
+  'Gabriel',
+  'Garcia',
+  'Marquez',
+  'prix',
+  'Nobel',
+  'de',
+  'littérature',
+  '1982',
+  'l',
+  'un',
+  'des',
+  'plus',
+  'grands',
+  'écrivains',
+  'du',
+  'XXe',
+  'siècle',
+  'est',
+  'mort',
+  'À',
+  'son',
+  'domicile',
+  'de',
+  'Mexico',
+  'jeudi',
+  '17',
+  'avril',
+  'Il',
+  'était',
+  'âgé',
+  'de',
+  '87',
+  'ans',
+  'Son',
+  'Œuvre',
+  'a',
+  'été',
+  'traduite',
+  'dans',
+  'toutes',
+  'les',
+  'langues',
+  'ou',
+  'presque',
+  'et',
+  'vendue',
+  'à',
+  'quelque',
+  '50',
+  'millions',
+  'd',
+  'exemplaires' ];
+
+describe('aggressive_tokenizer_fr', function() {
+  it('should tokenize strings', function() {
+    expect(tokenizer.tokenize(text)).toEqual(tokenized);
+  });
+
+  it('should tokenize strings via attached string method', function() {
+    tokenizer.attach();
+    expect(text.tokenize()).toEqual(tokenized);
+  });
+
+});

package/spec/bayes_classifier_spec.js

@@ -157,5 +157,17 @@ describe('bayes classifier', function() {
 		  });
             });
 	});
+
+
+        it('should accept an optional smoothing parameter for the Bayesian estimates', function() {
+            var defaultClassifier = new natural.BayesClassifier();
+            var PorterStemmer = require('../lib/natural/stemmers/porter_stemmer');
+            var newClassifier1 = new natural.BayesClassifier(PorterStemmer);
+            var newClassifier2 = new natural.BayesClassifier(PorterStemmer, 0.1);
+
+            expect(defaultClassifier.classifier.smoothing).toBe(1.0);
+            expect(newClassifier1.classifier.smoothing).toBe(1.0);
+            expect(newClassifier2.classifier.smoothing).toBe(0.1);
+        });
     });
 });