natural 0.1.28 → 0.4.0

package/.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

package/.npmignore

@@ -1,11 +1,4 @@
-*~
-\#*
-*.kpf
-*classifier.json
-node_modules
-io_spec/test_data/wordnet/download/*
-*.noun
-*.verb
-*.adj
-*.adv
-npm-debug.log
+benchmarks
+spec
+examples
+io_spec

package/README.md

@@ -1,6 +1,7 @@
 natural
 =======
 
+[![NPM version](https://img.shields.io/npm/v/natural.svg)](https://www.npmjs.com/package/natural)
 [![Build Status](https://travis-ci.org/NaturalNode/natural.png?branch=master)](https://travis-ci.org/NaturalNode/natural)
 
 "Natural" is a general natural language facility for nodejs. Tokenizing,
@@ -18,8 +19,30 @@ will be in order. Thanks to Polyakov Vladimir, Russian stemming has been added!,
 
 Aside from this README, the only documentation is [this DZone article](http://www.dzone.com/links/r/using_natural_a_nlp_module_for_nodejs.html) and [here on my blog](http://www.chrisumbel.com/article/node_js_natural_language_porter_stemmer_lancaster_bayes_naive_metaphone_soundex), which is a bit older.
 
-Installation
-------------
+### TABLE OF CONTENTS
+
+* [Installation](#installation)
+* [Tokenizers](#tokenizers)
+* [String Distance](#string-distance)
+* [Stemmers](#stemmers)
+* [Classifiers](#classifiers)
+* [Phonetics](#phonetics)
+* [Inflectors](#inflectors)
+* [N-Grams](#n-grams)
+* [tf-idf](#tf-idf)
+* [Tries](#tries)
+* [EdgeWeightedDigraph](#edgeweighteddigraph)
+* [ShortestPathTree](#shortestpathtree)
+* [LongestPathTree](#longestpathtree)
+* [WordNet](#wordnet)
+* [Spellcheck](#spellcheck)
+* [POS Tagger](#pos-tagger)
+* [Acknowledgements/references](#acknowledgements-and-references)
+* [Development](#development)
+* [License](#license)
+
+
+## Installation
 
 If you're just looking to use natural without your own node application,
 you can install via NPM like so:
@@ -29,8 +52,7 @@ you can install via NPM like so:
 If you're interested in contributing to natural, or just hacking on it, then by all
 means fork away!
 
-Tokenizers
-----------
+## Tokenizers
 
 Word, Regexp, and [Treebank tokenizers](http://www.cis.upenn.edu/~treebank/tokenization.html) are provided for breaking text up into
 arrays of tokens:
@@ -58,8 +80,8 @@ console.log(tokenizer.tokenize("my dog hasn't any fleas."));
 // [ 'my',  'dog',  'hasn',  '\'',  't',  'any',  'fleas',  '.' ]
 ```
 
-String Distance
-----------------------
+## String Distance
+
 Natural provides an implementation of the [Jaro–Winkler](http://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance) string distance measuring algorithm.
 This will return a number between 0 and 1 which tells how closely the strings match (0 = not at all, 1 = exact match):
 
@@ -122,8 +144,7 @@ Output:
 0
 ```
 
-Stemmers
---------
+## Stemmers
 
 Currently stemming is supported via the [Porter](http://tartarus.org/martin/PorterStemmer/index.html) and [Lancaster](http://www.comp.lancs.ac.uk/computing/research/stemming/) (Paice/Husk) algorithms.
 
@@ -167,8 +188,7 @@ console.log("i am waking up to the sounds of chainsaws".tokenizeAndStem());
 console.log("chainsaws".stem());
 ```
 
-Classifiers
-----------------------
+## Classifiers
 
 Two classifiers are currently supported, [Naive Bayes](http://en.wikipedia.org/wiki/Naive_Bayes_classifier) and [logistic regression](http://en.wikipedia.org/wiki/Logistic_regression).
 The following examples use the BayesClassifier class, but the 
@@ -184,7 +204,7 @@ tokenize and stem the text.
 
 ```javascript
 classifier.addDocument('i am long qqqq', 'buy');
-classifier.addDocument('buy the q''s', 'buy');
+classifier.addDocument('buy the q\'s', 'buy');
 classifier.addDocument('short gold', 'sell');
 classifier.addDocument('sell gold', 'sell');
 
@@ -208,8 +228,8 @@ You have access to the set of matched classes and the associated value from the
 Outputs:
 
 ```javascript
-[ { label: 'sell', value: 0.39999999999999997 },
-  { label: 'buy', value: 0.19999999999999998 } ]
+[ { label: 'buy', value: 0.39999999999999997 },
+  { label: 'sell', value: 0.19999999999999998 } ]
 ```
 
 From this:
@@ -234,7 +254,8 @@ The training process can be monitored by subscribing to the event `trainedWithDo
        *   total: 23 // There are 23 total documents being trained against
        *   index: 12 // The index/number of the document that's just been trained against
        *   doc: {...} // The document that has just been indexed
-       */ }
+       *  }
+       */ 
     });
 
 A classifier can also be persisted and recalled so you can reuse a training
@@ -268,8 +289,7 @@ var restoredClassifier = natural.BayesClassifier.restore(JSON.parse(raw));
 console.log(restoredClassifier.classify('i should sell that'));
 ```
 
-Phonetics
----------
+## Phonetics
 
 Phonetic matching (sounds-like) matching can be done with the [SoundEx](http://en.wikipedia.org/wiki/Soundex),
 [Metaphone](http://en.wikipedia.org/wiki/Metaphone) or [DoubleMetaphone](http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone) algorithms
@@ -356,8 +376,7 @@ if(wordA.soundsLike(wordB))
 console.log('phonetics'.phonetics());
 ```
 
-Inflectors
-----------
+## Inflectors
 
 ### Nouns
 
@@ -442,8 +461,7 @@ console.log('walks'.pluralizePresentVerb());
 ```
 
 
-N-Grams
--------
+## N-Grams
 
 n-grams can be obtained for either arrays or strings (which will be tokenized
 for you):
@@ -534,8 +552,7 @@ console.log(NGramsZH.ngrams(['一', '个', '中', '文', '测',
     '试'], 4));
 ```
 
-tf-idf
------
+## tf-idf
 
 [Term Frequency–Inverse Document Frequency (tf-idf)](http://en.wikipedia.org/wiki/Tf%E2%80%93idf) is implemented to determine how important a word (or words) is to a 
 document relative to a corpus. The following example will add four documents to 
@@ -597,7 +614,7 @@ tfidf.addFileSync('data_files/two.txt');
 
 Multiple terms can be measured as well, with their weights being added into
 a single measure value. The following example determines that the last document
-is the most relevent to the words "node" and "ruby".
+is the most relevant to the words "node" and "ruby".
 
 ```javascript
 var natural = require('natural'),
@@ -650,7 +667,7 @@ tfidf.listTerms(0 /*document index*/).forEach(function(item) {
 });
 ```
 
-A TfIdf instance can also be serialized and deserialzed for save and recall.
+A TfIdf instance can also be serialized and deserialized for save and recall.
 
 ```javascript
 var tfidf = new TfIdf();
@@ -663,12 +680,11 @@ var s = JSON.stringify(tfidf);
 var tfidf = new TfIdf(JSON.parse(s));
 ```
 
-Tries
------
+## Tries
 
 Tries are a very efficient data structure used for prefix-based searches. 
-Natural comes packaged with a basic Trie implementation wich can support match collection along a path,
-existance search and prefix search.
+Natural comes packaged with a basic Trie implementation which can support match collection along a path,
+existence search and prefix search.
 
 ### Building The Trie
 
@@ -741,9 +757,9 @@ ciTrie.contains("TEsT"); // true
 ```
 In the case of the searches which return strings, all strings returned will be in lower case if you are in case-_in_sensitive mode.
 
-EdgeWeightedDigraph
--------------------
-EdgeWeightedDigraph represents a digraph, you can add an edge, get the number vertexs, edges, get all edges and use toString to print the Digraph.
+## EdgeWeightedDigraph
+
+EdgeWeightedDigraph represents a digraph, you can add an edge, get the number vertexes, edges, get all edges and use toString to print the Digraph.
 
 initialize a digraph:
 
@@ -759,7 +775,7 @@ digraph.add(6,4,0.93);
 ```
 the api used is: add(from, to, weight).
 
-get the number of vertexs:
+get the number of vertexes:
 
 ```javascript
 console.log(digraph.v());
@@ -775,8 +791,7 @@ you will get 5.
 
 
 
-ShortestPathTree
-----------------
+## ShortestPathTree
 
 ShortestPathTree represents a data type for solving the single-source shortest paths problem in
 edge-weighted directed acyclic graphs (DAGs).
@@ -815,7 +830,7 @@ false
 ```
 
 ### pathTo(vertex)
-this will retuan a shortest path:
+this will return a shortest path:
 
 ```javascript
 console.log(spt.pathTo(4));
@@ -827,8 +842,7 @@ output will be:
 [5, 4]
 ```
 
-LongestPathTree
-----------------
+## LongestPathTree
 
 LongestPathTree represents a data type for solving the single-source shortest paths problem in
 edge-weighted directed acyclic graphs (DAGs).
@@ -867,7 +881,7 @@ false
 ```
 
 ### pathTo(vertex)
-this will retuan a shortest path:
+this will return a shortest path:
 
 ```javascript
 console.log(spt.pathTo(4));
@@ -879,21 +893,18 @@ output will be:
 [5, 1, 3, 6, 4]
 ```
 
-WordNet
--------
+## WordNet
 
 One of the newest and most experimental features in natural is WordNet integration. Here's an
 example of using natural to look up definitions of the word node. To use the WordNet module,
-first install the WordNet database files using the [WNdb module](https://github.com/moos/WNdb):
+first install the WordNet database files using [wordnet-db](https://github.com/moos/wordnet-db):
 
-    npm install WNdb
-
-(For node < v0.6, please use 'npm install WNdb@3.0.0')
+    npm install wordnet-db
 
 Keep in mind that the WordNet integration is to be considered experimental at this point,
-and not production-ready. The API is also subject to change.
+and not production-ready. The API is also subject to change.  For an implementation with vastly increased performance, as well as a command-line interface, see [wordpos](https://github.com/moos/wordpos). 
 
-Here's an example of looking up definitions for the word, "node".
+Here's an example of looking up definitions for the word "node".
 
 ```javascript
 var wordnet = new natural.WordNet();
@@ -935,8 +946,138 @@ As of v0.1.11, WordNet data files are no longer automatically downloaded.
 
 Princeton University "About WordNet." WordNet. Princeton University. 2010. <http://wordnet.princeton.edu>
 
-Development
------------
+## Spellcheck
+
+A probabilistic spellchecker based on http://norvig.com/spell-correct.html
+
+This is best constructed with an array of tokens from a corpus, but a simple list of words from a dictionary will work. 
+
+```javascript
+var corpus = ['something', 'soothing'];
+var spellcheck = new Spellcheck(corpus);
+```
+
+It uses the trie datastructure for fast boolean lookup of a word
+
+```javascript
+spellcheck.isCorrect('cat'); // false
+```
+
+It suggests corrections (sorted by probability in descending order) that are up to a maximum edit distance away from the input word. According to Norvig, a max distance of 1 will cover 80% to 95% of spelling mistakes. After a distance of 2, it becomes very slow.
+
+```javascript
+spellcheck.getCorrections('soemthing', 1); // ['something']
+spellcheck.getCorrections('soemthing', 2); // ['something', 'soothing']
+```
+
+## POS Tagger
+
+This is a part-of-speech tagger based on Eric Brill's transformational 
+algorithm. Transformation rules are specified in external files.
+
+### Usage
+```javascript
+var Tagger = require("./lib/natural").BrillPOSTagger;
+
+var base_folder = "some_path/lib/natural/brill_pos_tagger";
+var rules_file = base_folder + "/data/tr_from_posjs.txt";
+var lexicon_file = base_folder + "/data/lexicon_from_posjs.json";
+var default_category = 'N';
+
+var tagger = new Tagger(lexicon_file, rules_file, default_category, function(error) {
+  if (error) {
+    console.log(error);
+  }
+  else {
+    var sentence = ["I", "see", "the", "man", "with", "the", "telescope"];
+    console.log(JSON.stringify(tagger.tag(sentence)));
+  }
+});
+```
+
+### Lexicon
+The lexicon is either a JSON file that has the following structure:
+```javascript
+{
+  "word1": ["cat1"],
+  "word2": ["cat2", "cat3"],
+  ...
+}
+```
+or a text file:
+```
+word1 cat1 cat2
+word2 cat3
+...
+```
+Words may have multiple categories in the lexicon file. The tagger uses only 
+the first category specified.
+
+### Specifying transformation rules
+Transformation rules are specified as follows:
+```
+OLD_CAT NEW_CAT PREDICATE PARAMETER
+```
+This means that if the category of the current position is OLD_CAT and the predicate is true, the category is replaced by NEW_CAT. The predicate 
+may use the parameter in different ways: sometimes the parameter is used for 
+specifying the outcome of the predicate:
+```
+NN CD CURRENT-WORD-IS-NUMBER YES
+```
+This means that if the outcome of predicate CURRENT-WORD-IS-NUMBER is YES, the 
+category is replaced by <code>CD</code>.
+The parameter can also be used to check the category of a word in the sentence:
+```
+VBD NN PREV-TAG DT
+```
+Here the category of the previous word must be <code>DT</code> for the rule to be applied.
+
+### Algorithm
+The tagger applies transformation rules that may change the category of words. The input sentence must be split into words which are assigned with categories. The tagged sentence is then processed from left to right. At each step all rules are applied once; rules are applied in the order in which they are specified. Algorithm:
+```javascript
+function(sentence) {
+  var tagged_sentence = new Array(sentence.length);
+
+  // snip
+
+  // Apply transformation rules
+  for (var i = 0, size = sentence.length; i < size; i++) {
+    this.transformation_rules.forEach(function(rule) {
+      rule.apply(tagged_sentence, i);
+    });
+  }
+  return(tagged_sentence);
+}
+```
+
+### Adding a predicate
+Predicates are defined in module <code>lib/Predicate.js</code>. In that file 
+a function must be created that serves as predicate. A predicate accepts a 
+tagged sentence, the current position in the sentence that should be tagged, and
+ the
+ outcome(s) of the predicate. An example of a predicate that checks the category of the current word:
+```javascript
+function current_word_is_tag(tagged_sentence, i, parameter) {
+  return(tagged_sentence[i][0] === parameter);
+}
+```
+Some predicates accept two parameters. Next step is to map a keyword to this predicate so that it can be used in the transformation rules. The mapping is also defined in <code>lib/Predicate.js</code>:
+```javascript
+var predicates = {
+  "CURRENT-WORD-IS-TAG": current_word_is_tag,
+  "PREV-WORD-IS-CAP": prev_word_is_cap
+}
+```
+
+### Acknowledgements and References
+* Part of speech tagger by Percy Wegmann, https://code.google.com/p/jspos/
+* Node.js version of jspos: https://github.com/neopunisher/pos-js
+* A simple rule-based part of speech tagger, Eric Brill, Published in: Proceeding ANLC '92 Proceedings of the third conference on Applied natural language processing, Pages 152-155. http://dl.acm.org/citation.cfm?id=974526
+
+
+
+## Development
+
 When developing, please:
 
 + Write unit tests
@@ -947,8 +1088,7 @@ The current configuration of the unit tests requires the following environment v
     export NODE_PATH=.
 
 
-License
--------
+## License
 
 Copyright (c) 2011, 2012 Chris Umbel, Rob Ellis, Russell Mull
 

package/bayes_classifier.json

@@ -0,0 +1 @@
+{"classifier":{"classFeatures":{"computing":{"0":2,"1":2,"2":2,"3":3,"4":2,"5":2},"literature":{"2":2,"6":4,"7":2,"8":2}},"classTotals":{"computing":4,"literature":4},"totalExamples":7,"smoothing":1},"docs":[{"label":"computing","text":["fix","box"]},{"label":"computing","text":["write","code"]},{"label":"computing","text":["nasti","script","code"]},{"label":"literature","text":["write","book"]},{"label":"literature","text":["read","book"]},{"label":"literature","text":["studi","book"]}],"features":{"fix":1,"box":1,"write":2,"code":2,"nasti":1,"script":1,"book":3,"read":1,"studi":1},"stemmer":{},"lastAdded":6,"events":{"domain":null,"_events":{},"_maxListeners":10}}

package/lib/natural/analyzers/sentence_analyzer.js

@@ -135,13 +135,16 @@ Sentences.prototype.toString = function() {
 // This is quick and incomplete.
 Sentences.prototype.type = function(callback) {
     var callback = callback || false;
-	
+
+    // Check for implicit you before popping a tag.
+    var implicitYou = this.implicitYou();
+
     // FIXME - punct seems useless
     var lastElement = this.posObj.punct();
     lastElement = (lastElement.length != 0) ? lastElement.pop() : this.posObj.tags.pop();
-	
+
     if (lastElement.pos !== ".") {
-        if (this.implicitYou()) {
+        if (implicitYou) {
             this.senType = "COMMAND";
         } else if (_(["WDT","WP","WP$","WRB"]).contains(this.posObj.tags[0].pos)) {
             // Sentences that start with: who, what where when why and how, then they are questions
@@ -158,8 +161,8 @@ Sentences.prototype.type = function(callback) {
     } else {
         switch(lastElement.token) {
             case "?": this.senType = "INTERROGATIVE"; break;
-            case "!": this.senType = (this.implicitYou()) ? "COMMAND":"EXCLAMATORY"; break;
-            case ".": this.senType = (this.implicitYou()) ? "COMMAND":"DECLARATIVE";	break;
+            case "!": this.senType = (implicitYou) ? "COMMAND":"EXCLAMATORY"; break;
+            case ".": this.senType = (implicitYou) ? "COMMAND":"DECLARATIVE";	break;
         }
     }
     

package/lib/natural/brill_pos_tagger/data/Dutch/README.txt

@@ -0,0 +1,2 @@
+Transformation rules and lexicon are from
+https://mlnl.net/jg/software/bnl/