natural 0.1.28 → 0.4.0

package/lib/natural/brill_pos_tagger/lib/TF_Parser.pegjs

@@ -0,0 +1,55 @@
+{
+  var TransformationRule = require("./TransformationRule");
+}
+
+transformation_rules = rules: (S transformation_rule S) +
+{
+  var result = [];
+
+  for (var i = 0; i < rules.length; i++) {
+    result.push(rules[i][1]);
+  }
+  return(result);
+}
+
+transformation_rule = c1: category1 c2: category2 pred: predicate par1: parameter par2: parameter ?
+{
+  // Construct rule
+  var result = new TransformationRule(c1, c2, pred, par1, par2);
+  return(result);
+}
+
+category1 = wild_card / identifier
+
+category2 = identifier
+
+predicate = identifier
+
+parameter = identifier
+
+identifier =
+  characters: [a-zA-Z_0-9_\-\.,()]+ S_no_eol
+  {
+   var s = "";
+   for (var i = 0; i < characters.length; i++) {
+     s += characters[i];
+   }
+   return(s);
+  }
+
+wild_card = wc: "*" S_no_eol
+  {
+   return(wc)
+  }
+
+// Blanks
+EOL =
+  '\r\n' / '\n' / '\r'
+Comment =
+  "\/\/" (!EOL .)* (EOL/EOI)
+S =
+  (' ' / '\t' / EOL / Comment)*
+S_no_eol =
+  (' ' / '\t' / Comment)*
+EOI= 
+  !.

package/lib/natural/brill_pos_tagger/lib/TransformationRule.js

@@ -0,0 +1,49 @@
+/*
+    Transformation rules for the Brill tagger
+    Copyright (C) 2015 Hugo W.L. ter Doest
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+var log4js = require('log4js');
+var logger = log4js.getLogger();
+
+var Predicate = require("./Predicate");
+
+logger.setLevel('ERROR');
+
+var category_wild_card = "*";
+
+function TransformationRule(c1, c2, predicate, parameter1, parameter2) {
+  this.literal = [c1, c2, predicate, parameter1, parameter2];
+  this.predicate = new Predicate(predicate, parameter1, parameter2);
+  this.old_category = c1;
+  this.new_category = c2;
+  logger.debug('TransformationRule constructor: ' + this.literal);
+}
+
+TransformationRule.prototype.apply = function(tagged_sentence, position) {
+  if ((tagged_sentence[position][1] === this.old_category) ||
+      (this.old_category === category_wild_card)) {
+    if (this.predicate.evaluate(tagged_sentence, position)) {
+      tagged_sentence[position][1] = this.new_category;
+      logger.debug('TransformationRule.apply: changed category ' + 
+                   this.old_category + ' to ' + this.new_category +
+                   ' at position ' + position);
+    }
+  }
+};
+
+module.exports = TransformationRule;
+  

package/lib/natural/classifiers/bayes_classifier.js

@@ -20,10 +20,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-var PorterStemmer = require('../stemmers/porter_stemmer'),
-util = require('util'),
-Classifier = require('./classifier'),
-ApparatusBayesClassifier = require('apparatus').BayesClassifier;
+var util = require('util'),
+    Classifier = require('./classifier'),
+    ApparatusBayesClassifier = require('apparatus').BayesClassifier;
 
 var BayesClassifier = function(stemmer, smoothing) {
     var abc = new ApparatusBayesClassifier();
@@ -48,7 +47,9 @@ function load(filename, stemmer, callback) {
         if (err) {
             callback(err);
         }
-        callback(err, restore(classifier, stemmer));
+        else {
+            callback(err, restore(classifier, stemmer));
+        }
     });
 }
 

package/lib/natural/classifiers/classifier.js

@@ -47,8 +47,9 @@ function addDocument(text, classification) {
 	text: text
     });
 
-    for(var i = 0; i < text.length; i++) {
-	this.features[text[i]] = 1;
+    for (var i = 0; i < text.length; i++) {
+        var token = text[i];
+        this.features[token] = (this.features[token] || 0) + 1;
     }
 }
 

package/lib/natural/classifiers/logistic_regression_classifier.js

@@ -20,10 +20,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-var PorterStemmer = require('../stemmers/porter_stemmer'),
-util = require('util'),
-Classifier = require('./classifier'),
-ApparatusLogisticRegressionClassifier = require('apparatus').LogisticRegressionClassifier;
+var util = require('util'),
+    Classifier = require('./classifier'),
+    ApparatusLogisticRegressionClassifier = require('apparatus').LogisticRegressionClassifier;
 
 var LogisticRegressionClassifier = function(stemmer) {
     Classifier.call(this, new ApparatusLogisticRegressionClassifier(), stemmer);
@@ -41,7 +40,12 @@ function restore(classifier, stemmer) {
 
 function load(filename, stemmer, callback) {
     Classifier.load(filename, function(err, classifier) {
-        callback(err, restore(classifier, stemmer));
+        if (err) {
+            callback(err);
+        }
+        else {
+            callback(err, restore(classifier, stemmer));
+        }
     });
 }
 

package/lib/natural/distance/dice_coefficient.js

@@ -48,22 +48,32 @@ var sanitize = function (str) {
 
 // Compare two strings, and spit out a number from 0-1
 var compare = function (str1, str2) {
-  var pairs1 = wordLetterPairs(sanitize(str1));
-  var pairs2 = wordLetterPairs(sanitize(str2));
+  var sanitized_str1 = sanitize(str1);
+  var sanitized_str2 = sanitize(str2);
+  var pairs1 = wordLetterPairs(sanitized_str1);
+  var pairs2 = wordLetterPairs(sanitized_str2);
   var intersection = 0, union = pairs1.length + pairs2.length;
-  var i, j, pair1, pair2;
-  for (i = 0; i < pairs1.length; i++) {
-    pair1 = pairs1[i];
-    for (j = 0; j < pairs2.length; j++) {
-      pair2 = pairs2[j];
-      if (pair1 == pair2) {
-        intersection ++;
-        delete pairs2[j];
-        break;
+  if (union === 0) {
+      if (sanitized_str1 === sanitized_str2) {
+          return 1;
+      } else {
+          return 0;
+      }
+  } else {
+    var i, j, pair1, pair2;
+    for (i = 0; i < pairs1.length; i++) {
+      pair1 = pairs1[i];
+      for (j = 0; j < pairs2.length; j++) {
+        pair2 = pairs2[j];
+        if (pair1 == pair2) {
+          intersection ++;
+          delete pairs2[j];
+          break;
+        }
       }
     }
+    return 2 * intersection / union;
   }
-  return 2 * intersection / union;
 };
 
 module.exports = compare;

package/lib/natural/distance/jaro-winkler_distance.js

@@ -99,13 +99,18 @@ function distance(s1, s2) {
 // s2 is the second string to compare
 // dj is the Jaro Distance (if you've already computed it), leave blank and the method handles it
 function JaroWinklerDistance(s1, s2, dj) {
-    var jaro;
-    (typeof(dj) == 'undefined')? jaro = distance(s1,s2) : jaro = dj;
-    var p = 0.1; //
-    var l = 0 // length of the matching prefix
-    while(s1[l] == s2[l] && l < 4)
-        l++;
-    
-    return jaro + l * p * (1 - jaro);
+		if (s1 == s2) {
+				return 1 
+		}
+		else {
+		    var jaro;
+		    (typeof(dj) == 'undefined')? jaro = distance(s1,s2) : jaro = dj;
+		    var p = 0.1; //
+		    var l = 0 // length of the matching prefix
+		    while(s1[l] == s2[l] && l < 4)
+		        l++;
+		    
+		    return jaro + l * p * (1 - jaro);
+		}
 }
 module.exports = JaroWinklerDistance;

package/lib/natural/index.js

@@ -31,12 +31,14 @@ exports.PorterStemmerRu = require('./stemmers/porter_stemmer_ru');
 exports.PorterStemmerEs = require('./stemmers/porter_stemmer_es');
 exports.PorterStemmerIt = require('./stemmers/porter_stemmer_it');
 exports.PorterStemmerNo = require('./stemmers/porter_stemmer_no');
+exports.PorterStemmerPt = require('./stemmers/porter_stemmer_pt');
 exports.LancasterStemmer = require('./stemmers/lancaster_stemmer');
 exports.StemmerFr = require('./stemmers/stemmer_fr');
 exports.StemmerPl = require('./stemmers/stemmer_pl');
 exports.StemmerJa = require('./stemmers/stemmer_ja');
 exports.AggressiveTokenizerNl = require('./tokenizers/aggressive_tokenizer_nl');
 exports.AggressiveTokenizerFa = require('./tokenizers/aggressive_tokenizer_fa');
+exports.AggressiveTokenizerFr = require('./tokenizers/aggressive_tokenizer_fr');
 exports.AggressiveTokenizerRu = require('./tokenizers/aggressive_tokenizer_ru');
 exports.AggressiveTokenizerEs = require('./tokenizers/aggressive_tokenizer_es');
 exports.AggressiveTokenizerIt = require('./tokenizers/aggressive_tokenizer_it');
@@ -44,11 +46,13 @@ exports.AggressiveTokenizerPl = require('./tokenizers/aggressive_tokenizer_pl');
 exports.AggressiveTokenizerPt = require('./tokenizers/aggressive_tokenizer_pt');
 exports.AggressiveTokenizerNo = require('./tokenizers/aggressive_tokenizer_no');
 exports.AggressiveTokenizer = require('./tokenizers/aggressive_tokenizer');
+exports.CaseTokenizer = require('./tokenizers/tokenizer_case');
 exports.RegexpTokenizer = require('./tokenizers/regexp_tokenizer').RegexpTokenizer;
 exports.WordTokenizer = require('./tokenizers/regexp_tokenizer').WordTokenizer;
 exports.WordPunctTokenizer = require('./tokenizers/regexp_tokenizer').WordPunctTokenizer;
 exports.TreebankWordTokenizer = require('./tokenizers/treebank_word_tokenizer');
 exports.TokenizerJa = require('./tokenizers/tokenizer_ja');
+exports.SentenceTokenizer = require('./tokenizers/sentence_tokenizer');
 exports.BayesClassifier = require('./classifiers/bayes_classifier');
 exports.LogisticRegressionClassifier = require('./classifiers/logistic_regression_classifier');
 exports.NounInflector = require('./inflectors/noun_inflector');
@@ -62,6 +66,7 @@ exports.Trie = require('./trie/trie');
 exports.SentenceAnalyzer = require('./analyzers/sentence_analyzer');
 exports.stopwords = require('./util/stopwords').words;
 exports.ShortestPathTree = require('./util/shortest_path_tree');
+exports.Spellcheck = require('./spellcheck/spellcheck');
 exports.LongestPathTree = require('./util/longest_path_tree');
 exports.EdgeWeightedDigraph = require('./util/edge_weighted_digraph');
 exports.NGrams = require('./ngrams/ngrams');
@@ -73,3 +78,4 @@ exports.normalize = require('./normalizers/normalizer').normalize_tokens;
 exports.normalize_ja = require('./normalizers/normalizer_ja').normalize_ja;
 exports.removeDiacritics = require('./normalizers/remove_diacritics');
 exports.transliterate_ja = require('./transliterators/ja');
+exports.BrillPOSTagger = require('./brill_pos_tagger/lib/Brill_POS_Tagger');

package/lib/natural/inflectors/present_verb_inflector.js

@@ -51,6 +51,7 @@ var VerbInflector = function() {
     this.addIrregular("am", "are");    
     this.addIrregular("is", "are");
     this.addIrregular("was", "were");
+    this.addIrregular("has", "have");
     
     this.singularForms.regularForms.push([/ed$/i, 'ed']);
     this.singularForms.regularForms.push([/ss$/i, 'sses']);

package/lib/natural/ngrams/ngrams.js

@@ -25,6 +25,8 @@ var _ = require("underscore")._,
     tokenizer = new Tokenizer();
 
 exports.setTokenizer = function(t) {
+    if(!_.isFunction(t.tokenize))
+        throw new Error('Expected a valid Tokenizer');
     tokenizer = t;
 }
 
@@ -40,6 +42,10 @@ exports.trigrams = function(sequence, startSymbol, endSymbol) {
     return ngrams(sequence, 3, startSymbol, endSymbol);
 }
 
+exports.multrigrams = function(sequence, n, startSymbol, endSymbol) {
+    return ngrams(sequence, n, startSymbol, endSymbol);
+}
+
 var ngrams = function(sequence, n, startSymbol, endSymbol) {
     var result = [];
     

package/lib/natural/normalizers/normalizer_ja.js

@@ -120,26 +120,21 @@ var conversionTables = {
       '９': '9'
     },
 
-    punctuation: {
+    symbol: {
       '＿': '_',
       '－': '-',
-      '・': '･',
       '，': ',',
-      '、': '､',
       '；': ';',
       '：': ':',
       '！': '!',
       '？': '?',
       '．': '.',
-      '。': '｡',
       '（': '(',
       '）': ')',
       '［': '[',
       '］': ']',
       '｛': '{',
       '｝': '}',
-      '「': '｢',
-      '」': '｣',
       '＠': '@',
       '＊': '*',
       '＼': '\\',
@@ -154,7 +149,7 @@ var conversionTables = {
       '＝': '=',
       '＞': '>',
       '｜': '|',
-      '～': '~',
+      // Never converted: '～': '~',
       '≪': '«',
       '≫': '»',
       '─': '-',
@@ -162,6 +157,16 @@ var conversionTables = {
       '＂': '"'
     },
 
+    purePunctuation: {
+      '、': '､',
+      '。': '｡',
+      '・': '･',
+      '「': '｢',
+      '」': '｣'
+    },
+
+    punctuation: {},
+
     katakana: {
       '゛': 'ﾞ',
       '゜': 'ﾟ',
@@ -492,9 +497,17 @@ var fixCompositeSymbolsTable = {
   '㍗': 'ワット'
 };
 
+// punctuation is pure_punctuation
+conversionTables.fullwidthToHalfwidth.punctuation = merge(
+    conversionTables.fullwidthToHalfwidth.symbol,
+    conversionTables.fullwidthToHalfwidth.purePunctuation
+)
+
 // Fill in the conversion tables with the flipped tables.
 conversionTables.halfwidthToFullwidth.alphabet = flip(conversionTables.fullwidthToHalfwidth.alphabet);
 conversionTables.halfwidthToFullwidth.numbers = flip(conversionTables.fullwidthToHalfwidth.numbers);
+conversionTables.halfwidthToFullwidth.symbol = flip(conversionTables.fullwidthToHalfwidth.symbol);
+conversionTables.halfwidthToFullwidth.purePunctuation = flip(conversionTables.fullwidthToHalfwidth.purePunctuation);
 conversionTables.halfwidthToFullwidth.punctuation = flip(conversionTables.fullwidthToHalfwidth.punctuation);
 conversionTables.halfwidthToFullwidth.katakana = flip(conversionTables.fullwidthToHalfwidth.katakana);
 
@@ -502,7 +515,8 @@ conversionTables.halfwidthToFullwidth.katakana = flip(conversionTables.fullwidth
 conversionTables.normalize = merge(
     conversionTables.fullwidthToHalfwidth.alphabet,
     conversionTables.fullwidthToHalfwidth.numbers,
-    conversionTables.halfwidthToFullwidth.punctuation,
+    conversionTables.fullwidthToHalfwidth.symbol,
+    conversionTables.halfwidthToFullwidth.purePunctuation,
     conversionTables.halfwidthToFullwidth.katakana
     );
 
@@ -510,6 +524,8 @@ var converters = {
   fullwidthToHalfwidth: {
     alphabet: replacer(conversionTables.fullwidthToHalfwidth.alphabet),
     numbers: replacer(conversionTables.fullwidthToHalfwidth.numbers),
+    symbol: replacer(conversionTables.fullwidthToHalfwidth.symbol),
+    purePunctuation: replacer(conversionTables.fullwidthToHalfwidth.purePunctuation),
     punctuation: replacer(conversionTables.fullwidthToHalfwidth.punctuation),
     katakana: replacer(conversionTables.fullwidthToHalfwidth.katakana)
   },
@@ -517,6 +533,8 @@ var converters = {
   halfwidthToFullwidth: {
     alphabet: replacer(conversionTables.halfwidthToFullwidth.alphabet),
     numbers: replacer(conversionTables.halfwidthToFullwidth.numbers),
+    symbol: replacer(conversionTables.halfwidthToFullwidth.symbol),
+    purePunctuation: replacer(conversionTables.halfwidthToFullwidth.purePunctuation),
     punctuation: replacer(conversionTables.halfwidthToFullwidth.punctuation),
     katakana: replacer(conversionTables.halfwidthToFullwidth.katakana)
   },

package/lib/natural/phonetics/double_metaphone.js

@@ -66,7 +66,8 @@ function process(token, maxLength) {
     }
 
     function handleC() {
-        if(pos > 1 && !isVowel(token[pos - 2]) 
+
+        if(pos >= 1 && !isVowel(token[pos - 2]) 
                 && token[pos - 1] == 'A' && token[pos + 1] == 'H'
                     && (token[pos + 2] != 'I' && token[pos + 2] != 'I')
                         || subMatch(-2, 4, ['BACHER', 'MACHER'])) {
@@ -74,7 +75,9 @@ function process(token, maxLength) {
             pos++;
         } else if(pos == 0 && token.substring(1, 6) == 'EASAR') {
             add('S');
-            pos++;
+            add('S');
+            add('R');
+            pos += 6;
         } else if(token.substring(pos + 1, pos + 4) == 'HIA') {
             add('K');
             pos++;
@@ -84,7 +87,7 @@ function process(token, maxLength) {
                 pos++;
             } else if(pos == 0 
                         && (subMatch(1, 6, ['HARAC', 'HARIS']) 
-                            || subMatch(1, 3, ['HOR', 'HUM', 'HIA', 'HEM']))
+                            || subMatch(1, 4, ['HOR', 'HUM', 'HIA', 'HEM']))
                         && token.substring(pos + 1, pos + 5) != 'HORE') {
                 add('K');
                 pos++;
@@ -96,6 +99,7 @@ function process(token, maxLength) {
                         && subMatch(2, 3, ['B', 'F', 'H', 'L', 'M', 'N', 'R', 'V', 'W']))) {
                     add('K');
                 } else if(pos > 0) {
+
                     if(token.substring(0, 2) == 'MC') {
                         add('K');
                     } else {
@@ -415,6 +419,7 @@ function process(token, maxLength) {
     }
 
     while(pos < token.length) {
+
     	switch(token[pos]) {
 	        case 'A': case 'E': case 'I': case 'O': case 'U': case 'Y': 	        
 	        case 'Ê': case 'É': case 'É': case'À':
@@ -488,7 +493,7 @@ function process(token, maxLength) {
         }
 
     	pos++;
-    }    
+    }
 
     return [truncate(primary, maxLength), truncate(secondary, maxLength)];
 }

package/lib/natural/phonetics/metaphone.js

@@ -38,7 +38,11 @@ function dropBafterMAtEnd(token) {
 }
 
 function cTransform(token) {
+    
+
     token = token.replace(/([^s]|^)(c)(h)/g, '$1x$3').trim();
+
+
     token = token.replace(/cia/g, 'xia');
     token = token.replace(/c(i|e|y)/g, 's$1');
     token = token.replace(/c/g, 'k'); 

package/lib/natural/phonetics/soundex.js

@@ -47,7 +47,7 @@ function transformR(token) {
 }
 
 function condense(token) {
-    return token.replace(/(\d)[hw]?\1+/g, '$1').replace(/[hw]/g, '');
+    return token.replace(/(\d)?\1+/g, '$1');
 }
 
 function padRight0(token) {
@@ -57,16 +57,20 @@ function padRight0(token) {
         return token;
 }
 
+function transform(token) {
+    return transformLipps(transformThroats(
+        transformToungue(transformL(transformHum(transformR(token))))));
+}
+
 var SoundEx = new Phonetic();
 module.exports = SoundEx;
 
 SoundEx.process = function(token, maxLength) {
-    token = token.toLowerCase();
-    
-    return token.charAt(0).toUpperCase() + padRight0(condense(transformLipps(transformThroats(
-        transformToungue(transformL(transformHum(transformR(
-            token.substr(1, token.length - 1).replace(/[aeiouy]/g, '')))))))
-                )).substr(0, (maxLength && maxLength - 1) || 3);
+    token = token.toLowerCase();    
+    var transformed = condense(transform(token.substr(1, token.length - 1))); // anything that isn't a digit goes
+    // deal with duplicate INITIAL consonant SOUNDS
+    transformed = transformed.replace(new RegExp("^" + transform(token.charAt(0))), '');
+    return token.charAt(0).toUpperCase() + padRight0(transformed.replace(/\D/g, '')).substr(0, (maxLength && maxLength - 1) || 3);
 };
 
 // export for tests;

package/lib/natural/spellcheck/spellcheck.js

@@ -0,0 +1,80 @@
+
+var Trie = require('../trie/trie');
+
+// Probabilistic spellchecker based on http://norvig.com/spell-correct.html
+// The general idea is simple. Given a word, the spellchecker calculates all strings that are some user-defined edit distance away. Of those many candidates, it filters the ones that are not words and then returns an array of possible corrections in order of decreasing probability, based on the edit distance and the candidate's frequency in the input corpus
+// Words that are an edit distance of n away from the mispelled word are considered infinitely more probable than words that are of an edit distance >n
+
+// wordlist is a corpus (an array) from which word probabilities are calculated (so something like /usr/share/dict/words (on OSX) will work okay, but real world text will work better)
+function Spellcheck(wordlist) {
+    this.trie = new Trie();
+    this.trie.addStrings(wordlist);
+    this.word2frequency = {};
+    for(var i in wordlist) {
+        if(!this.word2frequency[wordlist[i]]) {
+            this.word2frequency[wordlist[i]] = 0;
+        }
+        this.word2frequency[wordlist[i]]++;
+    }
+}
+
+Spellcheck.prototype.isCorrect = function(word) {
+    return this.trie.contains(word);
+}
+
+// Returns a list of suggested corrections, from highest to lowest probability 
+// maxDistance is the maximum edit distance 
+// According to Norvig, literature suggests that 80% to 95% of spelling errors are an edit distance of 1 away from the correct word. This is good, because there are roughly 54n+25 strings 1 edit distance away from any given string of length n. So after maxDistance = 2, this becomes very slow.
+Spellcheck.prototype.getCorrections = function(word, maxDistance) {
+    var self = this;
+    if(!maxDistance) maxDistance = 1;
+    var edits = this.editsWithMaxDistance(word, maxDistance);
+    edits = edits.slice(1,edits.length);
+    edits = edits.map(function(editList) {
+       return editList.filter(function(word) { return self.isCorrect(word); })
+                      .map(function(word) { return [word, self.word2frequency[word]]; })
+                      .sort(function(a,b) { return a[1] > b[1] ? -1 : 1; })
+                      .map(function(wordscore) { return wordscore[0]; });
+    });
+    var flattened = [];
+    for(var i in edits) {
+        if(edits[i].length) flattened = flattened.concat(edits[i]);
+    }
+    return flattened.filter(function (v, i, a) { return a.indexOf(v) == i });
+}
+
+// Returns all edits that are 1 edit-distance away from the input word
+Spellcheck.prototype.edits = function(word) {
+    var alphabet = 'abcdefghijklmnopqrstuvwxyz';
+    var edits = [];
+    for(var i=0; i<word.length+1; i++) {
+        if(i>0) edits.push(word.slice(0,i-1)+word.slice(i,word.length)); // deletes
+        if(i>0 && i<word.length+1) edits.push(word.slice(0,i-1) + word.slice(i,i+1) + word.slice(i-1, i) + word.slice(i+1,word.length)); // transposes
+        for(var k=0; k<alphabet.length; k++) {
+            if(i>0) edits.push(word.slice(0,i-1)+alphabet[k]+word.slice(i,word.length)); // replaces
+            edits.push(word.slice(0,i)+alphabet[k]+word.slice(i,word.length)); // inserts
+        }
+    }
+    // Deduplicate edits
+    edits = edits.filter(function (v, i, a) { return a.indexOf(v) == i });
+    return edits;
+}
+
+// Returns all edits that are up to "distance" edit distance away from the input word
+Spellcheck.prototype.editsWithMaxDistance = function(word, distance) { 
+    return this.editsWithMaxDistanceHelper(distance, [[word]]);
+}
+
+Spellcheck.prototype.editsWithMaxDistanceHelper = function(distanceCounter, distance2edits) {
+    if(distanceCounter == 0) return distance2edits;
+    var currentDepth = distance2edits.length-1;
+    var words = distance2edits[currentDepth];
+    var edits = this.edits(words[0]);
+    distance2edits[currentDepth+1] = [];
+    for(var i in words) {
+        distance2edits[currentDepth+1] = distance2edits[currentDepth+1].concat(this.edits(words[i]));
+    }
+    return this.editsWithMaxDistanceHelper(distanceCounter-1, distance2edits);
+}
+
+module.exports = Spellcheck;