natural 6.10.2 → 6.10.3

package/lib/natural/tfidf/tfidf.js

@@ -49,20 +49,10 @@ function buildDocument (text, key) {
   }, { __key: key })
 }
 
-function tf (term, document) {
-  return document[term] ? document[term] : 0
-}
-
 function documentHasTerm (term, document) {
   return document[term] && document[term] > 0
 }
 
-function TfIdf (deserialized) {
-  if (deserialized) { this.documents = deserialized.documents } else { this.documents = [] }
-
-  this._idfCache = {}
-}
-
 // backwards compatibility for < node 0.10
 function isEncoding (encoding) {
   if (typeof Buffer.isEncoding !== 'undefined') { return Buffer.isEncoding(encoding) }
@@ -83,135 +73,149 @@ function isEncoding (encoding) {
   return false
 }
 
-module.exports = TfIdf
-TfIdf.tf = tf
-
-TfIdf.prototype.idf = function (term, force) {
-  // Lookup the term in the New term-IDF caching,
-  // this will cut search times down exponentially on large document sets.
-  // if (this._idfCache[term] && this._idfCache.hasOwnProperty(term) && force !== true) { return this._idfCache[term] }
-  if (this._idfCache[term] && force !== true) {
-    return this._idfCache[term]
+class TfIdf {
+  constructor (deserialized) {
+    if (deserialized) {
+      this.documents = deserialized.documents
+    } else {
+      this.documents = []
+    }
+    this._idfCache = {}
+  }
+
+  static tf (term, document) {
+    return document[term] ? document[term] : 0
   }
 
-  const docsWithTerm = this.documents.reduce(function (count, document) {
-    return count + (documentHasTerm(term, document) ? 1 : 0)
-  }, 0)
+  idf (term, force) {
+    // Lookup the term in the New term-IDF caching,
+    // this will cut search times down exponentially on large document sets.
+    // if (this._idfCache[term] && this._idfCache.hasOwnProperty(term) && force !== true) { return this._idfCache[term] }
+    if (this._idfCache[term] && force !== true) {
+      return this._idfCache[term]
+    }
 
-  const idf = 1 + Math.log((this.documents.length) / (1 + docsWithTerm))
+    const docsWithTerm = this.documents.reduce(function (count, document) {
+      return count + (documentHasTerm(term, document) ? 1 : 0)
+    }, 0)
 
-  // Add the idf to the term cache and return it
-  this._idfCache[term] = idf
-  return idf
-}
+    const idf = 1 + Math.log((this.documents.length) / (1 + docsWithTerm))
+
+    // Add the idf to the term cache and return it
+    this._idfCache[term] = idf
+    return idf
+  }
 
-// If restoreCache is set to true, all terms idf scores currently cached will be recomputed.
-// Otherwise, the cache will just be wiped clean
-TfIdf.prototype.addDocument = function (document, key, restoreCache) {
-  this.documents.push(buildDocument(document, key))
-
-  // make sure the cache is invalidated when new documents arrive
-  if (restoreCache === true) {
-    for (const term in this._idfCache) {
-      // invoking idf with the force option set will
-      // force a recomputation of the idf, and it will
-      // automatically refresh the cache value.
-      this.idf(term, true)
+  // If restoreCache is set to true, all terms idf scores currently cached will be recomputed.
+  // Otherwise, the cache will just be wiped clean
+  addDocument (document, key, restoreCache) {
+    this.documents.push(buildDocument(document, key))
+
+    // make sure the cache is invalidated when new documents arrive
+    if (restoreCache === true) {
+      for (const term in this._idfCache) {
+        // invoking idf with the force option set will
+        // force a recomputation of the idf, and it will
+        // automatically refresh the cache value.
+        this.idf(term, true)
+      }
+    } else {
+      // this._idfCache = {}
+      // so that we do not have trouble with terms that match property names
+      this._idfCache = Object.create(null)
     }
-  } else {
-    // this._idfCache = {}
-    // so that we do not have trouble with terms that match property names
-    this._idfCache = Object.create(null)
   }
-}
 
-// If restoreCache is set to true, all terms idf scores currently cached will be recomputed.
-// Otherwise, the cache will just be wiped clean
-TfIdf.prototype.addFileSync = function (path, encoding, key, restoreCache) {
-  if (!encoding) { encoding = 'utf8' }
-  if (!isEncoding(encoding)) { throw new Error('Invalid encoding: ' + encoding) }
-
-  const document = fs.readFileSync(path, encoding)
-  this.documents.push(buildDocument(document, key))
-
-  // make sure the cache is invalidated when new documents arrive
-  if (restoreCache === true) {
-    for (const term in this._idfCache) {
-      // invoking idf with the force option set will
-      // force a recomputation of the idf, and it will
-      // automatically refresh the cache value.
-      this.idf(term, true)
+  // If restoreCache is set to true, all terms idf scores currently cached will be recomputed.
+  // Otherwise, the cache will just be wiped clean
+  addFileSync (path, encoding, key, restoreCache) {
+    if (!encoding) { encoding = 'utf8' }
+    if (!isEncoding(encoding)) { throw new Error('Invalid encoding: ' + encoding) }
+
+    const document = fs.readFileSync(path, encoding)
+    this.documents.push(buildDocument(document, key))
+
+    // make sure the cache is invalidated when new documents arrive
+    if (restoreCache === true) {
+      for (const term in this._idfCache) {
+        // invoking idf with the force option set will
+        // force a recomputation of the idf, and it will
+        // automatically refresh the cache value.
+        this.idf(term, true)
+      }
+    } else {
+      this._idfCache = {}
     }
-  } else {
-    this._idfCache = {}
   }
-}
 
-TfIdf.prototype.tfidf = function (terms, d) {
-  const _this = this
+  tfidf (terms, d) {
+    const _this = this
 
-  if (!_.isArray(terms)) {
-    terms = tokenizer.tokenize(terms.toString().toLowerCase())
-  }
+    if (!_.isArray(terms)) {
+      terms = tokenizer.tokenize(terms.toString().toLowerCase())
+    }
 
-  return terms.reduce(function (value, term) {
-    let idf = _this.idf(term)
-    idf = idf === Infinity ? 0 : idf
-    return value + (tf(term, _this.documents[d]) * idf)
-  }, 0.0)
-}
+    return terms.reduce(function (value, term) {
+      let idf = _this.idf(term)
+      idf = idf === Infinity ? 0 : idf
+      return value + (TfIdf.tf(term, _this.documents[d]) * idf)
+    }, 0.0)
+  }
 
-TfIdf.prototype.listTerms = function (d) {
-  const terms = []
-  const _this = this
-  for (const term in this.documents[d]) {
-    if (this.documents[d]) {
-      if (term !== '__key') {
-        terms.push({
-          term: term,
-          tf: tf(term, _this.documents[d]),
-          idf: _this.idf(term),
-          tfidf: _this.tfidf(term, d)
-        })
+  listTerms (d) {
+    const terms = []
+    const _this = this
+    for (const term in this.documents[d]) {
+      if (this.documents[d]) {
+        if (term !== '__key') {
+          terms.push({
+            term: term,
+            tf: TfIdf.tf(term, _this.documents[d]),
+            idf: _this.idf(term),
+            tfidf: _this.tfidf(term, d)
+          })
+        }
       }
     }
+
+    return terms.sort(function (x, y) { return y.tfidf - x.tfidf })
   }
 
-  return terms.sort(function (x, y) { return y.tfidf - x.tfidf })
-}
+  tfidfs (terms, callback) {
+    const tfidfs = new Array(this.documents.length)
 
-TfIdf.prototype.tfidfs = function (terms, callback) {
-  const tfidfs = new Array(this.documents.length)
+    for (let i = 0; i < this.documents.length; i++) {
+      tfidfs[i] = this.tfidf(terms, i)
 
-  for (let i = 0; i < this.documents.length; i++) {
-    tfidfs[i] = this.tfidf(terms, i)
+      if (callback) { callback(i, tfidfs[i], this.documents[i].__key) }
+    }
 
-    if (callback) { callback(i, tfidfs[i], this.documents[i].__key) }
+    return tfidfs
   }
 
-  return tfidfs
-}
-
-// Define a tokenizer other than the default "WordTokenizer"
-TfIdf.prototype.setTokenizer = function (t) {
-  if (!_.isFunction(t.tokenize)) { throw new Error('Expected a valid Tokenizer') }
-  tokenizer = t
-}
+  // Define a tokenizer other than the default "WordTokenizer"
+  setTokenizer (t) {
+    if (!_.isFunction(t.tokenize)) { throw new Error('Expected a valid Tokenizer') }
+    tokenizer = t
+  }
 
-// Define a stopwords other than the default
-TfIdf.prototype.setStopwords = function (customStopwords) {
-  if (!Array.isArray(customStopwords)) { return false }
+  // Define a stopwords other than the default
+  setStopwords (customStopwords) {
+    if (!Array.isArray(customStopwords)) { return false }
 
-  let wrongElement = false
-  customStopwords.forEach(stopword => {
-    if ((typeof stopword) !== 'string') {
-      wrongElement = true
+    let wrongElement = false
+    customStopwords.forEach(stopword => {
+      if ((typeof stopword) !== 'string') {
+        wrongElement = true
+      }
+    })
+    if (wrongElement) {
+      return false
     }
-  })
-  if (wrongElement) {
-    return false
-  }
 
-  stopwords = customStopwords
-  return true
+    stopwords = customStopwords
+    return true
+  }
 }
+
+module.exports = TfIdf

package/lib/natural/trie/trie.js

@@ -25,205 +25,207 @@ THE SOFTWARE.
 /**
  * The basis of the TRIE structure.
  **/
-function Trie (caseSensitive) {
-  this.dictionary = Object.create(null) // {}
-  this.$ = false
+class Trie {
+  constructor (caseSensitive) {
+    this.dictionary = Object.create(null) // {}
+    this.$ = false
 
-  if (typeof caseSensitive === 'undefined') {
-    caseSensitive = true
-  }
-
-  this.cs = caseSensitive
-}
-
-/**
- * Add a single string to the TRIE, returns true if the word was already in the
- * trie.
- **/
-Trie.prototype.addString = function (string) {
-  if (this.cs === false) {
-    string = string.toLowerCase()
-  }
+    if (typeof caseSensitive === 'undefined') {
+      caseSensitive = true
+    }
 
-  // If the string has only one letter, mark this as a word.
-  if (string.length === 0) {
-    const wasWord = this.$
-    this.$ = true
-    return wasWord
+    this.cs = caseSensitive
   }
 
-  // Make sure theres a Trie node in our dictionary
-  let next = this.dictionary[string[0]]
+  /**
+   * Add a single string to the TRIE, returns true if the word was already in the
+   * trie.
+   **/
+  addString (string) {
+    if (this.cs === false) {
+      string = string.toLowerCase()
+    }
 
-  if (!next) {
-    this.dictionary[string[0]] = new Trie(this.cs)
-    next = this.dictionary[string[0]]
-  }
+    // If the string has only one letter, mark this as a word.
+    if (string.length === 0) {
+      const wasWord = this.$
+      this.$ = true
+      return wasWord
+    }
 
-  // Continue adding the string
-  return next.addString(string.substring(1))
-}
+    // Make sure theres a Trie node in our dictionary
+    let next = this.dictionary[string[0]]
 
-/**
- * Add multiple strings to the TRIE
- **/
-Trie.prototype.addStrings = function (list) {
-  for (const i in list) {
-    this.addString(list[i])
-  }
-}
-
-/**
- * A function to search the TRIE and return an array of
- * words which have same prefix <prefix>
- * for example if we had the following words in our database:
- * a, ab, bc, cd, abc, abd
- * and we search the string: a
- * we will get :
- * [a, ab, abc, abd]
- **/
-Trie.prototype.keysWithPrefix = function (prefix) {
-  if (this.caseSensitive === false) {
-    prefix = prefix.toLowerCase()
-  }
+    if (!next) {
+      this.dictionary[string[0]] = new Trie(this.cs)
+      next = this.dictionary[string[0]]
+    }
 
-  function isEmpty (object) {
-    for (const key in object) if (object[key]) return false
-    return true
+    // Continue adding the string
+    return next.addString(string.substring(1))
   }
 
-  function get (node, word) {
-    if (!node) return null
-    if (word.length === 0) return node
-    return get(node.dictionary[word[0]], word.substring(1))
+  /**
+   * Add multiple strings to the TRIE
+   **/
+  addStrings (list) {
+    for (const i in list) {
+      this.addString(list[i])
+    }
   }
 
-  function recurse (node, stringAgg, resultsAgg) {
-    if (!node) return
-
-    // Check if this is a word
-    if (node.$) {
-      resultsAgg.push(stringAgg)
+  /**
+   * A function to search the TRIE and return an array of
+   * words which have same prefix <prefix>
+   * for example if we had the following words in our database:
+   * a, ab, bc, cd, abc, abd
+   * and we search the string: a
+   * we will get :
+   * [a, ab, abc, abd]
+   **/
+  keysWithPrefix (prefix) {
+    if (this.caseSensitive === false) {
+      prefix = prefix.toLowerCase()
     }
 
-    if (isEmpty(node.dictionary)) {
-      return
+    function isEmpty (object) {
+      for (const key in object) if (object[key]) return false
+      return true
     }
 
-    for (const c in node.dictionary) {
-      recurse(node.dictionary[c], stringAgg + c, resultsAgg)
+    function get (node, word) {
+      if (!node) return null
+      if (word.length === 0) return node
+      return get(node.dictionary[word[0]], word.substring(1))
     }
-  }
 
-  const results = []
-  recurse(get(this, prefix), prefix, results)
-  return results
-}
+    function recurse (node, stringAgg, resultsAgg) {
+      if (!node) return
 
-/**
- * A function to search the given string and return true if it lands
- * on on a word. Essentially testing if the word exists in the database.
- **/
-Trie.prototype.contains = function (string) {
-  if (this.cs === false) {
-    string = string.toLowerCase()
-  }
+      // Check if this is a word
+      if (node.$) {
+        resultsAgg.push(stringAgg)
+      }
 
-  if (string.length === 0) {
-    return this.$
-  }
-
-  // Otherwise, we need to continue searching
-  const firstLetter = string[0]
-  const next = this.dictionary[firstLetter]
+      if (isEmpty(node.dictionary)) {
+        return
+      }
 
-  // If we don't have a node, this isn't a word
-  if (!next) {
-    return false
-  }
-
-  // Otherwise continue the search at the next node
-  return next.contains(string.substring(1))
-}
+      for (const c in node.dictionary) {
+        recurse(node.dictionary[c], stringAgg + c, resultsAgg)
+      }
+    }
 
-/**
- * A function to search the TRIE and return an array of words which were encountered along the way.
- * This will only return words with full prefix matches.
- * for example if we had the following words in our database:
- * a, ab, bc, cd, abc
- * and we searched the string: abcd
- * we would get only:
- * [a, ab, abc]
- **/
-Trie.prototype.findMatchesOnPath = function (search) {
-  if (this.cs === false) {
-    search = search.toLowerCase()
+    const results = []
+    recurse(get(this, prefix), prefix, results)
+    return results
   }
 
-  function recurse (node, search, stringAgg, resultsAgg) {
-    // Check if this is a word.
-    if (node.$) {
-      resultsAgg.push(stringAgg)
+  /**
+   * A function to search the given string and return true if it lands
+   * on on a word. Essentially testing if the word exists in the database.
+   **/
+  contains (string) {
+    if (this.cs === false) {
+      string = string.toLowerCase()
     }
 
-    // Check if the have completed the seearch
-    if (search.length === 0) {
-      return resultsAgg
+    if (string.length === 0) {
+      return this.$
     }
 
-    // Otherwise, continue searching
-    const next = node.dictionary[search[0]]
+    // Otherwise, we need to continue searching
+    const firstLetter = string[0]
+    const next = this.dictionary[firstLetter]
+
+    // If we don't have a node, this isn't a word
     if (!next) {
-      return resultsAgg
+      return false
     }
-    return recurse(next, search.substring(1), stringAgg + search[0], resultsAgg)
-  };
 
-  return recurse(this, search, '', [])
-}
-
-/**
- * Returns the longest match and the remaining part that could not be matched.
- * inspired by [NLTK containers.trie.find_prefix](http://nltk.googlecode.com/svn-/trunk/doc/api/nltk.containers.Trie-class.html).
- **/
-Trie.prototype.findPrefix = function (search) {
-  if (this.cs === false) {
-    search = search.toLowerCase()
+    // Otherwise continue the search at the next node
+    return next.contains(string.substring(1))
   }
 
-  function recurse (node, search, stringAgg, lastWord) {
-    // Check if this is a word
-    if (node.$) {
-      lastWord = stringAgg
+  /**
+   * A function to search the TRIE and return an array of words which were encountered along the way.
+   * This will only return words with full prefix matches.
+   * for example if we had the following words in our database:
+   * a, ab, bc, cd, abc
+   * and we searched the string: abcd
+   * we would get only:
+   * [a, ab, abc]
+   **/
+  findMatchesOnPath (search) {
+    if (this.cs === false) {
+      search = search.toLowerCase()
     }
 
-    // Check if we have no more to search
-    if (search.length === 0) {
-      return [lastWord, search]
-    }
+    function recurse (node, search, stringAgg, resultsAgg) {
+      // Check if this is a word.
+      if (node.$) {
+        resultsAgg.push(stringAgg)
+      }
+
+      // Check if the have completed the seearch
+      if (search.length === 0) {
+        return resultsAgg
+      }
+
+      // Otherwise, continue searching
+      const next = node.dictionary[search[0]]
+      if (!next) {
+        return resultsAgg
+      }
+      return recurse(next, search.substring(1), stringAgg + search[0], resultsAgg)
+    };
+
+    return recurse(this, search, '', [])
+  }
 
-    // Continue searching
-    const next = node.dictionary[search[0]]
-    if (!next) {
-      return [lastWord, search]
+  /**
+   * Returns the longest match and the remaining part that could not be matched.
+   * inspired by [NLTK containers.trie.find_prefix](http://nltk.googlecode.com/svn-/trunk/doc/api/nltk.containers.Trie-class.html).
+   **/
+  findPrefix (search) {
+    if (this.cs === false) {
+      search = search.toLowerCase()
     }
-    return recurse(next, search.substring(1), stringAgg + search[0], lastWord)
-  };
 
-  return recurse(this, search, '', null)
-}
+    function recurse (node, search, stringAgg, lastWord) {
+      // Check if this is a word
+      if (node.$) {
+        lastWord = stringAgg
+      }
+
+      // Check if we have no more to search
+      if (search.length === 0) {
+        return [lastWord, search]
+      }
+
+      // Continue searching
+      const next = node.dictionary[search[0]]
+      if (!next) {
+        return [lastWord, search]
+      }
+      return recurse(next, search.substring(1), stringAgg + search[0], lastWord)
+    };
+
+    return recurse(this, search, '', null)
+  }
 
-/**
- * Computes the number of actual nodes from this node to the end.
- * Note: This involves traversing the entire structure and may not be
- * good for frequent use.
- **/
-Trie.prototype.getSize = function () {
-  let total = 1
-  for (const c in this.dictionary) {
-    total += this.dictionary[c].getSize()
+  /**
+   * Computes the number of actual nodes from this node to the end.
+   * Note: This involves traversing the entire structure and may not be
+   * good for frequent use.
+   **/
+  getSize () {
+    let total = 1
+    for (const c in this.dictionary) {
+      total += this.dictionary[c].getSize()
+    }
+    return total
   }
-  return total
 }
 
 /**