natural 6.9.1 → 6.9.2

package/lib/natural/stemmers/stemmer_nl.js

@@ -23,7 +23,7 @@ THE SOFTWARE.
 'use strict'
 
 const stopwords = require('../util/stopwords_nl')
-const Tokenizer = require('../tokenizers/aggressive_tokenizer')
+const Tokenizer = require('../tokenizers/aggressive_tokenizer_nl')
 
 class Stemmer {
   stem (token) {

package/lib/natural/tokenizers/aggressive_tokenizer.js

@@ -35,5 +35,10 @@ module.exports = AggressiveTokenizer
 AggressiveTokenizer.prototype.tokenize = function (text) {
   // break a string up into an array of tokens by anything non-word
   // underscore is considered to be non word character
-  return this.trim(text.split(/[\W|_]+/))
+  // Old splitter:
+  // return this.trim(text.split(/[\W|_]+/))
+  // New splitter:
+  // Explicitly mentions which characters make up words.
+  // So words may contain hyphen -, single quote ' and slash /
+  return this.trim(text.split(/[^a-zA-Z0-9'\-/]+/))
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.9.1",
+  "version": "6.9.2",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",