natural 6.0.1 → 6.1.1

package/.github/workflows/node.js.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [10.x, 12.x, 14.x, 15.x]
+        node-version: [14.x, 16.x, 18.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

package/README.md

@@ -11,9 +11,11 @@
 "Natural" is a general natural language facility for nodejs. It offers a broad range of functionalities for natural language processing. Documentation can be found [here on GitHub Pages](http://naturalnode.github.io/natural/).
 
 
-## License
+## Open source licenses
 
-Copyright (c) 2011, 2012 Chris Umbel, Rob Ellis, Russell Mull
+### Natural: MIT License
+
+Copyright (c) 2011, 2012 Chris Umbel, Rob Ellis, Russell Mull, Hugo W.L. ter Doest
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -33,7 +35,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
-## WordNet License
+### WordNet License
 
 This license is available as the file LICENSE in any downloaded version of WordNet.
 WordNet 3.0 license: (Download)
@@ -58,3 +60,7 @@ Princeton University or Princeton may not be used in advertising or publicity
 pertaining to distribution of the software and/or database. Title to copyright
 in this software, database and any associated documentation shall at all times
 remain with Princeton University and LICENSEE agrees to preserve same.
+
+### Porter stemmer German: BSD License
+
+The Porter stemmer for German is licensed by a BSD license. It states Standard BSD License in the source code, interpreted as the original BSD license consisting of four clauses.

package/lib/natural/index.d.ts

@@ -30,7 +30,7 @@ export * from './ngrams'
 export * from './normalizers'
 export * from './phonetics'
 export * from './sentiment'
-export * from './spelcheck'
+export * from './spellcheck'
 export * from './stemmers'
 export * from './tfidf'
 export * from './tokenizers'

package/lib/natural/stemmers/index.d.ts

@@ -34,6 +34,7 @@ declare const PorterStemmer: Stemmer
 declare const PorterStemmerEs: Stemmer
 declare const PorterStemmerFa: Stemmer
 declare const PorterStemmerFr: Stemmer
+declare const PorterStemmerDe: Stemmer
 declare const PorterStemmerIt: Stemmer
 declare const PorterStemmerNl: Stemmer
 declare const PorterStemmerNo: Stemmer

package/lib/natural/stemmers/index.js

@@ -26,6 +26,7 @@ exports.PorterStemmer = require('./porter_stemmer')
 exports.PorterStemmerFa = require('./porter_stemmer_fa')
 exports.PorterStemmerFr = require('./porter_stemmer_fr')
 exports.CarryStemmerFr = require('./Carry')
+exports.PorterStemmerDe = require('./porter_stemmer_de')
 exports.PorterStemmerRu = require('./porter_stemmer_ru')
 exports.PorterStemmerEs = require('./porter_stemmer_es')
 exports.PorterStemmerIt = require('./porter_stemmer_it')

package/lib/natural/stemmers/porter_stemmer_de.js

@@ -0,0 +1,265 @@
+/*
+ * Original author: Joder Illi
+ * Integration in Natural by Hugo W.L. ter Doest
+ *
+ * Copyright (c) 2010, FormBlitz AG
+ * All rights reserved.
+ * Implementation of the stemming algorithm from http://snowball.tartarus.org/algorithms/german/stemmer.html
+ * Copyright of the algorithm is: Copyright (c) 2001, Dr Martin Porter and can be found at http://snowball.tartarus.org/license.php
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, is covered by the standard BSD license.
+ *
+*/
+
+'use strict'
+
+const Stemmer = require('./stemmer_de')
+
+const PorterStemmer = new Stemmer()
+PorterStemmer.stem = stemm
+module.exports = PorterStemmer
+
+/* by Joder Illi, Snowball mailing list */
+function stemm (word) {
+  /*
+  Put u and y between vowels into upper case
+  */
+  word = word.replace(/([aeiouyäöü])u([aeiouyäöü])/g, '$1U$2')
+  word = word.replace(/([aeiouyäöü])y([aeiouyäöü])/g, '$1Y$2')
+
+  /*
+  and then do the following mappings,
+  (a) replace ß with ss,
+  (a) replace ae with ä, Not doing these,
+  have trouble with diphtongs
+  (a) replace oe with ö, Not doing these,
+  have trouble with diphtongs
+  (a) replace ue with ü unless preceded by q. Not doing these,
+  have trouble with diphtongs
+  So in quelle, ue is not mapped to ü because it follows q, and in
+  feuer it is not mapped because the first part of the rule changes it to
+  feUer, so the u is not found.
+  */
+  word = word.replace(/ß/g, 'ss')
+  // word = word.replace(/ae/g, 'ä');
+  // word = word.replace(/oe/g, 'ö');
+  // word = word.replace(/([^q])ue/g, '$1ü');
+
+  /*
+  R1 and R2 are first set up in the standard way (see the note on R1
+  and R2), but then R1 is adjusted so that the region before it contains at
+  least 3 letters.
+  R1 is the region after the first non-vowel following a vowel, or is
+  the null region at the end of the word if there is no such non-vowel.
+  R2 is the region after the first non-vowel following a vowel in R1,
+  or is the null region at the end of the word if there is no such non-vowel.
+  */
+
+  let r1Index = word.search(/[aeiouyäöü][^aeiouyäöü]/)
+  let r1 = ''
+  if (r1Index !== -1) {
+    r1Index += 2
+    r1 = word.substring(r1Index)
+  }
+
+  let r2Index = -1
+  // let r2 = ''
+
+  if (r1Index !== -1) {
+    r2Index = r1.search(/[aeiouyäöü][^aeiouyäöü]/)
+    if (r2Index !== -1) {
+      r2Index += 2
+      // r2 = r1.substring(r2Index)
+      r2Index += r1Index
+    } else {
+      // r2 = ''
+    }
+  }
+
+  if (r1Index !== -1 && r1Index < 3) {
+    r1Index = 3
+    r1 = word.substring(r1Index)
+  }
+
+  /*
+  Define a valid s-ending as one of b, d, f, g, h, k, l, m, n, r or t.
+  Define a valid st-ending as the same list, excluding letter r.
+  */
+
+  /*
+  Do each of steps 1, 2 and 3.
+  */
+
+  /*
+  Step 1:
+  Search for the longest among the following suffixes,
+  (a) em ern er
+  (b) e en es
+  (c) s (preceded by a valid s-ending)
+  */
+  const a1Index = word.search(/(em|ern|er)$/g)
+  const b1Index = word.search(/(e|en|es)$/g)
+  let c1Index = word.search(/([bdfghklmnrt]s)$/g)
+  if (c1Index !== -1) {
+    c1Index++
+  }
+  let index1 = 10000
+  let optionUsed1 = ''
+  if (a1Index !== -1 && a1Index < index1) {
+    optionUsed1 = 'a'
+    index1 = a1Index
+  }
+  if (b1Index !== -1 && b1Index < index1) {
+    optionUsed1 = 'b'
+    index1 = b1Index
+  }
+  if (c1Index !== -1 && c1Index < index1) {
+    optionUsed1 = 'c'
+    index1 = c1Index
+  }
+
+  /*
+  and delete if in R1. (Of course the letter of the valid s-ending is
+  not necessarily in R1.) If an ending of group (b) is deleted, and the ending
+  is preceded by niss, delete the final s.
+  (For example, äckern -> äck, ackers -> acker, armes -> arm,
+  bedürfnissen -> bedürfnis)
+  */
+
+  if (index1 !== 10000 && r1Index !== -1) {
+    if (index1 >= r1Index) {
+      word = word.substring(0, index1)
+      if (optionUsed1 === 'b') {
+        if (word.search(/niss$/) !== -1) {
+          word = word.substring(0, word.length - 1)
+        }
+      }
+    }
+  }
+
+  /*
+  Step 2:
+  Search for the longest among the following suffixes,
+  (a) en er est
+  (b) st (preceded by a valid st-ending, itself preceded by at least 3
+  letters)
+  */
+
+  const a2Index = word.search(/(en|er|est)$/g)
+  let b2Index = word.search(/(.{3}[bdfghklmnt]st)$/g)
+  if (b2Index !== -1) {
+    b2Index += 4
+  }
+
+  let index2 = 10000
+  // const optionUsed2 = ''
+  if (a2Index !== -1 && a2Index < index2) {
+    // optionUsed2 = 'a'
+    index2 = a2Index
+  }
+  if (b2Index !== -1 && b2Index < index2) {
+    // optionUsed2 = 'b'
+    index2 = b2Index
+  }
+
+  /*
+  and delete if in R1.
+  (For example, derbsten -> derbst by step 1, and derbst -> derb by
+  step 2, since b is a valid st-ending, and is preceded by just 3 letters)
+  */
+
+  if (index2 !== 10000 && r1Index !== -1) {
+    if (index2 >= r1Index) {
+      word = word.substring(0, index2)
+    }
+  }
+
+  /*
+  Step 3: d-suffixes (*)
+  Search for the longest among the following suffixes, and perform the
+  action indicated.
+  end ung
+  delete if in R2
+  if preceded by ig, delete if in R2 and not preceded by e
+  ig ik isch
+  delete if in R2 and not preceded by e
+  lich heit
+  delete if in R2
+  if preceded by er or en, delete if in R1
+  keit
+  delete if in R2
+  if preceded by lich or ig, delete if in R2
+  */
+
+  const a3Index = word.search(/(end|ung)$/g)
+  let b3Index = word.search(/[^e](ig|ik|isch)$/g)
+  const c3Index = word.search(/(lich|heit)$/g)
+  const d3Index = word.search(/(keit)$/g)
+  if (b3Index !== -1) {
+    b3Index++
+  }
+
+  let index3 = 10000
+  let optionUsed3 = ''
+  if (a3Index !== -1 && a3Index < index3) {
+    optionUsed3 = 'a'
+    index3 = a3Index
+  }
+  if (b3Index !== -1 && b3Index < index3) {
+    optionUsed3 = 'b'
+    index3 = b3Index
+  }
+  if (c3Index !== -1 && c3Index < index3) {
+    optionUsed3 = 'c'
+    index3 = c3Index
+  }
+  if (d3Index !== -1 && d3Index < index3) {
+    optionUsed3 = 'd'
+    index3 = d3Index
+  }
+
+  if (index3 !== 10000 && r2Index !== -1) {
+    if (index3 >= r2Index) {
+      word = word.substring(0, index3)
+      let optionIndex = -1
+      // const optionSubsrt = ''
+      if (optionUsed3 === 'a') {
+        optionIndex = word.search(/[^e](ig)$/)
+        if (optionIndex !== -1) {
+          optionIndex++
+          if (optionIndex >= r2Index) {
+            word = word.substring(0, optionIndex)
+          }
+        }
+      } else if (optionUsed3 === 'c') {
+        optionIndex = word.search(/(er|en)$/)
+        if (optionIndex !== -1) {
+          if (optionIndex >= r1Index) {
+            word = word.substring(0, optionIndex)
+          }
+        }
+      } else if (optionUsed3 === 'd') {
+        optionIndex = word.search(/(lich|ig)$/)
+        if (optionIndex !== -1) {
+          if (optionIndex >= r2Index) {
+            word = word.substring(0, optionIndex)
+          }
+        }
+      }
+    }
+  }
+
+  /*
+  Finally,
+  turn U and Y back into lower case, and remove the umlaut accent from
+  a, o and u.
+  */
+  word = word.replace(/U/g, 'u')
+  word = word.replace(/Y/g, 'y')
+  word = word.replace(/ä/g, 'a')
+  word = word.replace(/ö/g, 'o')
+  word = word.replace(/ü/g, 'u')
+
+  return word
+}

package/lib/natural/stemmers/stemmer_de.js

@@ -0,0 +1,54 @@
+
+/*
+Copyright (c) 2023, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+const stopwords = require('stopwords-iso')
+const englishStopWords = stopwords.de
+const Tokenizer = require('../tokenizers/aggressive_tokenizer_de')
+
+class Stemmer {
+  stem (token) {
+    return token
+  }
+
+  tokenizeAndStem (text, keepStops) {
+    const stemmedTokens = []
+
+    const that = this
+    new Tokenizer().tokenize(text).forEach(function (token) {
+      if (keepStops || englishStopWords.indexOf(token) === -1) {
+        let resultToken = token.toLowerCase()
+        // if (resultToken.match(new RegExp('[a-záéíóúüñ0-9]+', 'gi'))) {
+        if (resultToken.match(/[a-záéíóúüñ0-9]+/gi)) {
+          resultToken = that.stem(resultToken)
+        }
+        stemmedTokens.push(resultToken)
+      }
+    })
+
+    return stemmedTokens
+  }
+}
+
+module.exports = Stemmer

package/lib/natural/tokenizers/aggressive_tokenizer_de.js

@@ -0,0 +1,38 @@
+/*
+Copyright (c) 2011, Chris Umbel, Martijn de Boer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+'use strict'
+
+const Tokenizer = require('./tokenizer')
+const util = require('util')
+
+const AggressiveTokenizer = function () {
+  Tokenizer.call(this)
+}
+util.inherits(AggressiveTokenizer, Tokenizer)
+
+module.exports = AggressiveTokenizer
+
+AggressiveTokenizer.prototype.tokenize = function (text) {
+  // break a string up into an array of tokens by anything non-word
+  return this.trim(text.split(/[^a-zA-Z0-9_'-]+/))
+}

package/lib/natural/tokenizers/index.d.ts

@@ -67,6 +67,9 @@ export declare class AggressiveTokenizerFa implements Tokenizer {
 export declare class AggressiveTokenizerFr implements Tokenizer {
   tokenize (text: string): string[]
 }
+export declare class AggressiveTokenizerDe implements Tokenizer {
+  tokenize (text: string): string[]
+}
 export declare class AggressiveTokenizerId implements Tokenizer {
   tokenize (text: string): string[]
 }

package/lib/natural/tokenizers/index.js

@@ -25,6 +25,7 @@ THE SOFTWARE.
 exports.AggressiveTokenizerNl = require('./aggressive_tokenizer_nl')
 exports.AggressiveTokenizerFa = require('./aggressive_tokenizer_fa')
 exports.AggressiveTokenizerFr = require('./aggressive_tokenizer_fr')
+exports.AggressiveTokenizerDe = require('./aggressive_tokenizer_de')
 exports.AggressiveTokenizerRu = require('./aggressive_tokenizer_ru')
 exports.AggressiveTokenizerEs = require('./aggressive_tokenizer_es')
 exports.AggressiveTokenizerIt = require('./aggressive_tokenizer_it')

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.0.1",
+  "version": "6.1.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -14,12 +14,14 @@
     "afinn-165": "^1.0.2",
     "apparatus": "^0.0.10",
     "safe-stable-stringify": "^2.2.0",
+    "stopwords-iso": "^1.1.0",
     "sylvester": "^0.0.12",
     "tsc": "^2.0.4",
     "underscore": "^1.9.1",
     "wordnet-db": "^3.1.11"
   },
   "devDependencies": {
+    "@types/node": "^18.11.18",
     "browserfs": "^1.4.3",
     "gulp": "^4.0.2",
     "gulp-jasmine": "^4.0.0",