natural 8.0.1 → 8.1.1

package/.github/workflows/node.js.yml

@@ -44,7 +44,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x]
+        node-version: [20.x, 22.x, 24.x]
         # See supported Node.js release schedule at 
         #https://nodejs.org/en/about/releases/
 
@@ -60,8 +60,14 @@ jobs:
       - name: Clean install
         run: npm ci
 
+      - name: Lint
+        run: npm run lint
+
       - name: Build
         run: npm run build
+
+      - name: ESM smoke test
+        run: npm run smoke:esm
     
       - name: Run Istanbul coverage
         run: npm run coverage

package/lib/natural/sentiment/SentimentAnalyzer.js

@@ -25,7 +25,7 @@
 const DEBUG = false
 
 // Afinn
-const englishAfinnVoca = require('afinn-165')
+const englishAfinnVoca = require('afinn-165').afinn165
 const spanishAfinnVoca = require('./Spanish/afinnShortSortedSpanish.json')
 const portugueseAfinnVoca = require('./Portuguese/afinnShortSortedPortuguese.json')
 

package/lib/natural/stemmers/index.d.ts

@@ -27,7 +27,7 @@ declare interface RegionsType {
 }
 
 export declare interface Stemmer {
-  stem: (token: string) => string
+  stem: (token: string, options?: any) => string
   addStopWord: (stopWord: string) => void
   addStopWords: (moreStopWords: string[]) => void
   removeStopWord: (stopWord: string) => void

package/lib/natural/stemmers/porter_stemmer_de.js

@@ -21,7 +21,7 @@ PorterStemmer.stem = stemm
 module.exports = PorterStemmer
 
 /* by Joder Illi, Snowball mailing list */
-function stemm (word) {
+function stemm (word, options = {}) {
   /*
   Put u and y between vowels into upper case
   */
@@ -257,6 +257,9 @@ function stemm (word) {
   */
   word = word.replace(/U/g, 'u')
   word = word.replace(/Y/g, 'y')
+  if (options.preserveUmlauts) {
+    return word
+  }
   word = word.replace(/ä/g, 'a')
   word = word.replace(/ö/g, 'o')
   word = word.replace(/ü/g, 'u')

package/lib/natural/tokenizers/sentence_tokenizer.js

@@ -41,13 +41,18 @@ function escapeRegExp (string) {
 }
 
 class SentenceTokenizer extends Tokenizer {
-  constructor (abbreviations) {
+  constructor (abbreviations, trimSentences) {
     super()
     if (abbreviations) {
       this.abbreviations = abbreviations
     } else {
       this.abbreviations = []
     }
+    if (trimSentences === undefined) {
+      this.trimSentences = true
+    } else {
+      this.trimSentences = trimSentences
+    }
     this.replacementMap = null
     this.replacementCounter = 0
   }
@@ -181,7 +186,7 @@ class SentenceTokenizer extends Tokenizer {
     const trimmedSentences = this.trim(newSentences)
     DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences))
 
-    const trimmedSentences2 = trimmedSentences.map(sent => sent.trim())
+    const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent)
     DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2))
     DEBUG && console.log('---End of sentence tokenization--------------------------')
     DEBUG && console.log('---Replacement map---------------------------------------')

package/lib/natural/util/abbreviations_es.js

@@ -0,0 +1,22 @@
+const knownAbbreviations = [
+  // Titles (Sir, Doctor, etc)
+  'Sr.', 'Sra.', 'Srta.', 'Srs.', 'Sras.', 'Dr.', 'Dra.', 'Drs.', 'Dras.',
+  'Lic.', 'Licda.', 'Licdo.', 'Licds.', 'Ings.', 'Ing.', 'Arq.', 'Arqs.', 'Prof.', 'Profa.', 'Profs.', 'Profas.',
+
+  // Generals
+  'etc.', 'e.g.', 'i.e.', 'p.ej.', 'p.e.', 'a.m.', 'p.m.',
+  'núm.', 'núms.', 'n.os', 'n.os.', 'ud.', 'uds.',
+  'c/ap.', 'c/u.', 's/n.', 'av.', 'pto.', 'ptos.', 'pág.', 'págs.', 'vol.', 'vols.', 'ed.', 'eds.', 'cap.', 'caps.',
+  'mín.', 'máx.', 'aprox.', 'ant.', 'sig.', 'hist.', 'biol.', 'quím.', 'mat.', 'psic.',
+  'adj.', 'adv.', 'art.', 'arts.', 'vb.', 'vbs.', 'sust.', 'susts.', 'prep.', 'preps.',
+
+  // Legal written in Spanish
+  'Art.', 'Arts.', 'Inc.', 'Incs.', 'const.', 'Cód.', 'Códs.',
+  'C.C.', 'C.P.', 'C.N.', 'DNU.', 'DTO.', 'Res.', 'Disp.', 'Disps.', 'C.P.C.C.', 'C.C.Y.C.',
+  'expte.', 'exptes.', 'fs.', 'fjs.', 'op.', 'cf.', 'cit.', 'loc. cit.', 'ut supra.', 'vgr.', 'ap.', 'cfr.', 'ss.',
+
+  // Latin-legal written in Spanish
+  'et al.', 'ibid.', 'ibíd.', 'op. cit.', 'loc. cit.', 'id.', 'vs.', 'a priori.', 'a posteriori.', 'sine die.'
+]
+
+exports.knownAbbreviations = knownAbbreviations

package/lib/natural/util/index.d.ts

@@ -25,6 +25,9 @@ THE SOFTWARE.
 
 export let stopwords: string[]
 
+export let abbreviations: string[]
+export let abbreviationsEs: string[]
+
 declare class Bag<T> {
   add (element: T): Bag<T>
   isEmpty (): boolean

package/lib/natural/util/index.js

@@ -24,6 +24,7 @@ THE SOFTWARE.
 
 exports.stopwords = require('./stopwords').words
 exports.abbreviations = require('./abbreviations_en').knownAbbreviations
+exports.abbreviations_es = require('./abbreviations_es').knownAbbreviations
 exports.ShortestPathTree = require('./shortest_path_tree')
 exports.LongestPathTree = require('./longest_path_tree')
 exports.DirectedEdge = require('./directed_edge')

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "8.0.1",
+  "version": "8.1.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -11,66 +11,46 @@
     "node": ">=0.4.10"
   },
   "dependencies": {
-    "afinn-165": "^1.0.2",
+    "afinn-165": "^2.0.2",
     "afinn-165-financialmarketnews": "^3.0.0",
     "apparatus": "^0.0.10",
-    "dotenv": "^16.4.5",
-    "http-server": "^14.1.1",
+    "dotenv": "^17.3.1",
     "memjs": "^1.3.2",
-    "mongoose": "^8.2.0",
-    "pg": "^8.11.3",
-    "redis": "^4.6.13",
-    "safe-stable-stringify": "^2.2.0",
+    "mongoose": "^9.2.1",
+    "pg": "^8.18.0",
+    "redis": "^5.11.0",
+    "safe-stable-stringify": "^2.5.0",
     "stopwords-iso": "^1.1.0",
-    "sylvester": "^0.0.12",
-    "underscore": "^1.9.1",
-    "uuid": "^9.0.1",
-    "wordnet-db": "^3.1.11"
+    "sylvester": "^0.0.21",
+    "underscore": "^1.13.0",
+    "uuid": "^13.0.0",
+    "wordnet-db": "^3.1.14"
   },
   "devDependencies": {
-    "@rollup/plugin-commonjs": "^25.0.8",
+    "@rollup/plugin-commonjs": "^29.0.0",
     "@rollup/plugin-json": "^6.1.0",
-    "@rollup/plugin-node-resolve": "^15.2.3",
-    "@types/jasmine": "^5.1.4",
-    "@types/node": "^18.11.18",
+    "@rollup/plugin-node-resolve": "^16.0.3",
+    "@types/jasmine": "^6.0.0",
+    "@types/node": "^25.2.3",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
-    "browserfs": "^1.4.3",
-    "cross-env": "^7.0.3",
+    "cross-env": "^10.1.0",
     "eslint": "^8.57.0",
     "eslint-config-standard-with-typescript": "^43.0.1",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-n": "^15.7.0",
-    "eslint-plugin-promise": "^6.1.1",
-    "gulp": "^4.0.2",
-    "gulp-jasmine": "^4.0.0",
-    "gulp-jasmine-browser": "^4.1.0",
-    "jasmine": "^3.3.1",
-    "jasmine-core": "^3.3.0",
-    "lint": "^0.8.19",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-n": "^16.6.2",
+    "eslint-plugin-promise": "^6.6.0",
+    "jasmine": "^6.0.0",
+    "jasmine-core": "^6.0.0",
     "ncp": "^2.0.0",
     "nyc": "^15.1.0",
     "pegjs": "^0.10.0",
-    "proxyquire": "^1.8.0",
-    "rimraf": "^2.6.3",
-    "rollup": "^4.18.0",
-    "sinon": "^1.12.2",
+    "proxyquire": "^2.1.3",
+    "rimraf": "^5.0.0",
+    "rollup": "^4.57.1",
+    "sinon": "^17.0.0",
     "source-map-support": "^0.5.21",
-    "standard": "^16.0.4",
     "ts-node": "^10.9.2",
-    "ts-standard": "^12.0.2",
-    "typescript": "^5.4.2",
-    "uubench": "^0.0.1",
-    "webpack": "^4.29.0",
-    "webpack-stream": "^5.2.1"
-  },
-  "standard": {
-    "ignore": [
-      "/lib/natural/brill_pos_tagger/lib/TF_Parser.js",
-      "/lib/natural/tokenizers/parser_sentence_tokenizer.js"
-    ],
-    "env": {
-      "jasmine": true
-    }
+    "typescript": "^5.9.3"
   },
   "jscpd": {
     "ignore": [
@@ -86,8 +66,8 @@
     "clean": "rimraf *~ #* *classifier.json dist io_spec/tmp/*.json coverage",
     "test_io": "jasmine --random=false io_spec/*_spec.js",
     "test": "cross-env NODE_PATH=. jasmine --random=false dist/cjs/spec/*_spec.js",
+    "smoke:esm": "node spec/esm/smoke.mjs",
     "coverage": "nyc --no-clean npm run test && nyc --no-clean npm run test_io",
-    "test_browser": "cross-env NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
     "lint": "eslint . --ext .ts"
   },
   "license": "MIT",

package/rollup.config.mjs

@@ -1,19 +1,51 @@
 import resolve from '@rollup/plugin-node-resolve'
 import commonjs from '@rollup/plugin-commonjs'
 import json from '@rollup/plugin-json'
+import { builtinModules, createRequire } from 'node:module'
+
+const require = createRequire(import.meta.url)
+const pkg = require('./package.json')
+
+const deps = Object.keys(pkg.dependencies || {})
+const peerDeps = Object.keys(pkg.peerDependencies || {})
+
+const jsonEntryDeps = new Set(
+  deps.filter((dep) => {
+    try {
+      return require.resolve(dep).endsWith('.json')
+    } catch {
+      return false
+    }
+  })
+)
+const nodeBuiltins = builtinModules.flatMap((name) => (
+  name.startsWith('node:') ? [name, name.slice(5)] : [name, `node:${name}`]
+))
+
+const external = (id) => (
+  nodeBuiltins.includes(id) ||
+  deps.some((dep) => (id === dep || id.startsWith(`${dep}/`)) && !jsonEntryDeps.has(dep)) ||
+  peerDeps.some((dep) => id === dep || id.startsWith(`${dep}/`))
+)
 
 export default {
   input: 'lib/natural/index.js', // Your entry point
+  external,
   output: {
     dir: 'dist/esm', // Output directory for ESM build
     format: 'esm', // Output format as ESM
-    sourcemap: true // Optional: generate source maps
+    sourcemap: true, // Optional: generate source maps
+    preserveModules: true,
+    preserveModulesRoot: 'lib/natural'
   },
   plugins: [
     resolve({
       preferBuiltins: true // Prefer Node.js built-in modules over local modules
     }), // Resolve modules from node_modules
-    commonjs(), // Convert CommonJS modules to ES modules
+    commonjs({
+      esmExternals: (id) => !nodeBuiltins.includes(id),
+      requireReturnsDefault: 'auto'
+    }), // Convert CommonJS modules to ES modules
     json() // Add the JSON plugin to handle JSON imports
   ]
 }

package/gulpfile.js

@@ -1,34 +0,0 @@
-/*
-Copyright (c) 2020, Hugo W.L. ter Doest
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-const { src } = require('gulp')
-const jasmineBrowser = require('gulp-jasmine-browser')
-const webpackStream = require('webpack-stream')
-
-exports.default = function () {
-  return src(['spec/*_spec.js'])
-    .pipe(webpackStream(require('webpack.config')))
-    .pipe(jasmineBrowser.specRunner({ random: false }))
-    // Server is at http://localhost:8888
-    // In the options tab uncheck "run tests in random order"
-    .pipe(jasmineBrowser.server())
-}

package/webpack.config.js

@@ -1,50 +0,0 @@
-const webpack = require('webpack')
-
-module.exports = {
-
-  mode: 'none',
-
-  target: 'web',
-
-  resolve: {
-    extensions: ['.js', '.json'],
-
-    modules: ['node_modules', '.'],
-
-    // Use our versions of Node modules.
-    alias: {
-      fs: 'browserfs/dist/shims/fs.js',
-      buffer: 'browserfs/dist/shims/buffer.js',
-      path: 'browserfs/dist/shims/path.js',
-      processGlobal: 'browserfs/dist/shims/process.js',
-      bufferGlobal: 'browserfs/dist/shims/bufferGlobal.js',
-      bfsGlobal: require.resolve('browserfs')
-    }
-  },
-
-  // REQUIRED to avoid issue "Uncaught TypeError: BrowserFS.BFSRequire is not a function"
-  // See: https://github.com/jvilk/BrowserFS/issues/201
-  module: {
-    noParse: /browserfs\.js/
-  },
-
-  watch: false,
-
-  output: { filename: 'spec.js' },
-
-  plugins: [
-    // Expose BrowserFS, process, and Buffer globals.
-    // NOTE: If you intend to use BrowserFS in a script tag, you do not need
-    // to expose a BrowserFS global.
-    new webpack.ProvidePlugin({ BrowserFS: 'bfsGlobal', process: 'processGlobal', Buffer: 'bufferGlobal' })
-  ],
-
-  // DISABLE Webpack's built-in process and Buffer polyfills!
-  node: {
-    process: false,
-    Buffer: false
-  },
-
-  stats: 'normal'
-
-}