natural 6.10.5 → 6.11.1

package/.env

@@ -0,0 +1,22 @@
+# Storage backend you want to use
+NATURAL_STORAGE=PG
+# Settings for Postgres
+PG_USER=user
+PG_PASSWORD=password
+PG_DATABASE=naturaldb
+PG_HOST=localhost
+PG_PORT=5432
+PG_TABLE=naturaltable
+# Settings for Redis
+REDIS_HOST=localhost
+REDIS_PORT=6379
+# Settings for Memcached
+MEM_HOST=localhost
+MEM_PORT=11211
+MEM_EXPIRES = 3600
+# Settings for MongoDB
+MONGO_HOST=localhost
+MONGO_PORT=27017
+MONGO_DATABASE=naturaldb
+# Path to the folder writing files 
+FS_PATH=.

package/.github/workflows/node.js.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [14.x, 16.x, 18.x]
+        node-version: [16.x, 18.x, 20.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

package/.github/workflows/npm-publish.yml

@@ -11,10 +11,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 12
+          node-version: 20
       - run: npm ci
       - run: npm test
 
@@ -22,10 +22,10 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 14
+          node-version: 20
           registry-url: https://registry.npmjs.org/
       - run: npm ci
       - run: git config user.name "$GITHUB_ACTOR"

package/example.txt

@@ -0,0 +1 @@
+{"attr1":"val1","attr2":"val2"}

package/f2f5d4bc-daa0-4392-8981-9614cd0c802e.json

@@ -0,0 +1 @@
+{"attr1":"val1","attr2":"val2"}

package/lib/natural/classifiers/classifier.js

@@ -23,7 +23,6 @@ THE SOFTWARE.
 'use strict'
 
 const EventEmitter = require('events')
-
 const PorterStemmer = require('../stemmers/porter_stemmer')
 const parallelTrainer = require('./classifier_train_parallel')
 

package/lib/natural/classifiers/index.d.ts

@@ -72,13 +72,12 @@ declare interface ClassifierOptions {
 
 declare type ClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: ClassifierBase) => void
 
-declare class ClassifierBase {
+declare class ClassifierBase extends events.EventEmitter {
   classifier: ApparatusClassifier
   docs: ClassifierDoc[]
   features: { [key: string]: number | undefined }
   stemmer: Stemmer
   lastAdded: number
-  events: events.EventEmitter
 
   constructor (classifier: ApparatusClassifier, stemmer?: Stemmer)
   addDocument (text: string | string[], classification: string): void
@@ -89,7 +88,7 @@ declare class ClassifierBase {
   getClassifications (observation: string | string[]): ApparatusClassification[]
   classify (observation: string | string[]): string
   setOptions (options: ClassifierOptions): void
-  save (filename: string, callback: ClassifierCallback): void
+  save (filename: string, callback?: ClassifierCallback): void
 }
 
 declare type BayesClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: BayesClassifier) => void

package/lib/natural/classifiers/logistic_regression_classifier.js

@@ -56,7 +56,7 @@ class LogisticRegressionClassifier extends Classifier {
     // needs its matricies to have their widths synced, etc.
     this.lastAdded = 0
     this.classifier = new ApparatusLogisticRegressionClassifier()
-    Classifier.prototype.train.call(this)
+    super.train()
   }
 }
 

package/lib/natural/util/storage/File.js

@@ -0,0 +1,21 @@
+require('dotenv').config()
+const { v4: uuidv4 } = require('uuid')
+const fs = require('fs').promises
+const path = process.env.FS_PATH
+
+// Class for creating file-based storage
+class FilePlugin {
+  async store (object, options) {
+    const key = uuidv4()
+    const data = JSON.stringify(object)
+    await fs.writeFile(path + '/' + key + '.json', data)
+    return key
+  }
+
+  async retrieve (key, options) {
+    const data = await fs.readFile(path + '/' + key + '.json', 'utf8')
+    return JSON.parse(data)
+  }
+}
+
+module.exports = FilePlugin

package/lib/natural/util/storage/Memcached.js

@@ -0,0 +1,27 @@
+const { v4: uuidv4 } = require('uuid')
+require('dotenv').config()
+const memjs = require('memjs')
+const memCachedConnectionString = process.env.MEM_HOST + ':' +
+  process.env.MEM_PORT
+const memCachedExpiration = process.env.MEM_EXPIRES || 2
+
+// Class for creating memcached client including store and retrieve methods
+class MemcachedPlugin {
+  constructor () {
+    this.client = memjs.Client.create(memCachedConnectionString)
+  }
+
+  async store (object, options) {
+    const key = uuidv4()
+    const data = JSON.stringify(object)
+    const isStored = await this.client.set(key, data, { expires: memCachedExpiration })
+    return isStored ? key : null
+  }
+
+  async retrieve (key, options) {
+    const result = await this.client.get(key)
+    return JSON.parse(result.value.toString())
+  }
+}
+
+module.exports = MemcachedPlugin

package/lib/natural/util/storage/MongoDB.js

@@ -0,0 +1,27 @@
+require('dotenv').config()
+const mongoose = require('mongoose')
+const mongoDBConnectionString = 'mongodb://' + process.env.MONGO_HOST + ':' +
+  process.env.MONGO_PORT + '/' + process.env.MONGO_DATABASE
+
+class MongoDBPlugin {
+  constructor () {
+    mongoose.connect(mongoDBConnectionString, {})
+    // We do not enforce a schema with MongoDB
+    this.ObjectModel = mongoose.model('ObjectModel', new mongoose.Schema({},
+      { strict: false, versionKey: false }))
+  }
+
+  async store (object, options) {
+    // To make sure the payload is stored as a whole and nog merged with the id object
+    const objectToStore = { naturalObject: object }
+    const storedObject = await this.ObjectModel.create(objectToStore)
+    return storedObject._id
+  }
+
+  async retrieve (key, options) {
+    const retrievedObject = await this.ObjectModel.findById(key)
+    return retrievedObject.naturalObject
+  }
+}
+
+module.exports = MongoDBPlugin

package/lib/natural/util/storage/Postgres.js

@@ -0,0 +1,89 @@
+const postgres = require('pg')
+require('dotenv').config()
+
+class PostgresPlugin {
+  constructor () {
+    this.configPostgres()
+  }
+
+  async configPostgres () {
+    // Initialize connection to Postgres
+    const client = new postgres.Client({
+      user: process.env.PG_USER,
+      host: process.env.PG_HOST,
+      database: process.env.PG_DATABASE,
+      password: process.env.PG_PASSWORD,
+      port: process.env.PG_PORT
+    })
+    this.postgresClient = client
+    this.postgresTableName = process.env.PG_TABLE
+    // Connect to the database
+    return new Promise((resolve, reject) => {
+      client.connect()
+        .then(() => {
+          // Call the function to create the table
+          createTable(this.postgresTableName)
+          resolve()
+        })
+        .catch(err => {
+          console.error('Error connecting to the database:', err)
+          reject(err)
+        })
+    })
+
+    // Function to create the table
+    function createTable (tableName) {
+      // We use JSONB (binary JSON format of Postgres) to store objects
+      const createTableQuery = `
+        CREATE TABLE IF NOT EXISTS ${tableName} (
+          id SERIAL PRIMARY KEY,
+          data JSONB
+        );
+      `
+      return new Promise((resolve, reject) => {
+        client.query(createTableQuery)
+          .then(() => {
+            resolve()
+          })
+          .catch(error => {
+            reject(error)
+          })
+      })
+    }
+  }
+
+  // Function to insert a JavaScript object by key
+  // Returns the id generated by the database
+  async store (object, options) {
+    try {
+      const query = `INSERT INTO ${this.postgresTableName} (data) VALUES ($1)  RETURNING id;`
+      const result = await this.postgresClient.query(query, [object])
+      if (result.rowCount > 0) {
+        return result.rows[0].id
+      } else {
+        return null
+      }
+    } catch (error) {
+      console.error('Error inserting object:', error)
+    }
+  }
+
+  // Function to retrieve a JavaScript object by key
+  async retrieve (key, options) {
+    try {
+      const query = `SELECT data FROM ${this.postgresTableName} WHERE id = $1`
+      const result = await this.postgresClient.query(query, [key])
+      if (result.rows.length > 0) {
+        return result.rows[0].data
+      } else {
+        console.log('No object found with the specified key')
+        return null
+      }
+    } catch (error) {
+      console.error('Error retrieving object:', error)
+      return null
+    }
+  }
+}
+
+module.exports = PostgresPlugin

package/lib/natural/util/storage/Redis.js

@@ -0,0 +1,32 @@
+const { v4: uuidv4 } = require('uuid')
+require('dotenv').config()
+const redis = require('redis')
+const redisConnectionString = 'redis://' + process.env.REDIS_HOST + ':' +
+  process.env.REDIS_PORT
+
+class RedisPlugin {
+  constructor () {
+    this.configRedis()
+  }
+
+  async configRedis () {
+    this.redisClient = await redis.createClient(redisConnectionString)
+    this.redisClient
+      .on('error', err => console.log('Redis Client Error', err))
+      .connect()
+  }
+
+  async store (object, options) {
+    const key = uuidv4()
+    const data = JSON.stringify(object)
+    await this.redisClient.set(key, data)
+    return key
+  }
+
+  async retrieve (key, options) {
+    const result = await this.redisClient.get(key)
+    return JSON.parse(result)
+  }
+}
+
+module.exports = RedisPlugin

package/lib/natural/util/storage/StorageBackend.js

@@ -0,0 +1,77 @@
+const PostgresPlugin = require('./Postgres')
+const MongoDBPlugin = require('./MongoDB')
+const RedisPlugin = require('./Redis')
+const MemcachedPlugin = require('./Memcached')
+const FilePlugin = require('./File')
+
+// Enum for different types of storage backends that can be used
+const STORAGE_TYPES = {
+  POSTGRES: 'POSTGRES',
+  REDIS: 'REDIS',
+  MONGODB: 'MONGODB',
+  MEMCACHED: 'MEMCACHED',
+  FILE: 'FILE'
+}
+
+// Standard behaviour of the backend is:
+// - Javascript in and out, so JSON.stringify and JSON.parse is done here.
+// - A key is returned after storing an object.
+// - The key is determined by the storage method, if possible: MongoDB and Postgres can do this.
+//   If not, a uuid is created.
+// - File-based storage is no different: a uuid is created and used as filename: uuid.json
+class StorageBackend {
+  constructor (storageType, options) {
+    if (storageType) {
+      this.setStorageType(storageType, options)
+    }
+  }
+
+  // Sets the storage type. Valid values are POSTGRES, REDIS, MONGODB, MEMCACHED, FILE
+  async setStorageType (storageType) {
+    if (!storageType) {
+      throw new Error('No storage type specified')
+    }
+    if (this.storageType) {
+      throw new Error('Storage type already set')
+    }
+    this.storageType = storageType
+    switch (storageType) {
+      case STORAGE_TYPES.POSTGRES:
+        this.client = new PostgresPlugin()
+        break
+      case STORAGE_TYPES.REDIS:
+        this.client = new RedisPlugin()
+        break
+      case STORAGE_TYPES.MONGODB:
+        this.client = new MongoDBPlugin()
+        break
+      case STORAGE_TYPES.MEMCACHED:
+        this.client = new MemcachedPlugin()
+        break
+      case STORAGE_TYPES.FILE:
+        this.client = new FilePlugin()
+        break
+      default:
+        throw new Error('Invalid storage type')
+    }
+  }
+
+  // Stores value to the selected storage type.
+  async store (object) {
+    if (this.storageType && this.client) {
+      return await this.client.store(object)
+    } else {
+      throw new Error('Storage type or client not set')
+    }
+  }
+
+  async retrieve (key) {
+    if (this.storageType && this.client) {
+      return await this.client.retrieve(key)
+    } else {
+      throw new Error('Storage type or client not set')
+    }
+  }
+}
+
+module.exports = { StorageBackend, STORAGE_TYPES }

package/lib/natural/util/storage/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3'
+
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_DB: naturaldb
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+    ports:
+      - "5432:5432"
+
+  mongodb:
+    image: mongo:latest
+    environment:
+      MONGO_INITDB_DATABASE: naturaldb
+    ports:
+      - "27017:27017"
+
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+
+  memcached:
+    image: memcached:latest
+    ports:
+      - "11211:11211"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.10.5",
+  "version": "6.11.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",
@@ -14,10 +14,16 @@
     "afinn-165": "^1.0.2",
     "afinn-165-financialmarketnews": "^3.0.0",
     "apparatus": "^0.0.10",
+    "dotenv": "^16.4.5",
+    "memjs": "^1.3.2",
+    "mongoose": "^8.2.0",
+    "pg": "^8.11.3",
+    "redis": "^4.6.13",
     "safe-stable-stringify": "^2.2.0",
     "stopwords-iso": "^1.1.0",
     "sylvester": "^0.0.12",
     "underscore": "^1.9.1",
+    "uuid": "^9.0.1",
     "wordnet-db": "^3.1.11"
   },
   "devDependencies": {