@signalk/streams 1.18.0 → 2.0.2

package/.prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "semi": false,
+  "singleQuote": true
+}

package/README.md

@@ -0,0 +1,14 @@
+# @signalk/streams
+
+Used within the [Signal K](http://signalk.org) [Node Server](https://github.com/SignalK/signalk-server-node) for streaming and converting input data in different formats.
+
+The code is not compiled and is not more effective, but allows using the streams and their conversions independently outside the server.
+
+## Development
+
+- Install dev packages with `npm i`.
+- Edit files with `/src`.
+- `npm link`
+- `cd ../../`
+- `npm link @signalk/streams`
+- Restart signalk `npm start`

package/autodetect.js

@@ -45,12 +45,12 @@ A => actisense-serial format N2K data
 1471172400153;A;2016-07-16T12:00:00.000Z,2,130306,105,255,8,00,d1,03,c9,23,fa,ff,ff
 */
 
-function DeMultiplexer (options) {
+function DeMultiplexer(options) {
   Writable.call(this)
 
   this.toTimestamped = new ToTimestamped(this, options)
   this.timestampThrottle = new TimestampThrottle({
-    getMilliseconds: msg => msg.timestamp
+    getMilliseconds: (msg) => msg.timestamp,
   })
   this.splitter = new Splitter(this, options)
   this.options = options
@@ -66,7 +66,7 @@ DeMultiplexer.prototype.write = function (chunk, encoding, callback) {
   return this.toTimestamped.write(chunk, encoding, callback)
 }
 
-function Splitter (deMultiplexer, options) {
+function Splitter(deMultiplexer, options) {
   Transform.call(this, { objectMode: true })
   this.demuxEmitData = function (msg) {
     deMultiplexer.emit('data', msg)
@@ -91,7 +91,7 @@ Splitter.prototype._transform = function (msg, encoding, _done) {
   let done = _done
   try {
     switch (msg.discriminator) {
-    case 'A': {
+      case 'A': {
         msg.fromFile = true
         const result = this.fromActisenseSerial.write(msg, encoding)
         if (!result) {
@@ -130,7 +130,7 @@ Splitter.prototype.pipe = function (target) {
   return Transform.prototype.pipe.call(this, target)
 }
 
-function ToTimestamped (deMultiplexer, options) {
+function ToTimestamped(deMultiplexer, options) {
   Transform.call(this, { objectMode: true })
   this.deMultiplexer = deMultiplexer
   this.options = options

package/canboatjs.js

@@ -16,21 +16,24 @@
 
 const Transform = require('stream').Transform
 const FromPgn = require('@canboat/canboatjs').FromPgn
-const debug = require('debug')('signalk:streams:canboatjs')
 const _ = require('lodash')
 
-function CanboatJs (options) {
+function CanboatJs(options) {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
 
   this.fromPgn = new FromPgn(options)
+  const createDebug = options.createDebug || require('debug')
+  const debug = createDebug('signalk:streams:nmea0183-signalk')
 
   this.fromPgn.on('warning', (pgn, warning) => {
     debug(`[warning] ${pgn.pgn} ${warning}`)
   })
 
-  this.fromPgn.on('error', (pgn, err) => {console.log(err)} )
+  this.fromPgn.on('error', (pgn, err) => {
+    console.log(err)
+  })
 
   this.app = options.app
 }
@@ -40,14 +43,14 @@ require('util').inherits(CanboatJs, Transform)
 CanboatJs.prototype._transform = function (chunk, encoding, done) {
   if (_.isObject(chunk) && chunk.fromFile) {
     const pgnData = this.fromPgn.parse(chunk.data)
-    if ( pgnData ) {
+    if (pgnData) {
       pgnData.timestamp = new Date(Number(chunk.timestamp)).toISOString()
       this.push(pgnData)
       this.app.emit('N2KAnalyzerOut', pgnData)
     }
   } else {
     const pgnData = this.fromPgn.parse(chunk)
-    if ( pgnData ) {
+    if (pgnData) {
       this.push(pgnData)
       this.app.emit('N2KAnalyzerOut', pgnData)
     }

package/execute.js

@@ -39,10 +39,11 @@
 const Transform = require('stream').Transform
 const { pgnToActisenseSerialFormat } = require('@canboat/canboatjs')
 
-function Execute (options) {
+function Execute(options) {
   Transform.call(this, {})
   this.options = options
-  this.debug = options.debug || require('debug')('signalk:streams:execute')
+  const createDebug = options.createDebug || require('debug')
+  this.debug = options.debug || createDebug('signalk:streams:execute')
 }
 
 require('util').inherits(Execute, Transform)
@@ -52,7 +53,7 @@ Execute.prototype._transform = function (chunk, encoding, done) {
   this.analyzerProcess.stdin.write(chunk.toString())
   done()
 }
-function start (command, that) {
+function start(command, that) {
   that.debug(`starting |${command}|`)
   if (process.platform === 'win32') {
     that.childProcess = require('child_process').spawn('cmd', ['/c', command])
@@ -75,7 +76,7 @@ function start (command, that) {
     that.push(data)
   })
 
-  that.childProcess.on('close', code => {
+  that.childProcess.on('close', (code) => {
     const msg = `|${command}| exited with ${code}`
     // that.options.app.setProviderError(that.options.providerId, msg)
     console.error(msg)
@@ -106,7 +107,9 @@ Execute.prototype.pipe = function (pipeTo) {
   start(this.options.command, this)
 
   const stdOutEvent = this.options.toChildProcess || 'toChildProcess'
-  this.debug('Using event ' + stdOutEvent + " for output to child process's stdin")
+  this.debug(
+    'Using event ' + stdOutEvent + " for output to child process's stdin"
+  )
   const that = this
   that.options.app.on(stdOutEvent, function (d) {
     try {
@@ -117,7 +120,7 @@ Execute.prototype.pipe = function (pipeTo) {
   })
 
   if (stdOutEvent === 'nmea2000out') {
-    that.options.app.on('nmea2000JsonOut', pgn => {
+    that.options.app.on('nmea2000JsonOut', (pgn) => {
       that.childProcess.stdin.write(pgnToActisenseSerialFormat(pgn) + '\r\n')
     })
     that.options.app.emit('nmea2000OutAvailable')

package/filestream.js

@@ -37,7 +37,7 @@ const path = require('path')
 const PassThrough = require('stream').PassThrough
 const fs = require('fs')
 
-function EndIgnoringPassThrough () {
+function EndIgnoringPassThrough() {
   PassThrough.call(this)
 }
 
@@ -72,7 +72,7 @@ FileStream.prototype.startStream = function () {
   }
 
   this.filestream = require('fs').createReadStream(filename)
-  this.filestream.on('error', err => {
+  this.filestream.on('error', (err) => {
     console.error(err.message)
     this.keepRunning = false
   })

package/folderstream.js

@@ -1,39 +1,36 @@
-const Transform = require("stream").Transform;
-const fs = require("fs");
+const Transform = require('stream').Transform
+const fs = require('fs')
 
 function FolderStreamProvider(folder) {
   Transform.call(this, {
     objectMode: false,
-  });
-  this.folder = folder;
-  this.fileIndex = 0;
+  })
+  this.folder = folder
+  this.fileIndex = 0
 }
 
-require("util").inherits(FolderStreamProvider, Transform);
+require('util').inherits(FolderStreamProvider, Transform)
 
-FolderStreamProvider.prototype.pipe = function(pipeTo) {
-  const files = fs.readdirSync(this.folder);
-  pipeNextFile.bind(this)();
+FolderStreamProvider.prototype.pipe = function (pipeTo) {
+  const files = fs.readdirSync(this.folder)
+  pipeNextFile.bind(this)()
 
   function pipeNextFile() {
     const fileStream = fs.createReadStream(
-      this.folder + "/" + files[this.fileIndex]
-    );
-    fileStream.pipe(
-      pipeTo,
-      { end: false }
-    );
-    fileStream.on("end", () => {
-      this.fileIndex++;
+      this.folder + '/' + files[this.fileIndex]
+    )
+    fileStream.pipe(pipeTo, { end: false })
+    fileStream.on('end', () => {
+      this.fileIndex++
       if (this.fileIndex === files.length) {
-        pipeTo.end();
+        pipeTo.end()
       } else {
-        pipeNextFile.bind(this)();
+        pipeNextFile.bind(this)()
       }
-    });
+    })
   }
 
-  return pipeTo;
-};
+  return pipeTo
+}
 
-module.exports = FolderStreamProvider;
+module.exports = FolderStreamProvider

package/from_json.js

@@ -27,9 +27,9 @@
 
 const Transform = require('stream').Transform
 
-function FromJson () {
+function FromJson() {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
 }
 

package/gpsd.js

@@ -32,35 +32,39 @@
 
 const Transform = require('stream').Transform
 const gpsd = require('node-gpsd')
-const debug = require('debug')('signalk:streams:gpsd')
 
-function Gpsd (options) {
+function Gpsd(options) {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
-  
+
   const port = options.port || 2947
   const hostname = options.hostname || options.host || 'localhost'
 
   function setProviderStatus(msg) {
     options.app.setProviderStatus(options.providerId, msg)
   }
-  
+
+  const createDebug = options.createDebug || require('debug')
+
   this.listener = new gpsd.Listener({
     port,
     hostname,
     logger: {
-      info: debug,
+      info: createDebug('signalk:streams:gpsd'),
       warn: console.warn,
       error: (msg) => {
-        options.app.setProviderError(options.providerId, `${hostname}:${port}: ` + msg)
-      }
+        options.app.setProviderError(
+          options.providerId,
+          `${hostname}:${port}: ` + msg
+        )
+      },
     },
-    parse: false
+    parse: false,
   })
 
   setProviderStatus(`Connecting to ${hostname}:${port}`)
-  
+
   this.listener.connect(function () {
     setProviderStatus(`Connected to ${hostname}:${port}`)
   })
@@ -72,7 +76,7 @@ function Gpsd (options) {
 
   this.listener.watch({
     class: 'WATCH',
-    nmea: true
+    nmea: true,
   })
 }
 

package/keys-filter.js

@@ -1,13 +1,14 @@
 'use strict'
 
 const Transform = require('stream').Transform
-const debug = require('debug')('signalk:streams:keys-filter')
 
-function ToSignalK (options) {
+function ToSignalK(options) {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
 
+  const createDebug = options.createDebug || require('debug')
+  this.debug = createDebug('signalk:streams:keys-filter')
   this.exclude = options.excludeMatchingPaths
 }
 
@@ -25,17 +26,17 @@ ToSignalK.prototype._transform = function (chunk, encoding, done) {
       delta = JSON.parse(chunk)
       string = true
     } catch (e) {
-      debug(`Error parsing chunk: ${e.message}`)
+      this.debug(`Error parsing chunk: ${e.message}`)
     }
   }
 
   if (Array.isArray(delta.updates)) {
     const updates = []
-    delta.updates.forEach(update => {
+    delta.updates.forEach((update) => {
       if (Array.isArray(update.values)) {
         const values = []
 
-        update.values.forEach(value => {
+        update.values.forEach((value) => {
           if (this.exclude.includes(value.path) !== true) {
             values.push(value)
           }
@@ -43,7 +44,7 @@ ToSignalK.prototype._transform = function (chunk, encoding, done) {
 
         if (values.length > 0) {
           const upd = {
-            values
+            values,
           }
 
           if (update.hasOwnProperty('$source')) {

package/liner.js

@@ -28,9 +28,9 @@ const Transform = require('stream').Transform
 
 require('util').inherits(Liner, Transform)
 
-function Liner (options) {
+function Liner(options) {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
   this.doPush = this.push.bind(this)
   this.lineSeparator = options.lineSeparator || '\n'

package/log.js

@@ -32,9 +32,9 @@
 const Transform = require('stream').Transform
 const getLogger = require('./logging').getLogger
 
-function Log (options) {
+function Log(options) {
   Transform.call(this, {
-    objectMode: true
+    objectMode: true,
   })
 
   this.logger = getLogger(options.app, options.discriminator, options.logdir)
@@ -48,7 +48,7 @@ Log.prototype._transform = function (msg, encoding, done) {
   done()
 }
 
-function pad (num) {
+function pad(num) {
   return (num > 9 ? '' : '0') + num
 }
 

package/logging.js

@@ -16,7 +16,7 @@
 
 const { FileTimestampStream } = require('file-timestamp-stream')
 const path = require('path')
-const debug = require('debug')('signalk:streams:logging')
+let debug = require('debug')('signalk:streams:logging')
 const fs = require('fs')
 
 const filenamePattern = /skserver\-raw\_\d\d\d\d\-\d\d\-\d\dT\d\d\.log/
@@ -25,10 +25,58 @@ const loggers = {}
 module.exports = {
   getLogger,
   getFullLogDir,
-  listLogFiles
+  listLogFiles,
 }
 
-function getLogger (app, discriminator = '', logdir) {
+class FileTimestampStreamWithDelete extends FileTimestampStream {
+  constructor(app, fullLogDir, filesToKeep, options) {
+    super(options)
+    this.app = app
+    this.filesToKeep = filesToKeep
+    this.fullLogDir = fullLogDir
+    this.prevFilename = undefined
+    debug = (options.createDebug || require('debug'))('signalk:streams:logging')
+  }
+
+  // This method of base class is called when new file name is contemplated
+  // So let's override it to check how many files are there and delete the oldest ones
+  newFilename() {
+    if (this.prevFilename !== this.currentFilename) {
+      // Only do that after new file created
+      this.prevFilename = this.currentFilename
+      this.deleteOldFiles()
+    }
+    return super.newFilename()
+  }
+
+  deleteOldFiles() {
+    debug(`Checking for old log files`)
+    listLogFiles(this.app, (err, files) => {
+      if (err) {
+        console.error(err)
+      } else {
+        if (files.length > this.filesToKeep) {
+          const sortedFiles = files.sort()
+          const numToDelete = files.length - this.filesToKeep
+          debug(`Will delete ${numToDelete} files`)
+          for (let i = 0; i < numToDelete; i++) {
+            const fileName = path.join(this.fullLogDir, sortedFiles[i])
+            debug(`Deleting ${fileName}`)
+            fs.unlink(fileName, (err) => {
+              if (err) {
+                console.error(err)
+              } else {
+                debug(`${fileName} was deleted`)
+              }
+            })
+          }
+        }
+      }
+    })
+  }
+}
+
+function getLogger(app, discriminator = '', logdir) {
   const fullLogdir = getFullLogDir(app, logdir)
 
   if (!loggers[fullLogdir]) {
@@ -36,16 +84,29 @@ function getLogger (app, discriminator = '', logdir) {
 
     debug(`logging to ${fileName}`)
 
-    loggers[fullLogdir] = new FileTimestampStream({
-      path: fileName
-    })
+    let fileTimestampStream
+    if (app.config.settings.keepMostRecentLogsOnly) {
+      // Delete old logs
+      fileTimestampStream = new FileTimestampStreamWithDelete(
+        app,
+        fullLogdir,
+        app.config.settings.logCountToKeep,
+        { path: fileName }
+      )
+    } else {
+      // Don't delete any logs
+      fileTimestampStream = new FileTimestampStream({ path: fileName })
+    }
+
+    loggers[fullLogdir] = fileTimestampStream
   }
+
   const logger = loggers[fullLogdir]
-  logger.on('error', err => {
+  logger.on('error', (err) => {
     console.error(`Error opening data logging file: ${err.message}`)
   })
 
-  return msg => {
+  return (msg) => {
     try {
       logger.write(
         Date.now() +
@@ -61,7 +122,7 @@ function getLogger (app, discriminator = '', logdir) {
   }
 }
 
-function getFullLogDir (app, logdir) {
+function getFullLogDir(app, logdir) {
   if (!logdir) {
     logdir = app.config.settings.loggingDirectory || app.config.configPath
   }
@@ -70,10 +131,13 @@ function getFullLogDir (app, logdir) {
     : path.join(app.config.configPath, logdir)
 }
 
-function listLogFiles (app, cb) {
+function listLogFiles(app, cb) {
   fs.readdir(getFullLogDir(app), (err, files) => {
     if (!err) {
-      cb(undefined, files.filter(filename => filename.match(filenamePattern)))
+      cb(
+        undefined,
+        files.filter((filename) => filename.match(filenamePattern))
+      )
     } else {
       cb(err)
     }