aedes 0.51.3 → 1.0.1

package/aedes.js

@@ -1,24 +1,21 @@
-'use strict'
-
-const EventEmitter = require('events')
-const util = require('util')
-const parallel = require('fastparallel')
-const series = require('fastseries')
-const { v4: uuidv4 } = require('uuid')
-const reusify = require('reusify')
-const { pipeline } = require('stream')
-const Packet = require('aedes-packet')
-const memory = require('aedes-persistence')
-const mqemitter = require('mqemitter')
-const Client = require('./lib/client')
-const { $SYS_PREFIX, bulk } = require('./lib/utils')
-
-module.exports = Aedes.createBroker = Aedes
+import EventEmitter from 'node:events'
+import parallel from 'fastparallel'
+import series from 'fastseries'
+import { v4 as uuidv4 } from 'uuid'
+import reusify from 'reusify'
+import { pipeline } from 'stream'
+import Packet from 'aedes-packet'
+import memory from 'aedes-persistence'
+import mqemitter from 'mqemitter'
+import Client from './lib/client.js'
+import { $SYS_PREFIX, bulk } from './lib/utils.js'
+import pkg from './package.json' with { type: 'json' }
 
 const defaultOptions = {
   concurrency: 100,
   heartbeatInterval: 60000, // 1 minute
   connectTimeout: 30000, // 30 secs
+  drainTimeout: 60000, // 60 secs - protects against slow/frozen clients by default, set to 0 to disable
   decodeProtocol: null,
   preConnect: defaultPreConnect,
   authenticate: defaultAuthenticate,
@@ -32,166 +29,255 @@ const defaultOptions = {
   maxClientsIdLength: 23,
   keepaliveLimit: 0
 }
+const version = pkg.version
+
+export class Aedes extends EventEmitter {
+  constructor (opts) {
+    super()
+    const that = this
+
+    opts = Object.assign({}, defaultOptions, opts)
+    this.opts = opts
+    this.id = opts.id || uuidv4()
+    // +1 when construct a new aedes-packet
+    // internal track for last brokerCounter
+    this.counter = 0
+    this.queueLimit = opts.queueLimit
+    this.connectTimeout = opts.connectTimeout
+    this.keepaliveLimit = opts.keepaliveLimit
+    this.maxClientsIdLength = opts.maxClientsIdLength
+    this.mq = opts.mq || mqemitter({
+      concurrency: opts.concurrency,
+      matchEmptyLevels: true // [MQTT-4.7.1-3]
+    })
+    this.handle = function handle (conn, req) {
+      conn.setMaxListeners(opts.concurrency * 2)
+      // create a new Client instance for a new connection
+      // return, just to please standard
+      return new Client(that, conn, req)
+    }
 
-function Aedes (opts) {
-  const that = this
+    this._parallel = parallel()
+    this._series = series()
+    this._enqueuers = reusify(DoEnqueues)
 
-  if (!(this instanceof Aedes)) {
-    return new Aedes(opts)
-  }
+    this.preConnect = opts.preConnect
+    this.authenticate = opts.authenticate
+    this.authorizePublish = opts.authorizePublish
+    this.authorizeSubscribe = opts.authorizeSubscribe
+    this.authorizeForward = opts.authorizeForward
+    this.published = opts.published
 
-  opts = Object.assign({}, defaultOptions, opts)
-
-  this.id = opts.id || uuidv4()
-  // +1 when construct a new aedes-packet
-  // internal track for last brokerCounter
-  this.counter = 0
-  this.queueLimit = opts.queueLimit
-  this.connectTimeout = opts.connectTimeout
-  this.keepaliveLimit = opts.keepaliveLimit
-  this.maxClientsIdLength = opts.maxClientsIdLength
-  this.mq = opts.mq || mqemitter({
-    concurrency: opts.concurrency,
-    matchEmptyLevels: true // [MQTT-4.7.1-3]
-  })
-  this.handle = function handle (conn, req) {
-    conn.setMaxListeners(opts.concurrency * 2)
-    // create a new Client instance for a new connection
-    // return, just to please standard
-    return new Client(that, conn, req)
-  }
-  this.persistence = opts.persistence || memory()
-  this.persistence.broker = this
-  this._parallel = parallel()
-  this._series = series()
-  this._enqueuers = reusify(DoEnqueues)
+    this.decodeProtocol = opts.decodeProtocol
+    this.trustProxy = opts.trustProxy
+    this.trustedProxies = opts.trustedProxies
 
-  this.preConnect = opts.preConnect
-  this.authenticate = opts.authenticate
-  this.authorizePublish = opts.authorizePublish
-  this.authorizeSubscribe = opts.authorizeSubscribe
-  this.authorizeForward = opts.authorizeForward
-  this.published = opts.published
+    this.clients = {}
+    this.brokers = {}
+    this.closed = true
+  }
 
-  this.decodeProtocol = opts.decodeProtocol
-  this.trustProxy = opts.trustProxy
-  this.trustedProxies = opts.trustedProxies
+  async listen () {
+    const opts = this.opts
+    const that = this
 
-  this.clients = {}
-  this.brokers = {}
+    // metadata
+    this.connectedClients = 0
+    this.closed = false
 
-  const heartbeatTopic = $SYS_PREFIX + that.id + '/heartbeat'
-  const birthTopic = $SYS_PREFIX + that.id + '/birth'
+    this.persistence = opts.persistence || memory()
+    if (this.persistence.setup.constructor.name !== 'AsyncFunction') {
+      throw new Error('persistence.setup() must be an async function')
+    }
+    await this.persistence.setup(this)
 
-  this._heartbeatInterval = setInterval(heartbeat, opts.heartbeatInterval)
+    const heartbeatTopic = $SYS_PREFIX + that.id + '/heartbeat'
+    const birthTopic = $SYS_PREFIX + that.id + '/birth'
 
-  const bufId = Buffer.from(that.id, 'utf8')
+    this._heartbeatInterval = setInterval(heartbeat, opts.heartbeatInterval)
 
-  // in a cluster env this is used to warn other broker instances
-  // that this broker is alive
-  that.publish({
-    topic: birthTopic,
-    payload: bufId
-  }, noop)
+    const bufId = Buffer.from(that.id, 'utf8')
 
-  function heartbeat () {
+    // in a cluster env this is used to warn other broker instances
+    // that this broker is alive
     that.publish({
-      topic: heartbeatTopic,
+      topic: birthTopic,
       payload: bufId
     }, noop)
-  }
 
-  function deleteOldBrokers (broker) {
-    if (that.brokers[broker] + (3 * opts.heartbeatInterval) < Date.now()) {
-      delete that.brokers[broker]
+    function heartbeat () {
+      that.publish({
+        topic: heartbeatTopic,
+        payload: bufId
+      }, noop)
+    }
+
+    function deleteOldBrokers (broker) {
+      if (that.brokers[broker] + (3 * opts.heartbeatInterval) < Date.now()) {
+        delete that.brokers[broker]
+      }
     }
-  }
 
-  this._clearWillInterval = setInterval(function () {
-    Object.keys(that.brokers).forEach(deleteOldBrokers)
+    this._clearWillInterval = setInterval(function () {
+      Object.keys(that.brokers).forEach(deleteOldBrokers)
 
-    pipeline(
-      that.persistence.streamWill(that.brokers),
-      bulk(receiveWills),
-      function done (err) {
-        if (err) {
-          that.emit('error', err)
+      pipeline(
+        that.persistence.streamWill(that.brokers),
+        bulk(receiveWills),
+        function done (err) {
+          if (err) {
+            that.emit('error', err)
+          }
         }
-      }
-    )
-  }, opts.heartbeatInterval * 4)
+      )
+    }, opts.heartbeatInterval * 4)
 
-  function receiveWills (chunks, done) {
-    that._parallel(that, checkAndPublish, chunks, done)
-  }
+    function receiveWills (chunks, done) {
+      that._parallel(that, checkAndPublish, chunks, done)
+    }
+
+    function checkAndPublish (will, done) {
+      const notPublish = that.brokers[will.brokerId] !== undefined && that.brokers[will.brokerId] + (3 * opts.heartbeatInterval) >= Date.now()
 
-  function checkAndPublish (will, done) {
-    const notPublish =
-      that.brokers[will.brokerId] !== undefined && that.brokers[will.brokerId] + (3 * opts.heartbeatInterval) >= Date.now()
+      if (notPublish) return done()
 
-    if (notPublish) return done()
+      // randomize this, so that multiple brokers
+      // do not publish the same wills at the same time
+      this.authorizePublish(that.clients[will.clientId] || null, will, function (err) {
+        if (err) { return doneWill() }
+        that.publish(will, doneWill)
 
-    // randomize this, so that multiple brokers
-    // do not publish the same wills at the same time
-    this.authorizePublish(that.clients[will.clientId] || null, will, function (err) {
-      if (err) { return doneWill() }
-      that.publish(will, doneWill)
+        function doneWill (err) {
+          if (err) { return done(err) }
+          that.persistence.delWill({
+            id: will.clientId,
+            brokerId: will.brokerId
+          }).then(will => done(undefined, will), done)
+        }
+      })
+    }
 
-      function doneWill (err) {
-        if (err) { return done(err) }
-        that.persistence.delWill({
-          id: will.clientId,
-          brokerId: will.brokerId
-        }, done)
+    this.mq.on($SYS_PREFIX + '+/heartbeat', function storeBroker (packet, done) {
+      that.brokers[packet.payload.toString()] = Date.now()
+      done()
+    })
+
+    this.mq.on($SYS_PREFIX + '+/birth', function brokerBorn (packet, done) {
+      const brokerId = packet.payload.toString()
+
+      // reset duplicates counter
+      if (brokerId !== that.id) {
+        for (const clientId in that.clients) {
+          delete that.clients[clientId].duplicates[brokerId]
+        }
+      }
+
+      done()
+    })
+
+    this.mq.on($SYS_PREFIX + '+/new/clients', function closeSameClients (packet, done) {
+      const serverId = packet.topic.split('/')[1]
+      const clientId = packet.payload.toString()
+
+      if (that.clients[clientId] && serverId !== that.id) {
+        if (that.clients[clientId].closed) {
+          // remove the client from the list if it is already closed
+          that.deleteClient(clientId)
+          done()
+        } else {
+          that.clients[clientId].close(done)
+        }
+      } else {
+        done()
       }
     })
   }
 
-  this.mq.on($SYS_PREFIX + '+/heartbeat', function storeBroker (packet, done) {
-    that.brokers[packet.payload.toString()] = Date.now()
-    done()
-  })
+  get version () {
+    return version
+  }
 
-  this.mq.on($SYS_PREFIX + '+/birth', function brokerBorn (packet, done) {
-    const brokerId = packet.payload.toString()
+  static async createBroker (opts) {
+    const aedes = new Aedes(opts)
+    await aedes.listen()
+    return aedes
+  }
 
-    // reset duplicates counter
-    if (brokerId !== that.id) {
-      for (const clientId in that.clients) {
-        delete that.clients[clientId].duplicates[brokerId]
-      }
+  publish (packet, client, done) {
+    if (typeof client === 'function') {
+      done = client
+      client = null
     }
+    const p = new Packet(packet, this)
+    const publishFuncs = p.qos > 0 ? publishFuncsQoS : publishFuncsSimple
 
-    done()
-  })
+    this._series(new PublishState(this, client, packet), publishFuncs, p, done)
+  }
 
-  this.mq.on($SYS_PREFIX + '+/new/clients', function closeSameClients (packet, done) {
-    const serverId = packet.topic.split('/')[1]
-    const clientId = packet.payload.toString()
+  subscribe (topic, func, done) {
+    this.mq.on(topic, func, done)
+  }
 
-    if (that.clients[clientId] && serverId !== that.id) {
-      if (that.clients[clientId].closed) {
-        // remove the client from the list if it is already closed
-        that.deleteClient(clientId)
-        done()
-      } else {
-        that.clients[clientId].close(done)
-      }
+  unsubscribe (topic, func, done) {
+    this.mq.removeListener(topic, func, done)
+  }
+
+  registerClient (client) {
+    const that = this
+    if (this.clients[client.id]) {
+      // [MQTT-3.1.4-2]
+      this.clients[client.id].close(function closeClient () {
+        that._finishRegisterClient(client)
+      })
     } else {
-      done()
+      this._finishRegisterClient(client)
     }
-  })
+  }
 
-  // metadata
-  this.connectedClients = 0
-  this.closed = false
-}
+  _finishRegisterClient (client) {
+    this.connectedClients++
+    this.clients[client.id] = client
+    this.emit('client', client)
+    this.publish({
+      topic: $SYS_PREFIX + this.id + '/new/clients',
+      payload: Buffer.from(client.id, 'utf8')
+    }, noop)
+  }
+
+  unregisterClient (client) {
+    this.deleteClient(client.id)
+    this.emit('clientDisconnect', client)
+    this.publish({
+      topic: $SYS_PREFIX + this.id + '/disconnect/clients',
+      payload: Buffer.from(client.id, 'utf8')
+    }, noop)
+  }
 
-util.inherits(Aedes, EventEmitter)
+  deleteClient (clientId) {
+    this.connectedClients--
+    delete this.clients[clientId]
+  }
+
+  close (cb = noop) {
+    const that = this
+    if (this.closed) {
+      return cb()
+    }
+    this.closed = true
+    clearInterval(this._heartbeatInterval)
+    clearInterval(this._clearWillInterval)
+    this._parallel(this, closeClient, Object.keys(this.clients), doneClose)
+    function doneClose () {
+      that.emit('closed')
+      that.mq.close(cb)
+    }
+  }
+}
 
 function storeRetained (packet, done) {
   if (packet.retain) {
-    this.broker.persistence.storeRetained(packet, done)
+    this.broker.persistence.storeRetained(packet)
+      .then(() => done(null), done)
   } else {
     done()
   }
@@ -209,45 +295,46 @@ function enqueueOffline (packet, done) {
   enqueuer.packet = packet
   enqueuer.topic = packet.topic
   enqueuer.broker = this.broker
-  this.broker.persistence.subscriptionsByTopic(
-    packet.topic,
-    enqueuer.done
-  )
+  this.broker.persistence.subscriptionsByTopic(packet.topic)
+    .then(subs => enqueuer.done(null, subs), enqueuer.done)
 }
 
-function DoEnqueues () {
-  this.next = null
-  this.complete = null
-  this.packet = null
-  this.topic = null
-  this.broker = null
+class DoEnqueues {
+  constructor () {
+    this.next = null
+    this.complete = null
+    this.packet = null
+    this.topic = null
+    this.broker = null
+
+    const that = this
+
+    this.done = function doneEnqueue (err, subs) {
+      const broker = that.broker
+
+      if (err) {
+        // is this really recoverable?
+        // let's just error the whole aedes
+        // https://nodejs.org/api/events.html#events_error_events
+        broker.emit('error', err)
+        return
+      }
 
-  const that = this
+      if (that.topic.indexOf($SYS_PREFIX) === 0) {
+        subs = subs.filter(removeSharp)
+      }
 
-  this.done = function doneEnqueue (err, subs) {
-    const broker = that.broker
+      const packet = that.packet
+      const complete = that.complete
 
-    if (err) {
-      // is this really recoverable?
-      // let's just error the whole aedes
-      // https://nodejs.org/api/events.html#events_error_events
-      broker.emit('error', err)
-      return
-    }
+      that.packet = null
+      that.complete = null
+      that.topic = null
 
-    if (that.topic.indexOf($SYS_PREFIX) === 0) {
-      subs = subs.filter(removeSharp)
+      broker.persistence.outgoingEnqueueCombi(subs, packet)
+        .then(() => complete(null), complete)
+      broker._enqueuers.release(that)
     }
-
-    const packet = that.packet
-    const complete = that.complete
-
-    that.packet = null
-    that.complete = null
-    that.topic = null
-
-    broker.persistence.outgoingEnqueueCombi(subs, packet, complete)
-    broker._enqueuers.release(that)
   }
 }
 
@@ -274,82 +361,11 @@ const publishFuncsQoS = [
   emitPacket,
   callPublished
 ]
-Aedes.prototype.publish = function (packet, client, done) {
-  if (typeof client === 'function') {
-    done = client
-    client = null
-  }
-  const p = new Packet(packet, this)
-  const publishFuncs = p.qos > 0 ? publishFuncsQoS : publishFuncsSimple
-
-  this._series(new PublishState(this, client, packet), publishFuncs, p, done)
-}
-
-Aedes.prototype.subscribe = function (topic, func, done) {
-  this.mq.on(topic, func, done)
-}
-
-Aedes.prototype.unsubscribe = function (topic, func, done) {
-  this.mq.removeListener(topic, func, done)
-}
-
-Aedes.prototype.registerClient = function (client) {
-  const that = this
-  if (this.clients[client.id]) {
-    // [MQTT-3.1.4-2]
-    this.clients[client.id].close(function closeClient () {
-      that._finishRegisterClient(client)
-    })
-  } else {
-    this._finishRegisterClient(client)
-  }
-}
-
-Aedes.prototype._finishRegisterClient = function (client) {
-  this.connectedClients++
-  this.clients[client.id] = client
-  this.emit('client', client)
-  this.publish({
-    topic: $SYS_PREFIX + this.id + '/new/clients',
-    payload: Buffer.from(client.id, 'utf8')
-  }, noop)
-}
-
-Aedes.prototype.unregisterClient = function (client) {
-  this.deleteClient(client.id)
-  this.emit('clientDisconnect', client)
-  this.publish({
-    topic: $SYS_PREFIX + this.id + '/disconnect/clients',
-    payload: Buffer.from(client.id, 'utf8')
-  }, noop)
-}
-
-Aedes.prototype.deleteClient = function (clientId) {
-  this.connectedClients--
-  delete this.clients[clientId]
-}
 
 function closeClient (client, cb) {
   this.clients[client].close(cb)
 }
 
-Aedes.prototype.close = function (cb = noop) {
-  const that = this
-  if (this.closed) {
-    return cb()
-  }
-  this.closed = true
-  clearInterval(this._heartbeatInterval)
-  clearInterval(this._clearWillInterval)
-  this._parallel(this, closeClient, Object.keys(this.clients), doneClose)
-  function doneClose () {
-    that.emit('closed')
-    that.mq.close(cb)
-  }
-}
-
-Aedes.prototype.version = require('./package.json').version
-
 function defaultPreConnect (client, packet, callback) {
   callback(null, true)
 }
@@ -377,10 +393,22 @@ function defaultPublished (packet, client, callback) {
   callback(null)
 }
 
-function PublishState (broker, client, packet) {
-  this.broker = broker
-  this.client = client
-  this.packet = packet
+class PublishState {
+  constructor (broker, client, packet) {
+    this.broker = broker
+    this.client = client
+    this.packet = packet
+  }
 }
 
 function noop () {}
+
+function warnMigrate () {
+  throw new Error(
+` Aedes default export has been removed.
+ Use 'const aedes = await Aedes.createBroker()' instead.
+ See: https://github.com/moscajs/aedes/docs/MIGRATION.MD
+ `)
+}
+
+export default warnMigrate

package/benchmarks/README.md

@@ -0,0 +1,33 @@
+# Benchmarks
+
+This folder contains a number of scripts to perform benchmark testing and
+reporting on it:
+
+- sender.js publishes messages with a serialnumber as payload (add '-q 1' to get QoS1)
+- receiver.js subscribes and validates the serialnumber (add '-q 1' to get QoS1)
+- pingpong.js measures latency between sending and receiving the same
+message
+- server.js starts the Aedes server to use in the test
+- runBenchmarks.js starts the server and runs Publish/Subscribe tests
+with QoS0 and QoS1 using the scripts above it produces CSV data, this
+ data includes the current git branch name.
+- report.js reads the CSV data from STDIN and turns it into a Markdown
+report.
+
+Examples:
+
+```bash
+node runBenchmark.js > result.csv
+cat result.csv | node report.csv > result.md
+```
+
+```bash
+node runBenchmark.js > result.branch.csv
+cat result.main.csv result.branch.csv | node report.csv > result.combined.md
+```
+
+## WARNING
+
+Running benchmarks and especially interpreting results can be misleading
+E.g. performance of the benchmark run might depend on the presence (or absence)
+ of other, unrelated, activity in the system.