aedes 0.46.3 → 0.48.0

package/.github/workflows/ci.yml

@@ -11,21 +11,39 @@ on:
       - '*.md'
 
 jobs:
+  dependency-review:
+    name: Dependency Review
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+        - name: Check out repo
+          uses: actions/checkout@v3
+          with:
+            persist-credentials: false
+
+        - name: Dependency review
+          uses: actions/dependency-review-action@v2
+
   test:
     runs-on: ${{ matrix.os }}
-
+    permissions:
+      contents: read
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14, 16, '*']
         os: [ubuntu-latest, windows-latest, macOS-latest]
-
     steps:
-      - uses: actions/checkout@v2.4.0
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
 
       - name: Use Node.js
-        uses: actions/setup-node@v2.5.1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
+          check-latest: true
 
       - name: Install
         run: |
@@ -49,6 +67,8 @@ jobs:
   coverage:
     needs: test
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - name: Coveralls Finished
         uses: coverallsapp/github-action@master

package/.github/workflows/sast.yml

@@ -0,0 +1,29 @@
+name: sast
+
+on:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+  pull_request:
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: true
+      matrix:
+        language: [ 'javascript' ]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      - uses: github/codeql-action/analyze@v2

package/README.md

@@ -24,7 +24,7 @@ Barebone MQTT server that can run on any stream servers
   - [Features](#features)
   - [Examples](#examples)
   - [Clusters](#clusters)
-  - [Exensions](#exensions)
+  - [Extensions](#extensions)
   - [Middleware Plugins](#middleware-plugins)
     - [Persistence](#persistence)
     - [MQEmitter](#mqemitter)
@@ -107,7 +107,7 @@ Brokers that support the [Bridge Protocol][bridge_protocol] can connect to
 Aedes.  When connecting with this special protocol, subscriptions work as usual
 except that the `retain` flag in the packet is propagated as-is.
 
-## Exensions
+## Extensions
 
 - [aedes-logging]: Logging module for Aedes, based on Pino
 - [aedes-stats]: Stats for Aedes

package/aedes.d.ts

@@ -1,9 +1,3 @@
-import { Aedes, AedesOptions } from './types/instance'
-
-export declare function aedes (options?: AedesOptions): Aedes
-export declare function Server (options?: AedesOptions): Aedes
-
-export * from './types/instance'
-export * from './types/packet'
-export * from './types/client'
-export default aedes
+/// <reference path="./types/instance.d.ts" />
+/// <reference path="./types/client.d.ts" />
+/// <reference path="./types/packet.d.ts" />

package/aedes.js

@@ -5,14 +5,13 @@ const util = require('util')
 const parallel = require('fastparallel')
 const series = require('fastseries')
 const { v4: uuidv4 } = require('uuid')
-const bulk = require('bulk-write-stream')
 const reusify = require('reusify')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const Packet = require('aedes-packet')
 const memory = require('aedes-persistence')
 const mqemitter = require('mqemitter')
 const Client = require('./lib/client')
-const { $SYS_PREFIX } = require('./lib/utils')
+const { $SYS_PREFIX, bulk } = require('./lib/utils')
 
 module.exports = Aedes.Server = Aedes
 
@@ -80,10 +79,19 @@ function Aedes (opts) {
   this.brokers = {}
 
   const heartbeatTopic = $SYS_PREFIX + that.id + '/heartbeat'
+  const birthTopic = $SYS_PREFIX + that.id + '/birth'
+
   this._heartbeatInterval = setInterval(heartbeat, opts.heartbeatInterval)
 
   const bufId = Buffer.from(that.id, 'utf8')
 
+  // in a cluster env this is used to warn other broker instances
+  // that this broker is alive
+  that.publish({
+    topic: birthTopic,
+    payload: bufId
+  }, noop)
+
   function heartbeat () {
     that.publish({
       topic: heartbeatTopic,
@@ -102,7 +110,7 @@ function Aedes (opts) {
 
     pipeline(
       that.persistence.streamWill(that.brokers),
-      bulk.obj(receiveWills),
+      bulk(receiveWills),
       function done (err) {
         if (err) {
           that.emit('error', err)
@@ -116,27 +124,25 @@ function Aedes (opts) {
   }
 
   function checkAndPublish (will, done) {
-    const needsPublishing =
-      !that.brokers[will.brokerId] ||
-      that.brokers[will.brokerId] + (3 * opts.heartbeatInterval) <
-      Date.now()
-
-    if (needsPublishing) {
-      // randomize this, so that multiple brokers
-      // do not publish the same wills at the same time
-      that.publish(will, function publishWill (err) {
-        if (err) {
-          return done(err)
-        }
+    const notPublish =
+      that.brokers[will.brokerId] !== undefined && that.brokers[will.brokerId] + (3 * opts.heartbeatInterval) >= Date.now()
 
+    if (notPublish) return done()
+
+    // randomize this, so that multiple brokers
+    // do not publish the same wills at the same time
+    this.authorizePublish(that.clients[will.clientId] || null, will, function (err) {
+      if (err) { return doneWill() }
+      that.publish(will, doneWill)
+
+      function doneWill (err) {
+        if (err) { return done(err) }
         that.persistence.delWill({
           id: will.clientId,
           brokerId: will.brokerId
         }, done)
-      })
-    } else {
-      done()
-    }
+      }
+    })
   }
 
   this.mq.on($SYS_PREFIX + '+/heartbeat', function storeBroker (packet, done) {
@@ -144,6 +150,19 @@ function Aedes (opts) {
     done()
   })
 
+  this.mq.on($SYS_PREFIX + '+/birth', function brokerBorn (packet, done) {
+    const brokerId = packet.payload.toString()
+
+    // reset duplicates counter
+    if (brokerId !== that.id) {
+      for (const clientId in that.clients) {
+        delete that.clients[clientId].duplicates[brokerId]
+      }
+    }
+
+    done()
+  })
+
   this.mq.on($SYS_PREFIX + '+/new/clients', function closeSameClients (packet, done) {
     const serverId = packet.topic.split('/')[1]
     const clientId = packet.payload.toString()
@@ -171,6 +190,7 @@ function storeRetained (packet, done) {
 }
 
 function emitPacket (packet, done) {
+  if (this.client) packet.clientId = this.client.id
   this.broker.mq.emit(packet, done)
 }
 

package/docs/Aedes.md

@@ -186,7 +186,7 @@ const server = require('net').createServer(aedes.handle)
 
 Directly subscribe a `topic` in server side. Bypass [`authorizeSubscribe`](#handler-authorizesubscribe-client-subscription-callback)
 
-The `topic` and `deliverfunc` is a compound key to differentiate the uniqueness of its subscription pool. `topic` could be the one that is existed, in this case `deliverfunc` will be invoked as well as `SUBSCRIBE` does.
+The `topic` and `deliverfunc` is a compound key to differentiate the uniqueness of its subscription pool. `topic` could be the one that is existed, in this case `deliverfunc` will be invoked as well as [`SUBSCRIBE`][SUBSCRIBE] does.
 
 `deliverfunc` supports backpressue.
 
@@ -291,7 +291,7 @@ Please refer to [Connect Return Code](http://docs.oasis-open.org/mqtt/mqtt/v3.1.
 
 ## Handler: authorizePublish (client, packet, callback)
 
-- client: [`<Client>`](./Client.md)
+- client: [`<Client>`](./Client.md) | `null`
 - packet: `<object>` [`PUBLISH`][PUBLISH]
 - callback: `<Function>` `(error) => void`
   - error `<Error>` | `null`
@@ -301,6 +301,8 @@ Invoked when
 1. publish LWT to all online clients
 2. incoming client publish
 
+`client` is `null` when aedes publishes obsolete LWT without connected clients
+
 If invoked `callback` with no errors, server authorizes the packet otherwise emits `clientError` with `error`. If an `error` occurs the client connection will be closed, but no error is returned to the client (MQTT-3.3.5-2)
 
 ```js
@@ -337,7 +339,7 @@ function defaultAuthorizePublish (client, packet, callback) {
 Invoked when
 
 1. restore subscriptions in non-clean session.
-2. incoming client `SUBSCRIBE`
+2. incoming client [`SUBSCRIBE`][SUBSCRIBE]
 
 `subscription` is a dictionary object like `{ topic: hello, qos: 0 }`.
 

package/lib/client.js

@@ -10,7 +10,7 @@ const QoSPacket = require('./qos-packet')
 const handleSubscribe = require('./handlers/subscribe')
 const handleUnsubscribe = require('./handlers/unsubscribe')
 const handle = require('./handlers')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const { through } = require('./utils')
 
 module.exports = Client
@@ -89,9 +89,20 @@ function Client (broker, conn, req) {
   conn.on('end', this.close.bind(this))
   this._eos = eos(this.conn, this.close.bind(this))
 
-  this.deliver0 = function deliverQoS0 (_packet, cb) {
+  const getToForwardPacket = (_packet) => {
+    // Mqttv5 3.8.3.1: https://docs.oasis-open.org/mqtt/mqtt/v5.0/mqtt-v5.0.html#_Toc3901169
+    // prevent to forward messages sent by the same client when no-local flag is set
+    if (_packet.clientId === that.id && _packet.nl) return
+
     const toForward = dedupe(that, _packet) &&
       that.broker.authorizeForward(that, _packet)
+
+    return toForward
+  }
+
+  this.deliver0 = function deliverQoS0 (_packet, cb) {
+    const toForward = getToForwardPacket(_packet)
+
     if (toForward) {
       // Give nodejs some time to clear stacks, or we will see
       // "Maximum call stack size exceeded" in a very high load
@@ -114,8 +125,8 @@ function Client (broker, conn, req) {
       that.deliver0(_packet, cb)
       return
     }
-    const toForward = dedupe(that, _packet) &&
-      that.broker.authorizeForward(that, _packet)
+    const toForward = getToForwardPacket(_packet)
+
     if (toForward) {
       setImmediate(() => {
         const packet = new QoSPacket(toForward, that)

package/lib/handlers/connect.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const retimer = require('retimer')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const write = require('../write')
 const QoSPacket = require('../qos-packet')
 const { through } = require('../utils')
@@ -70,7 +70,7 @@ function init (client, packet, done) {
     const error = new Error(errorMessages[returnCode])
     error.errorCode = returnCode
     doConnack(
-      { client: client, returnCode: returnCode, sessionPresent: false },
+      { client, returnCode, sessionPresent: false },
       done.bind(this, error))
     return
   }
@@ -156,8 +156,8 @@ function fetchSubs (arg, done) {
   if (!this.packet.clean) {
     client.broker.persistence.subscriptionsByClient({
       id: client.id,
-      done: done,
-      arg: arg
+      done,
+      arg
     }, gotSubs)
     return
   }

package/lib/handlers/subscribe.js

@@ -142,13 +142,12 @@ function addSubs (sub, done) {
   const nl = this.nl
   let func = qos > 0 ? client.deliverQoS : client.deliver0
 
-  if (!rap) {
-    const deliverFunc = func
-    func = function handlePacketSubscription (_packet, cb) {
-      _packet = new Packet(_packet, broker)
-      _packet.retain = false
-      deliverFunc(_packet, cb)
-    }
+  const deliverFunc = func
+  func = function handlePacketSubscription (_packet, cb) {
+    _packet = new Packet(_packet, broker)
+    _packet.nl = nl
+    if (!rap) _packet.retain = false
+    deliverFunc(_packet, cb)
   }
 
   // [MQTT-4.7.2-1]
@@ -218,7 +217,7 @@ function completeSubscribe (err) {
     topic: $SYS_PREFIX + broker.id + '/new/subscribes',
     payload: Buffer.from(JSON.stringify({
       clientId: client.id,
-      subs: subs
+      subs
     }), 'utf8')
   }, noop)
 

package/lib/utils.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Transform } = require('readable-stream')
+const { Transform, Writable } = require('stream')
 
 function validateTopic (topic, message) {
   const end = topic.length - 1
@@ -37,8 +37,18 @@ function through (transform) {
   })
 }
 
+function bulk (fn) {
+  return new Writable({
+    objectMode: true,
+    writev: function (chunks, cb) {
+      fn(chunks.map(chunk => chunk.chunk), cb)
+    }
+  })
+}
+
 module.exports = {
   validateTopic,
   through,
+  bulk,
   $SYS_PREFIX: '$SYS/'
 }

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "aedes",
-  "version": "0.46.3",
+  "version": "0.48.0",
   "description": "Stream-based MQTT broker",
   "main": "aedes.js",
   "types": "aedes.d.ts",
   "scripts": {
     "lint": "npm run lint:standard && npm run lint:typescript && npm run lint:markdown",
-    "lint:fix": "standard --fix",
+    "lint:fix": "standard --fix && eslint -c types/.eslintrc.json --fix aedes.d.ts types/**/*.ts test/types/**/*.test-d.ts",
     "lint:standard": "standard --verbose | snazzy",
     "lint:typescript": "eslint -c types/.eslintrc.json aedes.d.ts types/**/*.ts test/types/**/*.test-d.ts",
     "lint:markdown": "markdownlint docs/*.md README.md",
@@ -94,43 +94,41 @@
   },
   "homepage": "https://github.com/moscajs/aedes#readme",
   "engines": {
-    "node": ">=12"
+    "node": ">=14"
   },
   "devDependencies": {
-    "@sinonjs/fake-timers": "^9.1.0",
-    "@types/node": "^17.0.15",
-    "@typescript-eslint/eslint-plugin": "^5.10.2",
-    "@typescript-eslint/parser": "^5.10.2",
+    "@sinonjs/fake-timers": "^9.1.2",
+    "@types/node": "^18.6.1",
+    "@typescript-eslint/eslint-plugin": "^5.31.0",
+    "@typescript-eslint/parser": "^5.31.0",
     "concat-stream": "^2.0.0",
     "duplexify": "^4.1.2",
     "license-checker": "^25.0.1",
-    "markdownlint-cli": "^0.31.0",
-    "mqtt": "^4.3.4",
+    "markdownlint-cli": "^0.32.1",
+    "mqtt": "^4.3.7",
     "mqtt-connection": "^4.1.0",
     "pre-commit": "^1.2.2",
     "proxyquire": "^2.1.3",
-    "release-it": "^14.12.4",
+    "release-it": "^15.2.0",
     "snazzy": "^9.0.0",
-    "standard": "^16.0.4",
-    "tap": "^15.1.6",
-    "tsd": "^0.19.1",
-    "typescript": "^4.5.5",
+    "standard": "^17.0.0",
+    "tap": "^16.3.0",
+    "tsd": "^0.23.0",
+    "typescript": "^4.7.4",
     "websocket-stream": "^5.5.2"
   },
   "dependencies": {
-    "aedes-packet": "^2.3.1",
-    "aedes-persistence": "^8.1.3",
-    "bulk-write-stream": "^2.0.1",
+    "aedes-packet": "^3.0.0",
+    "aedes-persistence": "^9.1.2",
     "end-of-stream": "^1.4.4",
     "fastfall": "^1.5.1",
     "fastparallel": "^2.4.1",
     "fastseries": "^2.0.0",
-    "hyperid": "^3.0.0",
-    "mqemitter": "^4.5.0",
-    "mqtt-packet": "^7.1.2",
-    "readable-stream": "^3.6.0",
+    "hyperid": "^3.0.1",
+    "mqemitter": "^5.0.0",
+    "mqtt-packet": "^8.1.1",
     "retimer": "^3.0.0",
     "reusify": "^1.0.4",
-    "uuid": "^8.3.2"
+    "uuid": "^9.0.0"
   }
 }

package/test/auth.js

@@ -422,7 +422,7 @@ test('authentication error when non numeric return code is passed', function (t)
 test('authorize publish', function (t) {
   t.plan(4)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -445,6 +445,7 @@ test('authorize publish', function (t) {
     t.notOk(Object.prototype.hasOwnProperty.call(packet, 'messageId'), 'should not contain messageId in QoS 0')
     expected.brokerId = s.broker.id
     expected.brokerCounter = s.broker.counter
+    expected.clientId = 'my-client-xyz'
     delete expected.length
     t.same(packet, expected, 'packet matches')
     cb()
@@ -460,7 +461,7 @@ test('authorize publish', function (t) {
 test('authorize waits for authenticate', function (t) {
   t.plan(6)
 
-  const s = setup()
+  const s = setup(aedes({ clientId: 'my-client-xyz-2' }))
   t.teardown(s.broker.close.bind(s.broker))
 
   s.broker.authenticate = function (client, username, password, cb) {
@@ -485,7 +486,8 @@ test('authorize waits for authenticate', function (t) {
     qos: 0,
     retain: false,
     length: 12,
-    dup: false
+    dup: false,
+    clientId: 'my-client'
   }
 
   s.broker.mq.on('hello', function (packet, cb) {
@@ -519,12 +521,13 @@ test('authorize publish from configOptions', function (t) {
   t.plan(4)
 
   const s = connect(setup(aedes({
+    clientId: 'my-client-xyz-3',
     authorizePublish: function (client, packet, cb) {
       t.ok(client, 'client exists')
       t.same(packet, expected, 'packet matches')
       cb()
     }
-  })))
+  })), { clientId: 'my-client-xyz-3' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -541,6 +544,7 @@ test('authorize publish from configOptions', function (t) {
     t.notOk(Object.prototype.hasOwnProperty.call(packet, 'messageId'), 'should not contain messageId in QoS 0')
     expected.brokerId = s.broker.id
     expected.brokerCounter = s.broker.counter
+    expected.clientId = 'my-client-xyz-3'
     delete expected.length
     t.same(packet, expected, 'packet matches')
     cb()
@@ -589,7 +593,7 @@ test('do not authorize publish', function (t) {
 test('modify qos out of range in authorize publish ', function (t) {
   t.plan(2)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz-4' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -599,7 +603,8 @@ test('modify qos out of range in authorize publish ', function (t) {
     qos: 0,
     retain: false,
     length: 12,
-    dup: false
+    dup: false,
+    clientId: 'my-client-xyz-4'
   }
 
   s.broker.authorizePublish = function (client, packet, cb) {
@@ -805,12 +810,19 @@ test('negate multiple subscriptions', function (t) {
 test('negate subscription with correct persistence', function (t) {
   t.plan(6)
 
+  // rh, rap, nl are undefined because mqtt.parser is set to MQTT 3.1.1 and will thus erase these props from s.inStream.write
   const expected = [{
     topic: 'hello',
-    qos: 0
+    qos: 0,
+    rh: undefined,
+    rap: undefined,
+    nl: undefined
   }, {
     topic: 'world',
-    qos: 0
+    qos: 0,
+    rh: undefined,
+    rap: undefined,
+    nl: undefined
   }]
 
   const broker = aedes()
@@ -839,10 +851,16 @@ test('negate subscription with correct persistence', function (t) {
     messageId: 24,
     subscriptions: [{
       topic: 'hello',
-      qos: 0
+      qos: 0,
+      rh: 0,
+      rap: true,
+      nl: false
     }, {
       topic: 'world',
-      qos: 0
+      qos: 0,
+      rh: 0,
+      rap: true,
+      nl: false
     }]
   })
 })

package/test/basic.js

@@ -20,7 +20,7 @@ test('test aedes.Server', function (t) {
 test('publish QoS 0', function (t) {
   t.plan(2)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz-5' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -29,7 +29,8 @@ test('publish QoS 0', function (t) {
     payload: Buffer.from('world'),
     qos: 0,
     retain: false,
-    dup: false
+    dup: false,
+    clientId: 'my-client-xyz-5'
   }
 
   s.broker.mq.on('hello', function (packet, cb) {
@@ -128,7 +129,7 @@ test('publish to $SYS topic throws error', function (t) {
       qos: 0,
       retain: false
     }
-    const expectedSubs = ele.clean ? null : [{ topic: 'hello', qos: ele.qos }]
+    const expectedSubs = ele.clean ? null : [{ topic: 'hello', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined }]
 
     subscribe(t, s, 'hello', ele.qos, function () {
       s.outStream.once('data', function (packet) {
@@ -188,7 +189,10 @@ test('return write errors to callback', function (t) {
       qos: 0,
       retain: false
     }
-    const subs = [{ topic: 'hello', qos: ele.qos }, { topic: 'world', qos: ele.qos }]
+    const subs = [
+      { topic: 'hello', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined },
+      { topic: 'world', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined }
+    ]
     const expectedSubs = ele.clean ? null : subs
 
     subscribeMultiple(t, s, subs, [ele.qos, ele.qos], function () {