aedes 0.46.3 → 0.47.0

package/.github/workflows/ci.yml

@@ -16,14 +16,14 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14.x, 16.x, "*"]
         os: [ubuntu-latest, windows-latest, macOS-latest]
 
     steps:
-      - uses: actions/checkout@v2.4.0
+      - uses: actions/checkout@v3
 
       - name: Use Node.js
-        uses: actions/setup-node@v2.5.1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
 

package/README.md

@@ -24,7 +24,7 @@ Barebone MQTT server that can run on any stream servers
   - [Features](#features)
   - [Examples](#examples)
   - [Clusters](#clusters)
-  - [Exensions](#exensions)
+  - [Extensions](#extensions)
   - [Middleware Plugins](#middleware-plugins)
     - [Persistence](#persistence)
     - [MQEmitter](#mqemitter)

package/aedes.js

@@ -5,14 +5,13 @@ const util = require('util')
 const parallel = require('fastparallel')
 const series = require('fastseries')
 const { v4: uuidv4 } = require('uuid')
-const bulk = require('bulk-write-stream')
 const reusify = require('reusify')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const Packet = require('aedes-packet')
 const memory = require('aedes-persistence')
 const mqemitter = require('mqemitter')
 const Client = require('./lib/client')
-const { $SYS_PREFIX } = require('./lib/utils')
+const { $SYS_PREFIX, bulk } = require('./lib/utils')
 
 module.exports = Aedes.Server = Aedes
 
@@ -102,7 +101,7 @@ function Aedes (opts) {
 
     pipeline(
       that.persistence.streamWill(that.brokers),
-      bulk.obj(receiveWills),
+      bulk(receiveWills),
       function done (err) {
         if (err) {
           that.emit('error', err)
@@ -171,6 +170,7 @@ function storeRetained (packet, done) {
 }
 
 function emitPacket (packet, done) {
+  if (this.client) packet.clientId = this.client.id
   this.broker.mq.emit(packet, done)
 }
 

package/lib/client.js

@@ -10,7 +10,7 @@ const QoSPacket = require('./qos-packet')
 const handleSubscribe = require('./handlers/subscribe')
 const handleUnsubscribe = require('./handlers/unsubscribe')
 const handle = require('./handlers')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const { through } = require('./utils')
 
 module.exports = Client
@@ -89,9 +89,20 @@ function Client (broker, conn, req) {
   conn.on('end', this.close.bind(this))
   this._eos = eos(this.conn, this.close.bind(this))
 
-  this.deliver0 = function deliverQoS0 (_packet, cb) {
+  const getToForwardPacket = (_packet) => {
+    // Mqttv5 3.8.3.1: https://docs.oasis-open.org/mqtt/mqtt/v5.0/mqtt-v5.0.html#_Toc3901169
+    // prevent to forward messages sent by the same client when no-local flag is set
+    if (_packet.clientId === that.id && _packet.nl) return
+
     const toForward = dedupe(that, _packet) &&
       that.broker.authorizeForward(that, _packet)
+
+    return toForward
+  }
+
+  this.deliver0 = function deliverQoS0 (_packet, cb) {
+    const toForward = getToForwardPacket(_packet)
+
     if (toForward) {
       // Give nodejs some time to clear stacks, or we will see
       // "Maximum call stack size exceeded" in a very high load
@@ -114,8 +125,8 @@ function Client (broker, conn, req) {
       that.deliver0(_packet, cb)
       return
     }
-    const toForward = dedupe(that, _packet) &&
-      that.broker.authorizeForward(that, _packet)
+    const toForward = getToForwardPacket(_packet)
+
     if (toForward) {
       setImmediate(() => {
         const packet = new QoSPacket(toForward, that)

package/lib/handlers/connect.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const retimer = require('retimer')
-const { pipeline } = require('readable-stream')
+const { pipeline } = require('stream')
 const write = require('../write')
 const QoSPacket = require('../qos-packet')
 const { through } = require('../utils')

package/lib/handlers/subscribe.js

@@ -142,13 +142,12 @@ function addSubs (sub, done) {
   const nl = this.nl
   let func = qos > 0 ? client.deliverQoS : client.deliver0
 
-  if (!rap) {
-    const deliverFunc = func
-    func = function handlePacketSubscription (_packet, cb) {
-      _packet = new Packet(_packet, broker)
-      _packet.retain = false
-      deliverFunc(_packet, cb)
-    }
+  const deliverFunc = func
+  func = function handlePacketSubscription (_packet, cb) {
+    _packet = new Packet(_packet, broker)
+    _packet.nl = nl
+    if (!rap) _packet.retain = false
+    deliverFunc(_packet, cb)
   }
 
   // [MQTT-4.7.2-1]

package/lib/utils.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Transform } = require('readable-stream')
+const { Transform, Writable } = require('stream')
 
 function validateTopic (topic, message) {
   const end = topic.length - 1
@@ -37,8 +37,18 @@ function through (transform) {
   })
 }
 
+function bulk (fn) {
+  return new Writable({
+    objectMode: true,
+    writev: function (chunks, cb) {
+      fn(chunks.map(chunk => chunk.chunk), cb)
+    }
+  })
+}
+
 module.exports = {
   validateTopic,
   through,
+  bulk,
   $SYS_PREFIX: '$SYS/'
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aedes",
-  "version": "0.46.3",
+  "version": "0.47.0",
   "description": "Stream-based MQTT broker",
   "main": "aedes.js",
   "types": "aedes.d.ts",
@@ -94,41 +94,39 @@
   },
   "homepage": "https://github.com/moscajs/aedes#readme",
   "engines": {
-    "node": ">=12"
+    "node": ">=14"
   },
   "devDependencies": {
-    "@sinonjs/fake-timers": "^9.1.0",
-    "@types/node": "^17.0.15",
-    "@typescript-eslint/eslint-plugin": "^5.10.2",
-    "@typescript-eslint/parser": "^5.10.2",
+    "@sinonjs/fake-timers": "^9.1.2",
+    "@types/node": "^17.0.24",
+    "@typescript-eslint/eslint-plugin": "^5.19.0",
+    "@typescript-eslint/parser": "^5.19.0",
     "concat-stream": "^2.0.0",
     "duplexify": "^4.1.2",
     "license-checker": "^25.0.1",
-    "markdownlint-cli": "^0.31.0",
-    "mqtt": "^4.3.4",
+    "markdownlint-cli": "^0.31.1",
+    "mqtt": "^4.3.7",
     "mqtt-connection": "^4.1.0",
     "pre-commit": "^1.2.2",
     "proxyquire": "^2.1.3",
-    "release-it": "^14.12.4",
+    "release-it": "^14.14.2",
     "snazzy": "^9.0.0",
     "standard": "^16.0.4",
-    "tap": "^15.1.6",
-    "tsd": "^0.19.1",
-    "typescript": "^4.5.5",
+    "tap": "^16.0.1",
+    "tsd": "^0.20.0",
+    "typescript": "^4.6.3",
     "websocket-stream": "^5.5.2"
   },
   "dependencies": {
-    "aedes-packet": "^2.3.1",
-    "aedes-persistence": "^8.1.3",
-    "bulk-write-stream": "^2.0.1",
+    "aedes-packet": "^3.0.0",
+    "aedes-persistence": "^9.1.1",
     "end-of-stream": "^1.4.4",
     "fastfall": "^1.5.1",
     "fastparallel": "^2.4.1",
     "fastseries": "^2.0.0",
-    "hyperid": "^3.0.0",
+    "hyperid": "^3.0.1",
     "mqemitter": "^4.5.0",
     "mqtt-packet": "^7.1.2",
-    "readable-stream": "^3.6.0",
     "retimer": "^3.0.0",
     "reusify": "^1.0.4",
     "uuid": "^8.3.2"

package/test/auth.js

@@ -422,7 +422,7 @@ test('authentication error when non numeric return code is passed', function (t)
 test('authorize publish', function (t) {
   t.plan(4)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -445,6 +445,7 @@ test('authorize publish', function (t) {
     t.notOk(Object.prototype.hasOwnProperty.call(packet, 'messageId'), 'should not contain messageId in QoS 0')
     expected.brokerId = s.broker.id
     expected.brokerCounter = s.broker.counter
+    expected.clientId = 'my-client-xyz'
     delete expected.length
     t.same(packet, expected, 'packet matches')
     cb()
@@ -460,7 +461,7 @@ test('authorize publish', function (t) {
 test('authorize waits for authenticate', function (t) {
   t.plan(6)
 
-  const s = setup()
+  const s = setup(aedes({ clientId: 'my-client-xyz-2' }))
   t.teardown(s.broker.close.bind(s.broker))
 
   s.broker.authenticate = function (client, username, password, cb) {
@@ -485,7 +486,8 @@ test('authorize waits for authenticate', function (t) {
     qos: 0,
     retain: false,
     length: 12,
-    dup: false
+    dup: false,
+    clientId: 'my-client'
   }
 
   s.broker.mq.on('hello', function (packet, cb) {
@@ -519,12 +521,13 @@ test('authorize publish from configOptions', function (t) {
   t.plan(4)
 
   const s = connect(setup(aedes({
+    clientId: 'my-client-xyz-3',
     authorizePublish: function (client, packet, cb) {
       t.ok(client, 'client exists')
       t.same(packet, expected, 'packet matches')
       cb()
     }
-  })))
+  })), { clientId: 'my-client-xyz-3' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -541,6 +544,7 @@ test('authorize publish from configOptions', function (t) {
     t.notOk(Object.prototype.hasOwnProperty.call(packet, 'messageId'), 'should not contain messageId in QoS 0')
     expected.brokerId = s.broker.id
     expected.brokerCounter = s.broker.counter
+    expected.clientId = 'my-client-xyz-3'
     delete expected.length
     t.same(packet, expected, 'packet matches')
     cb()
@@ -589,7 +593,7 @@ test('do not authorize publish', function (t) {
 test('modify qos out of range in authorize publish ', function (t) {
   t.plan(2)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz-4' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -599,7 +603,8 @@ test('modify qos out of range in authorize publish ', function (t) {
     qos: 0,
     retain: false,
     length: 12,
-    dup: false
+    dup: false,
+    clientId: 'my-client-xyz-4'
   }
 
   s.broker.authorizePublish = function (client, packet, cb) {
@@ -805,12 +810,19 @@ test('negate multiple subscriptions', function (t) {
 test('negate subscription with correct persistence', function (t) {
   t.plan(6)
 
+  // rh, rap, nl are undefined because mqtt.parser is set to MQTT 3.1.1 and will thus erase these props from s.inStream.write
   const expected = [{
     topic: 'hello',
-    qos: 0
+    qos: 0,
+    rh: undefined,
+    rap: undefined,
+    nl: undefined
   }, {
     topic: 'world',
-    qos: 0
+    qos: 0,
+    rh: undefined,
+    rap: undefined,
+    nl: undefined
   }]
 
   const broker = aedes()
@@ -839,10 +851,16 @@ test('negate subscription with correct persistence', function (t) {
     messageId: 24,
     subscriptions: [{
       topic: 'hello',
-      qos: 0
+      qos: 0,
+      rh: 0,
+      rap: true,
+      nl: false
     }, {
       topic: 'world',
-      qos: 0
+      qos: 0,
+      rh: 0,
+      rap: true,
+      nl: false
     }]
   })
 })

package/test/basic.js

@@ -20,7 +20,7 @@ test('test aedes.Server', function (t) {
 test('publish QoS 0', function (t) {
   t.plan(2)
 
-  const s = connect(setup())
+  const s = connect(setup(), { clientId: 'my-client-xyz-5' })
   t.teardown(s.broker.close.bind(s.broker))
 
   const expected = {
@@ -29,7 +29,8 @@ test('publish QoS 0', function (t) {
     payload: Buffer.from('world'),
     qos: 0,
     retain: false,
-    dup: false
+    dup: false,
+    clientId: 'my-client-xyz-5'
   }
 
   s.broker.mq.on('hello', function (packet, cb) {
@@ -128,7 +129,7 @@ test('publish to $SYS topic throws error', function (t) {
       qos: 0,
       retain: false
     }
-    const expectedSubs = ele.clean ? null : [{ topic: 'hello', qos: ele.qos }]
+    const expectedSubs = ele.clean ? null : [{ topic: 'hello', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined }]
 
     subscribe(t, s, 'hello', ele.qos, function () {
       s.outStream.once('data', function (packet) {
@@ -188,7 +189,10 @@ test('return write errors to callback', function (t) {
       qos: 0,
       retain: false
     }
-    const subs = [{ topic: 'hello', qos: ele.qos }, { topic: 'world', qos: ele.qos }]
+    const subs = [
+      { topic: 'hello', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined },
+      { topic: 'world', qos: ele.qos, rh: undefined, rap: undefined, nl: undefined }
+    ]
     const expectedSubs = ele.clean ? null : subs
 
     subscribeMultiple(t, s, subs, [ele.qos, ele.qos], function () {

package/test/bridge.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { test } = require('tap')
+const { setup, connect, subscribe } = require('./helper')
+
+for (const qos of [0, 1, 2]) {
+  const packet = {
+    qos,
+    cmd: 'publish',
+    topic: 'hello',
+    payload: 'world'
+  }
+
+  if (qos > 0) packet.messageId = 42
+
+  test('normal client sends a publish message and shall receive it back, qos = ' + qos, function (t) {
+    const s = connect(setup())
+    t.teardown(s.broker.close.bind(s.broker))
+
+    const handle = setTimeout(() => {
+      t.fail('did not receive packet back')
+      t.end()
+    }, 1000)
+
+    subscribe(t, s, 'hello', qos, function () {
+      s.outStream.on('data', (packet) => {
+        if (packet.cmd === 'publish') {
+          clearTimeout(handle)
+          t.end()
+        } else if (packet.cmd === 'pubrec') {
+          s.inStream.write({ cmd: 'pubrel', messageId: 42 })
+        }
+      })
+
+      s.inStream.write(packet)
+    })
+  })
+
+  test('bridge client sends a publish message but shall not receive it back, qos = ' + qos, function (t) {
+    // protocolVersion 128 + 4 means mqtt 3.1.1 with bridgeMode enabled
+    // https://github.com/mqttjs/mqtt-packet/blob/7f7c2ed8bcb4b2c582851d120a94e0b4a731f661/parser.js#L171
+    const s = connect(setup(), { clientId: 'my-client-bridge-1', protocolVersion: 128 + 4 })
+    t.teardown(s.broker.close.bind(s.broker))
+
+    const handle = setTimeout(() => t.end(), 1000)
+
+    subscribe(t, s, 'hello', qos, function () {
+      s.outStream.on('data', function () {
+        clearTimeout(handle)
+        t.fail('should not receive packet back')
+        t.end()
+      })
+
+      s.inStream.write(packet)
+    })
+  })
+}

package/test/client-pub-sub.js

@@ -905,10 +905,10 @@ test('should not receive a message on negated subscription', function (t) {
 test('programmatically add custom subscribe', function (t) {
   t.plan(6)
 
-  const broker = aedes()
+  const broker = aedes({ clientId: 'my-client-xyz-7' })
   t.teardown(broker.close.bind(broker))
 
-  const s = connect(setup(broker))
+  const s = connect(setup(broker), { clientId: 'my-client-xyz-7' })
   const expected = {
     cmd: 'publish',
     topic: 'hello',
@@ -924,7 +924,8 @@ test('programmatically add custom subscribe', function (t) {
     payload: Buffer.from('world'),
     qos: 0,
     retain: false,
-    dup: false
+    dup: false,
+    clientId: 'my-client-xyz-7'
   }
   subscribe(t, s, 'hello', 0, function () {
     broker.subscribe('hello', deliver, function () {
@@ -963,9 +964,10 @@ test('custom function in broker.subscribe', function (t) {
     qos: 1,
     retain: false,
     dup: false,
-    messageId: undefined
+    messageId: undefined,
+    clientId: 'my-client-xyz-6'
   }
-  connect(s, {}, function () {
+  connect(s, { clientId: 'my-client-xyz-6' }, function () {
     broker.subscribe('hello', deliver, function () {
       t.pass('subscribed')
     })

package/test/qos2.js

@@ -239,7 +239,7 @@ test('call published method with client with QoS 2', function (t) {
     t.teardown(broker.close.bind(broker))
 
     const opts = { clean: cleanSession }
-    const publisher = connect(setup(broker))
+    const publisher = connect(setup(broker), { clientId: 'my-client-xyz-8' })
     const subscriber = connect(setup(broker), { ...opts, clientId: 'abcde' })
     const forwarded = {
       cmd: 'publish',
@@ -248,7 +248,8 @@ test('call published method with client with QoS 2', function (t) {
       qos: 2,
       retain: false,
       dup: false,
-      messageId: undefined
+      messageId: undefined,
+      clientId: 'my-client-xyz-8'
     }
     const expected = {
       cmd: 'publish',
@@ -262,6 +263,7 @@ test('call published method with client with QoS 2', function (t) {
     broker.authorizeForward = function (client, packet) {
       forwarded.brokerId = broker.id
       forwarded.brokerCounter = broker.counter
+      delete packet.nl
       t.same(packet, forwarded, 'forwarded packet must match')
       return packet
     }

package/types/packet.d.ts

@@ -1,9 +1,10 @@
 import { AedesPacket } from 'aedes-packet'
 import { IConnackPacket, IConnectPacket, IPingreqPacket, IPublishPacket, IPubrelPacket, ISubscribePacket, ISubscription, IUnsubscribePacket } from 'mqtt-packet'
+import { Client } from './client'
 
 export type SubscribePacket = ISubscribePacket & { cmd: 'subscribe' }
 export type UnsubscribePacket = IUnsubscribePacket & { cmd: 'unsubscribe' }
-export type Subscription = ISubscription & { clientId?: string }
+export type Subscription = ISubscription & { clientId?: Client['id'] }
 export type Subscriptions = { subscriptions: Subscription[] }
 
 export type PublishPacket = IPublishPacket & { cmd: 'publish' }