aedes 0.51.3 → 1.0.1

package/benchmarks/pingpong.js

@@ -1,36 +1,115 @@
 #! /usr/bin/env node
-
-const mqtt = require('mqtt')
-const convertHrtime = require('convert-hrtime')
-const mode = require('compute-mode')
-const client = mqtt.connect({ port: 1883, host: 'localhost', clean: true, keepalive: 0 })
+import { parseArgs } from 'node:util'
+import { hrtime } from 'node:process'
+import mqtt from 'mqtt'
 const interval = 5000
 
-let sent = 0
-const latencies = []
+function processsLatencies (latencies, counter) {
+  let total = 0
+  let count = 0
+  let median
+  let perc95
+  let perc99
+  const posMedian = Math.floor(counter / 2)
+  const pos95 = Math.floor(counter * 0.95)
+  const pos99 = Math.floor(counter * 0.99)
+  // sort keys from smallest to largest
+  const keys = Object.keys(latencies).sort((a, b) => a - b)
+  for (const key of keys) {
+    const value = latencies[key]
+    total += value * key
+    count += value
+
+    if (count >= posMedian && median === undefined) {
+      median = key
+    }
+    if (count >= pos95 && perc95 === undefined) {
+      perc95 = key
+    }
+    if (count >= pos99 && perc99 === undefined) {
+      perc99 = key
+    }
+  }
+  return {
+    buckets: keys.length,
+    mean: Math.floor(total / counter),
+    minimum: keys[0],
+    maximum: keys.pop(),
+    median,
+    perc95,
+    perc99,
+  }
+}
+
+let counter = 0
+let latencies = {}
+
+const { values } = parseArgs({
+  options: {
+    qos: {
+      type: 'string',
+      default: '0',
+      choices: ['0', '1'],
+      description: 'QoS level to use for publishing messages',
+      short: 'q'
+    },
+    help: {
+      type: 'boolean',
+      default: false,
+      description: 'Show this help message',
+      short: 'h'
+    }
+  }
+})
+
+if (values.help) {
+  console.log('Usage: node pingpong.js [options]')
+  console.log('Options:')
+  console.log('  -q, --qos <0|1>       QoS level to use for publishing messages (default: 0)')
+  console.log('  -h, --help            Show this help message')
+  process.exit(0)
+}
+
+if (!process.send) {
+  console.error(`Starting pingpong with options: qos=${values.qos}`)
+}
+const client = mqtt.connect({ port: 1883, host: 'localhost', clean: true, encoding: 'binary', keepalive: 0 })
+
+const qosOpts = {
+  qos: parseInt(values.qos, 10),
+}
 
 function count () {
-  console.log('sent/s', sent / interval * 1000)
-  sent = 0
+  // reset latencies while keeping the counts
+
+  const latencyResult = processsLatencies(latencies, counter)
+  latencies = {}
+  if (process.send) {
+    process.send({ type: 'latency', data: latencyResult })
+  } else {
+    console.log('latencies', latencyResult)
+  }
+  counter = 0
 }
 
 setInterval(count, interval)
 
 function publish () {
-  sent++
-  client.publish('test', JSON.stringify(process.hrtime()), { qos: 1 })
+  counter++
+  client.publish('test', process.hrtime.bigint().toString(), qosOpts)
 }
 
 function subscribe () {
-  client.subscribe('test', { qos: 1 }, publish)
+  client.subscribe('test', qosOpts, publish)
 }
 
 client.on('connect', subscribe)
 client.on('message', publish)
 client.on('message', function (topic, payload) {
-  const sentAt = JSON.parse(payload)
-  const diff = process.hrtime(sentAt)
-  latencies.push(convertHrtime(diff).ms)
+  const receivedAt = hrtime.bigint()
+  const sentAt = BigInt(payload.toString())
+  const msDiff = Math.floor(Number(receivedAt - sentAt) / 1e6) // Convert from nanoseconds to milliseconds
+  latencies[msDiff] = (latencies[msDiff] || 0) + 1
 })
 
 client.on('offline', function () {
@@ -41,13 +120,3 @@ client.on('error', function () {
   console.log('reconnect!')
   client.stream.end()
 })
-
-process.on('SIGINT', function () {
-  const total = latencies.reduce(function (acc, num) {
-    return acc + num
-  })
-  console.log('total', total)
-  console.log('average', total / latencies.length)
-  console.log('mode', mode(latencies))
-  process.exit(0)
-})

package/benchmarks/receiver.js

@@ -0,0 +1,77 @@
+#! /usr/bin/env node
+import { parseArgs } from 'node:util'
+import mqtt from 'mqtt'
+const interval = 5000
+
+let counter = 0
+let previousSerial
+
+const { values } = parseArgs({
+  options: {
+    qos: {
+      type: 'string',
+      default: '0',
+      choices: ['0', '1'],
+      description: 'QoS level to use for publishing messages',
+      short: 'q'
+    },
+    help: {
+      type: 'boolean',
+      default: false,
+      description: 'Show this help message',
+      short: 'h'
+    }
+  }
+})
+
+if (values.help) {
+  console.log('Usage: node receiver.js [options]')
+  console.log('Options:')
+  console.log('  -q, --qos <0|1>       QoS level to use for publishing messages (default: 0)')
+  console.log('  -h, --help            Show this help message')
+  process.exit(0)
+}
+
+const client = mqtt.connect({ port: 1883, host: 'localhost', clean: true, encoding: 'binary', keepalive: 0 })
+
+if (!process.send) {
+  console.error(`Starting receiver with options: qos=${values.qos}`)
+}
+const subscribeOpts = {
+  qos: parseInt(values.qos, 10),
+}
+
+function count () {
+  if (process.send) {
+    const rate = Math.floor(counter / interval * 1000)
+    process.send({ type: 'rate', data: rate })
+  } else {
+    console.log('received/s', counter / interval * 1000)
+  }
+  counter = 0
+}
+
+setInterval(count, interval)
+
+client.on('connect', function () {
+  this.subscribe('test', subscribeOpts)
+})
+
+client.handleMessage = function (packet, done) {
+  const serial = BigInt(packet.payload.toString())
+  if (previousSerial !== undefined && (serial !== (previousSerial + 1n))) {
+    console.error(`Received out of order message: expected ${previousSerial}, got ${serial}`)
+  }
+  previousSerial = serial
+  counter++
+  done()
+}
+
+client.on('offline', function () {
+  console.log('offline')
+})
+
+client.on('error', function () {
+  console.log('reconnect!')
+  client.stream.end()
+})

package/benchmarks/report.js

@@ -0,0 +1,150 @@
+#! /usr/bin/env node
+
+import readline from 'readline'
+// if any average is more than threshold% off we exit with 1
+const threshold = 10
+let failed = false
+
+const defaultUnit = 'msg/s' // default unit for the results
+const units = {
+  'pingpong.js': 'ms'
+}
+
+function parseConfig (config) {
+  // parse config string like "QoS=0, Cores=2" into an object
+  const configObj = {}
+  const parts = config.split(',').map(part => part.trim())
+  for (const part of parts) {
+    const [key, value] = part.split('=').map(s => s.trim())
+    if (key && value) {
+      configObj[key] = value
+    }
+  }
+  return configObj
+}
+
+async function gatherData () {
+  // read CSV data from STDIN
+  const results = {}
+  let maxCounts = 0
+  const rl = readline.createInterface({
+    input: process.stdin
+  })
+  // split each line by comma but retain commas inside quotes
+  for await (const line of rl) {
+    const fields = line.match(/(".*?"|[^",\s]+)(?=\s*,|\s*$)/g).map(s => s.replace(/^"|"$/g, ''))
+    const label = fields[0]
+    if (!label) {
+      continue // skips empty lines
+    }
+    const benchmark = fields[1]
+    const config = fields[2]
+    const parsedConfig = parseConfig(config)
+    const value = Number(fields[3])
+    const key = `${benchmark} QoS${parsedConfig.QoS}`
+    if (!results[label]) {
+      results[label] = {}
+    }
+    const resultsL2 = results[label]
+    if (!resultsL2[key]) {
+      resultsL2[key] = {
+        values: [],
+        benchmark,
+        config,
+        unit: units[benchmark] || defaultUnit
+      }
+    }
+    const resultsL3 = resultsL2[key]
+    resultsL3.values.push(value)
+    if (resultsL3.values.length > maxCounts) {
+      maxCounts = resultsL3.values.length
+    }
+  }
+  return { results, maxCounts }
+}
+
+function reportPerLabel (label, results, maxCounts, avg) {
+  const roundLabels = []
+  for (let i = 0; i < maxCounts; i++) {
+    roundLabels.push(`Round ${i + 1}`)
+  }
+
+  console.log(`\n # Benchmark Results for ${label}`)
+  console.log(`|Benchmark | Config | Units | ${roundLabels.join(' |')}`)
+  console.log(`|----------|--------|-------|${roundLabels.map(() => '---').join('|')}`)
+  for (const key in results) {
+    const { unit, values, benchmark, config } = results[key]
+    console.log(`| ${benchmark} | ${config} | ${unit}| ${values.join(' |')}`)
+  }
+  console.log('\n')
+}
+
+function calculateAverages (results) {
+  const avg = {}
+  for (const label in results) {
+    const resultsL2 = results[label]
+    for (const key in resultsL2) {
+      const { unit, values, benchmark, config } = resultsL2[key]
+      if (!avg[key]) {
+        avg[key] = {}
+      }
+      avg[key][label] = {
+        value: values.reduce((acc, num) => acc + num, 0) / values.length,
+        benchmark,
+        unit,
+        config
+      }
+    }
+  }
+  return avg
+}
+
+function calculatePercentage (ref, value) {
+  if (ref === undefined) {
+    return { ref: value, diff: 0 }
+  }
+  const perc = ((value / ref) * 100)
+  if (perc > 100) {
+    const diff = perc - 100
+    return { ref, diff }
+  }
+  const diff = (100 - perc) * -1
+  return { ref, diff }
+}
+
+function reportAverages (avg) {
+  console.log('\n # Overall Benchmark Results')
+  console.log(`\n +x% is better, -x% is worse, current threshold to fail at -${threshold}%\n\n`)
+  console.log('| Label | Benchmark | Config | Average | Units | Percentage')
+  console.log('|-------|-----------|--------|---------|-------|-----------')
+  for (const key in avg) {
+    let oldRef
+    for (const label in avg[key]) {
+      const { value, unit, benchmark, config } = avg[key][label]
+      const { ref, diff } = calculatePercentage(oldRef, value)
+      oldRef = ref
+      // for unit = ms lower is better
+      const correctedDiff = unit === 'ms' ? diff * -1 : diff
+      const sign = correctedDiff > 0 ? '+' : ''
+      const perc = correctedDiff === 0 ? 100 : `${sign}${correctedDiff.toFixed(2)}`
+      if ((-1 * correctedDiff) > threshold) {
+        console.error(`\n\nError: ${key} is ${sign}${correctedDiff.toFixed(2)}% off the reference (${ref} ${unit}) which is more than the threshold of ${threshold}% `)
+        failed = true
+      }
+      console.log(`| ${label} | ${benchmark} | ${config} | ${value.toFixed(0)} | ${unit} | ${perc}%`)
+    }
+  }
+}
+
+async function report () {
+  const { results, maxCounts } = await gatherData()
+  const avg = calculateAverages(results)
+  reportAverages(avg)
+  for (const label in results) {
+    reportPerLabel(label, results[label], maxCounts)
+  }
+}
+
+report().then(() => {
+  process.exit(failed ? 1 : 0)
+})

package/benchmarks/runBenchmarks.js

@@ -0,0 +1,118 @@
+import { fork, execSync } from 'node:child_process'
+import { cpus } from 'node:os'
+import path from 'node:path'
+const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim()
+const numCores = cpus().length
+const cpuType = cpus()[0].model.trim()
+const dirname = import.meta.dirname
+
+const scripts = {
+  server: path.join(dirname, 'server.js'),
+  sender: path.join(dirname, 'sender.js'),
+  receiver: path.join(dirname, 'receiver.js'),
+  pingpong: path.join(dirname, 'pingpong.js'),
+}
+
+function spawn (script, cmdArgs = [], msgType = 'rate', args = {}) {
+  const child = fork(script, cmdArgs, { env: { ...process.env, ...args } })
+  const results = []
+  child.on('message', msg => {
+    if (msg.type === msgType) results.push(msg.data)
+  })
+  return { child, results }
+}
+
+async function throughputTest (scripts, qos, warmupCount, maxCount) {
+  console.error('Starting test for QoS', qos)
+  await new Promise(resolve => {
+    console.error(`Starting warmup for QoS${qos}`)
+    // 1. Start server (no rates to collect)
+    const server = fork(scripts.server)
+
+    // 2. Start throughput receiver
+    const receiver = spawn(scripts.receiver, [`-q ${qos}`], 'rate')
+
+    // 3. Start throughput sender
+    const sender = spawn(scripts.sender, [`-q ${qos}`], 'rate')
+    const config = `"QoS=${qos}, Cores=${numCores}"`
+    sender.child.on('message', checkDone)
+
+    // 4. Collect and print rates
+    let counter = 0
+    function checkDone () {
+      counter++
+      process.stderr.write('.')
+      if (counter === warmupCount) {
+        console.error('\n starting measurement')
+        receiver.results.length = 0
+        sender.results.length = 0
+      }
+      if (counter === (maxCount + warmupCount)) {
+        sender.child.kill()
+        receiver.child.kill()
+        server.kill()
+        for (const result of sender.results) {
+          console.log(`${gitBranch}, sender.js,${config} , ${result}`)
+        }
+        for (const result of receiver.results) {
+          console.log(`${gitBranch}, receiver.js,${config} , ${result}`)
+        }
+        resolve()
+      }
+    }
+  })
+}
+async function latencyTest (scripts, qos, warmupCount, maxCount, score) {
+  console.error('Starting latency test for QoS', qos)
+  await new Promise(resolve => {
+    console.error(`Starting warmup for QoS${qos}`)
+    // 1. Start server (no rates to collect)
+    const server = fork(scripts.server)
+
+    // 2. Start latency measurement
+    const pingpong = spawn(scripts.pingpong, [`-q ${qos}`], 'latency')
+    const config = `"QoS=${qos}, Cores=${numCores}, Score='${score}'"`
+    pingpong.child.on('message', checkDone)
+
+    // 4. Collect and print latency
+    let counter = 0
+    function checkDone () {
+      counter++
+      process.stderr.write('.')
+      if (counter === warmupCount) {
+        console.error('\n starting measurement')
+        pingpong.results.length = 0
+      }
+      if (counter === (maxCount + warmupCount)) {
+        pingpong.child.kill()
+        server.kill()
+        for (const result of pingpong.results) {
+          console.log(`${gitBranch}, pingpong.js ,${config} , ${result[score]}`)
+        }
+        resolve()
+      }
+    }
+  })
+}
+
+async function main (opts) {
+  console.error(`Running benchmarks on branch: ${gitBranch} using ${cpuType} with ${numCores} cores`)
+  if (numCores < 3) {
+    console.error('WARNING: Not enough CPU cores to run proper benchmarks, at least 4 cores are recommended')
+  }
+  await throughputTest(scripts, 0, opts.warmupCount, opts.maxCount)
+  await throughputTest(scripts, 1, opts.warmupCount, opts.maxCount)
+  await latencyTest(scripts, 1, opts.warmupCount, opts.maxCount, opts.latencyScore)
+  console.error('Tests done')
+  process.exit(0)
+}
+
+const defaultopts = {
+  warmupCount: 5,
+  maxCount: 10,
+  latencyScore: 'perc95'   // valid score names are mean, median, perc95 and perc99
+}
+main(defaultopts).catch(err => {
+  console.error('Error in main:', err)
+  process.exit(1)
+})

package/benchmarks/sender.js

@@ -0,0 +1,86 @@
+#! /usr/bin/env node
+
+import mqtt from 'mqtt'
+import { parseArgs } from 'node:util'
+const interval = 5000
+
+let sent = 0
+let serial = 0n
+
+const { values } = parseArgs({
+  options: {
+    qos: {
+      type: 'string',
+      default: '0',
+      choices: ['0', '1'],
+      description: 'QoS level to use for publishing messages',
+      short: 'q'
+    },
+    help: {
+      type: 'boolean',
+      default: false,
+      description: 'Show this help message',
+      short: 'h'
+    }
+  }
+})
+
+if (values.help) {
+  console.log('Usage: node sender.js [options]')
+  console.log('Options:')
+  console.log('  -q, --qos <0|1>       QoS level to use for publishing messages (default: 0)')
+  console.log('  -h, --help            Show this help message')
+  process.exit(0)
+}
+
+if (!process.send) {
+  console.error(`Starting sender with options: qos=${values.qos}`)
+}
+const publishOpts = {
+  qos: parseInt(values.qos, 10),
+  retain: false,
+  dup: false
+}
+
+const client = mqtt.connect({ port: 1883, host: 'localhost', clean: true, keepalive: 0 })
+
+function count () {
+  if (process.send) {
+    const rate = Math.floor(sent / interval * 1000)
+    process.send({ type: 'rate', data: rate })
+  } else {
+    console.log('sent/s', sent / interval * 1000)
+  }
+  sent = 0
+}
+
+setInterval(count, interval)
+
+function immediatePublish () {
+  setImmediate(publish)
+}
+
+function publish () {
+  sent++
+  serial++
+  const payload = serial.toString()
+
+  client.publish('test', payload, publishOpts, immediatePublish)
+}
+
+client.setMaxListeners(100)
+
+client.on('connect', function () {
+  for (let i = 0; i < 50; i++) {
+    publish()
+  }
+})
+
+client.on('offline', function () {
+  console.log('offline')
+})
+
+client.on('error', function () {
+  console.log('reconnect!')
+  client.stream.end()
+})

package/benchmarks/server.js

@@ -1,25 +1,26 @@
-'use strict'
+import { Aedes } from '../aedes.js'
+import { createServer } from 'net'
 
 // To be used with cpuprofilify http://npm.im/cpuprofilify
+Aedes.createBroker().then(aedes => {
+  const server = createServer(aedes.handle)
+  const port = 1883
 
-const aedes = require('../')()
-const server = require('net').createServer(aedes.handle)
-const port = 1883
+  server.listen(port, function () {
+    console.error('server listening on port', port, 'pid', process.pid)
+  })
 
-server.listen(port, function () {
-  console.error('server listening on port', port, 'pid', process.pid)
-})
-
-aedes.on('clientError', function (client, err) {
-  console.error('client error', client.id, err.message)
-})
+  aedes.on('clientError', function (client, err) {
+    console.error('client error', client.id, err.message)
+  })
 
-// Cleanly shut down process on SIGTERM to ensure that perf-<pid>.map gets flushed
-process.on('SIGINT', onSIGINT)
+  // Cleanly shut down process on SIGTERM to ensure that perf-<pid>.map gets flushed
+  process.on('SIGINT', onSIGINT)
 
-function onSIGINT () {
+  function onSIGINT () {
   // IMPORTANT to log on stderr, to not clutter stdout which is purely for data, i.e. dtrace stacks
-  console.error('Caught SIGTERM, shutting down.')
-  server.close()
-  process.exit(0)
-}
+    console.error('Caught SIGTERM, shutting down.')
+    server.close()
+    process.exit(0)
+  }
+})

package/checkVersion.js

@@ -0,0 +1,20 @@
+// node 20 on windows has issues with node:test wildcard
+// this script is only used by package.json
+// example of usage in package.json:
+// "unit": "node checkVersion.js && npm run unit:v20win32 || npm run unit:other",
+// "unit:v20win32": "node --test --test-timeout=180000",
+// "unit:other": "node --test --test-timeout=180000 test/*.js test/*.cjs",
+
+/* c8 ignore start */
+import { platform, version, exit } from 'node:process'
+
+const major = version.split('.')[0]
+let exitCode = 1
+// node 20 on windows returns 0
+if ((major === 'v20') && platform === 'win32') {
+  exitCode = 0
+}
+// all the others 1
+console.log(`Running ${major} on ${platform} returning exitCode ${exitCode}`)
+exit(exitCode)
+/* c8 ignore stop */