@seriousme/opifex 1.11.3 → 2.1.0

package/examples/mqtt.test.ts

@@ -0,0 +1,89 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { TcpClient } from "@seriousme/opifex/tcpClient";
+import { TcpServer } from "@seriousme/opifex/tcpServer";
+import { delay, logger, LogLevel } from "@seriousme/opifex/utils";
+import type { PublishPacket, QoS } from "@seriousme/opifex/mqttPacket";
+
+logger.level(LogLevel.info);
+
+test("Test pubSub using TCP client and server using memoryPersistence", async function () {
+  const server = new TcpServer({ port: 0 }, {
+    // just an example on how to configure the server
+    configuration: { context: { maximumConnectPacketSize: 3000 } },
+  });
+  server.start();
+
+  assert.deepStrictEqual(
+    server.port !== undefined,
+    true,
+    "server runs a a random port",
+  );
+  logger.info(
+    `TCP server running on port: ${server.port}, address: ${server.address}`,
+  );
+
+  const params = {
+    url: new URL(`mqtt://${server.address}:${server.port}`),
+    numberOfRetries: 0,
+  };
+
+  const client = new TcpClient();
+  await client.connect(params);
+  logger.info("Client connected to server at", client.url);
+
+  const publishSet: { topic: string; qos: QoS }[] = [
+    { topic: "t0@q0", qos: 0 },
+    { topic: "t1@q0", qos: 0 },
+    { topic: "t2@q0", qos: 0 },
+    { topic: "t0@q1", qos: 1 },
+    { topic: "t1@q1", qos: 1 },
+    { topic: "t2@q1", qos: 1 },
+    { topic: "t0@q2", qos: 2 },
+    { topic: "t1@q2", qos: 2 },
+    { topic: "t2@q2", qos: 2 },
+  ];
+
+  const subscriptions = publishSet.map((item) => ({
+    topicFilter: item.topic,
+    qos: item.qos,
+  }));
+  await client.subscribe({
+    subscriptions,
+  });
+
+  // the IIFE ensures message reception runs in parallel
+  logger.info("Start receiving");
+  const received: PublishPacket[] = [];
+  (async function () {
+    for await (const item of client.messages()) {
+      logger.verbose("Receiving:", item.topic, "--", item.qos);
+      received.push(item);
+    }
+  })();
+  // end of IIFE
+  for (const item of publishSet) {
+    logger.verbose("Publishing:", item.topic, "--", item.qos);
+    await client.publish({
+      topic: item.topic,
+      qos: item.qos,
+      payload: new Uint8Array([0x01]),
+    });
+  }
+
+  await delay(100);
+  logger.info("Disconnect client");
+  await client.disconnect();
+
+  logger.info("Check completeness");
+  for (const item of publishSet) {
+    const found = received.find((f) =>
+      f.topic == item.topic && f.qos === item.qos
+    );
+    logger.verbose("Found:", item.topic, "--", item.qos);
+    assert(found, `${item.topic} -- ${item.qos}`);
+  }
+
+  logger.info("Stop server");
+  server.stop();
+});

package/examples/mqtts.test.ts

@@ -0,0 +1,100 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { TcpClient } from "@seriousme/opifex/tcpClient";
+import { TlsServer } from "@seriousme/opifex/tlsServer";
+import { SqlitePersistence } from "@seriousme/opifex/persistence";
+import { delay, logger, LogLevel } from "@seriousme/opifex/utils";
+import type { PublishPacket, QoS } from "@seriousme/opifex/mqttPacket";
+import { generateLocalhostCerts } from "../dev_utils/generateCert.ts";
+
+logger.level(LogLevel.info);
+
+test("Test pubSub using TLS client and server and sqlitePersistence", async function () {
+  // the localhost certs generate here are great for testing but lack securiity
+  // use something like LetsEncrypt for any serious server
+  // generateLocalhostCerts is a dev tool and relies on node-forge as a (dev)dependency
+  const { key, cert, caCert } = generateLocalhostCerts();
+  const dbFile = ":memory:";
+  // .start() combines instance creation and async initialization
+  const persistence = await SqlitePersistence.start(dbFile);
+  const server = new TlsServer({ port: 0, key, cert }, { persistence });
+  server.start();
+
+  assert.deepStrictEqual(
+    server.port !== undefined,
+    true,
+    "TLS server runs on a random port",
+  );
+  logger.info(
+    `TLS server running on port: ${server.port}, address: ${server.address}`,
+  );
+  // pick the correct hostname
+  const hostname = server.address === "0.0.0.0" ? "127.0.0.1" : server.address;
+
+  const params = {
+    url: new URL(`mqtts://${hostname}:${server.port}`),
+    numberOfRetries: 0,
+    // since we signed our server cert ourselves we need to add the caCert
+    // this is not required if you use a public Certificate Authority
+    caCerts: [caCert],
+  };
+
+  const client = new TcpClient();
+  await client.connect(params);
+  logger.info("Client connected to server at", client.url);
+
+  const publishSet: { topic: string; qos: QoS }[] = [
+    { topic: "t0@q0", qos: 0 },
+    { topic: "t1@q0", qos: 0 },
+    { topic: "t2@q0", qos: 0 },
+    { topic: "t0@q1", qos: 1 },
+    { topic: "t1@q1", qos: 1 },
+    { topic: "t2@q1", qos: 1 },
+    { topic: "t0@q2", qos: 2 },
+    { topic: "t1@q2", qos: 2 },
+    { topic: "t2@q2", qos: 2 },
+  ];
+
+  const subscriptions = publishSet.map((item) => ({
+    topicFilter: item.topic,
+    qos: item.qos,
+  }));
+  await client.subscribe({
+    subscriptions,
+  });
+
+  // the IIFE ensures message reception runs in parallel
+  logger.info("Start receiving");
+  const received: PublishPacket[] = [];
+  (async function () {
+    for await (const item of client.messages()) {
+      logger.verbose("Receiving:", item.topic, "--", item.qos);
+      received.push(item);
+    }
+  })();
+  // end of IIFE
+  for (const item of publishSet) {
+    logger.verbose("Publishing:", item.topic, "--", item.qos);
+    await client.publish({
+      topic: item.topic,
+      qos: item.qos,
+      payload: new Uint8Array([0x01]),
+    });
+  }
+
+  await delay(100);
+  logger.info("Disconnect client");
+  await client.disconnect();
+
+  logger.info("Check completeness");
+  for (const item of publishSet) {
+    const found = received.find((f) =>
+      f.topic == item.topic && f.qos === item.qos
+    );
+    logger.verbose("Found:", item.topic, "--", item.qos);
+    assert(found, `${item.topic} -- ${item.qos}`);
+  }
+
+  logger.info("Stop server");
+  server.stop();
+});

package/examples/webSocket.test.ts

@@ -0,0 +1,94 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { delay, logger, LogLevel } from "@seriousme/opifex/utils";
+
+import type { PublishPacket, QoS } from "@seriousme/opifex/mqttPacket";
+
+logger.level(LogLevel.info);
+
+test("Deno: Test pubSub using WebSocket client and server and memoryPersistence", async function () {
+  // Nodejs does not natively support WebSockets so we only import Websocket client and server here
+  if (typeof Deno !== "undefined") {
+    const { WsClient } = await import("@seriousme/opifex/wsClient");
+    const { WsServer } = await import("@seriousme/opifex/wsServer");
+    const server = new WsServer({ port: 0 }, {});
+    server.start();
+
+    assert.deepStrictEqual(
+      server.port !== undefined,
+      true,
+      "WebSocket server runs on a random port",
+    );
+    logger.info(
+      `WebSocket server running on port: ${server.port}, address: ${server.address}`,
+    );
+    // pick the correct hostname
+    const hostname = server.address === "0.0.0.0"
+      ? "127.0.0.1"
+      : server.address;
+
+    const params = {
+      url: new URL(`ws://${hostname}:${server.port}`),
+      numberOfRetries: 0,
+    };
+
+    const client = new WsClient();
+    await client.connect(params);
+    logger.info("Client connected to server at", client.url);
+
+    const publishSet: { topic: string; qos: QoS }[] = [
+      { topic: "t0@q0", qos: 0 },
+      { topic: "t1@q0", qos: 0 },
+      { topic: "t2@q0", qos: 0 },
+      { topic: "t0@q1", qos: 1 },
+      { topic: "t1@q1", qos: 1 },
+      { topic: "t2@q1", qos: 1 },
+      { topic: "t0@q2", qos: 2 },
+      { topic: "t1@q2", qos: 2 },
+      { topic: "t2@q2", qos: 2 },
+    ];
+
+    const subscriptions = publishSet.map((item) => ({
+      topicFilter: item.topic,
+      qos: item.qos,
+    }));
+    await client.subscribe({
+      subscriptions,
+    });
+
+    // the IIFE ensures message reception runs in parallel
+    logger.info("Start receiving");
+    const received: PublishPacket[] = [];
+    (async function () {
+      for await (const item of client.messages()) {
+        logger.verbose("Receiving:", item.topic, "--", item.qos);
+        received.push(item);
+      }
+    })();
+    // end of IIFE
+    for (const item of publishSet) {
+      logger.verbose("Publishing:", item.topic, "--", item.qos);
+      await client.publish({
+        topic: item.topic,
+        qos: item.qos,
+        payload: new Uint8Array([0x01]),
+      });
+    }
+
+    await delay(100);
+    logger.info("Disconnect client");
+    await client.disconnect();
+
+    logger.info("Check completeness");
+    for (const item of publishSet) {
+      const found = received.find((f) =>
+        f.topic == item.topic && f.qos === item.qos
+      );
+      logger.verbose("Found:", item.topic, "--", item.qos);
+      assert(found, `${item.topic} -- ${item.qos}`);
+    }
+
+    logger.info("Stop server");
+    server.stop();
+  }
+});

package/mqttConn/mqttConn.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { test } from "node:test";
-import { makeDummySockConn } from "../dev_utils/mod.ts";
+import { createMockSockConn } from "../dev_utils/mod.ts";
 import { encode, MQTTLevel, PacketType } from "../mqttPacket/mod.ts";
 import { MqttConn, MqttConnError } from "./mqttConn.ts";
 import type { AnyPacket, CodecOpts } from "../mqttPacket/mod.ts";
@@ -50,7 +50,7 @@ test("MqttConn should act as asyncIterator", async () => {
   const publish = encode(publishPacket, codecOptsV4);
   const disconnect = encode(disconnectPacket, codecOptsV4);
 
-  const conn = makeDummySockConn(
+  const conn = createMockSockConn(
     [connect, publish, disconnect],
     new Uint8Array(),
   );
@@ -62,13 +62,13 @@ test("MqttConn should act as asyncIterator", async () => {
   }
 
   assert.deepStrictEqual(packets.length, 3);
-  assert.deepStrictEqual(packets[0], connectPacket);
-  assert.deepStrictEqual(packets[1], publishPacket);
-  assert.deepStrictEqual(packets[2], disconnectPacket);
+  assert.deepStrictEqual(packets[0], connectPacket, "has connect packet");
+  assert.deepStrictEqual(packets[1], publishPacket, "has publish packet");
+  assert.deepStrictEqual(packets[2], disconnectPacket, "has disconnect packet");
 });
 
 test("MqttConn should close on malformed length", async () => {
-  const conn = makeDummySockConn([new Uint8Array([1, 175])], new Uint8Array());
+  const conn = createMockSockConn([new Uint8Array([1, 175])], new Uint8Array());
   const mqttConn = new MqttConn({ conn });
 
   const packets: AnyPacket[] = [];
@@ -86,7 +86,7 @@ test("MqttConn should close on failed packets", async () => {
   const publish = encode(publishPacket, codecOptsV4);
   const brokenPublish = publish.slice(0, 7);
 
-  const conn = makeDummySockConn([connect, brokenPublish], new Uint8Array());
+  const conn = createMockSockConn([connect, brokenPublish], new Uint8Array());
   const mqttConn = new MqttConn({ conn });
 
   const packets: AnyPacket[] = [];
@@ -103,7 +103,7 @@ test("MqttConn should close on failed packets", async () => {
 test("MqttConn should close on packets too large", async () => {
   const connect = encode(connectPacket, codecOptsV4);
 
-  const conn = makeDummySockConn([connect], new Uint8Array());
+  const conn = createMockSockConn([connect], new Uint8Array());
   const mqttConn = new MqttConn({ conn, maxIncomingPacketSize: 20 });
 
   const packets: AnyPacket[] = [];
@@ -119,7 +119,7 @@ test("MqttConn should close on packets too large", async () => {
 test("MqttConn should be writable", async () => {
   const connect = encode(connectPacket, codecOptsV4);
   const writer = new Uint8Array(24);
-  const conn = makeDummySockConn([connect], writer);
+  const conn = createMockSockConn([connect], writer);
   const mqttConn = new MqttConn({ conn });
   await mqttConn.send(connectPacket);
   assert.deepStrictEqual(writer, connect);

package/mqttConn/mqttConn.ts

@@ -6,7 +6,6 @@ import type {
   AnyPacket,
   CodecOpts,
   LengthDecoderResult,
-  ProtocolLevel,
 } from "../mqttPacket/mod.ts";
 
 import {
@@ -26,26 +25,19 @@ import { Conn } from "../socket/socket.ts";
 export const MqttConnError = {
   invalidPacket: "Invalid Packet",
   packetTooLarge: "Packet too large",
+  connectPacketTooLarge: "Connect Packet too large",
   UnexpectedEof: "Unexpected EOF",
 } as const;
 
-const DEFAULT_MAX_PACKETSIZE = 2 * 1024 * 1024; // 2MB
+const DEFAULT_MAX_PACKETSIZE = 4 * 1024; // 4Kb
 
-/**
- * Interface for MQTT connection handling
- */
-export interface IMqttConn extends AsyncIterable<AnyPacket> {
-  /** Underlying connection */
-  readonly conn: Conn;
-  /** Whether connection is closed */
-  readonly isClosed: boolean;
-  /** Reason for connection closure if any */
-  readonly reason: string | undefined;
-  /** Send an MQTT packet */
-  send(data: AnyPacket): Promise<void>;
-  /** Close the connection */
-  close(): void;
-}
+export type SendResult =
+  | { sent: true }
+  | {
+    sent: false;
+    reason: "packetTooLarge" | "connectionClosed";
+    size?: number;
+  };
 
 /**
  * Read a single byte from the connection
@@ -54,12 +46,11 @@ export interface IMqttConn extends AsyncIterable<AnyPacket> {
  * @throws Error if EOF reached unexpectedly
  */
 async function readByte(conn: Conn): Promise<number> {
-  const buf = new Uint8Array(1);
-  const bytesRead = await conn.read(buf);
-  assert(bytesRead !== null, MqttConnError.UnexpectedEof);
-  assert(bytesRead !== undefined, MqttConnError.UnexpectedEof);
-  assert(bytesRead !== 0, MqttConnError.UnexpectedEof);
-  return buf[0] || 0;
+  const buff = await conn.read(1);
+  assert(buff !== null, MqttConnError.UnexpectedEof);
+  assert(buff !== undefined, MqttConnError.UnexpectedEof);
+  assert(buff.byteLength !== 0, MqttConnError.UnexpectedEof);
+  return buff[0] || 0;
 }
 
 /**
@@ -68,31 +59,30 @@ async function readByte(conn: Conn): Promise<number> {
  * @param buf Buffer to read into
  * @throws Error if EOF reached unexpectedly
  */
-async function readFull(conn: Conn, buf: Uint8Array): Promise<void> {
-  let bytesRead = 0;
-  while (bytesRead < buf.length) {
-    const read = await conn.read(buf.subarray(bytesRead));
-    assert(read !== null, MqttConnError.UnexpectedEof);
-    assert(read !== 0, MqttConnError.UnexpectedEof);
-    bytesRead += read;
-  }
+async function readFull(conn: Conn, length: number): Promise<Uint8Array> {
+  const buff = await conn.read(length);
+  assert(buff !== null, MqttConnError.UnexpectedEof);
+  assert(buff.byteLength === length, MqttConnError.UnexpectedEof);
+  return buff;
 }
 
 /**
  * Read a complete MQTT packet from the connection
  * @param conn Connection to read from
- * @param maxIncomingPacketSize Maximum allowed packet size
+ * @param codecOpts the CodecOpts
  * @returns Decoded MQTT packet
  * @throws Error if packet is invalid or too large
  */
 export async function readPacket(
   conn: Conn,
   codecOpts: CodecOpts,
+  maxConnectPacketSize: number,
 ): Promise<AnyPacket> {
   // fixed header is 1 byte of type + flags
   // + a maximum of 4 bytes to encode the remaining length
   const decodeLength = getLengthDecoder();
   const firstByte = await readByte(conn);
+  const isConnect = (firstByte >> 4) === 1;
   let result: LengthDecoderResult;
   do {
     const byte = await readByte(conn);
@@ -100,25 +90,32 @@ export async function readPacket(
   } while (!result.done);
 
   const remainingLength = result.length;
+  if (isConnect) {
+    assert(
+      remainingLength < maxConnectPacketSize - 1,
+      MqttConnError.connectPacketTooLarge,
+    );
+  }
   const maxIncomingPacketSize = codecOpts.maxIncomingPacketSize;
   assert(
     remainingLength < maxIncomingPacketSize - 1,
     MqttConnError.packetTooLarge,
   );
-  const packetBuf = new Uint8Array(remainingLength);
   // read the rest of the packet
-  await readFull(conn, packetBuf);
+  const packetBuf = await readFull(conn, remainingLength);
   const packet = decodePayload(firstByte, packetBuf, codecOpts);
   assert(packet !== null, MqttConnError.UnexpectedEof);
   return packet;
 }
 
 /**
- * MQTT Connection class implementing IMqttConn interface
+ * MQTT Connection class
  */
-export class MqttConn implements IMqttConn {
+export class MqttConn {
   /** Underlying connection */
   readonly conn: Conn;
+  /** Maximum Connect packet size */
+  readonly maxConnectPacketSize: number;
   /** codecOpts */
   codecOpts: CodecOpts;
   /** Reason for connection closure if any */
@@ -130,24 +127,25 @@ export class MqttConn implements IMqttConn {
    * Create new MQTT connection
    * @param options Connection options
    * @param options.conn Underlying socket connection
-   * @param options.maxIncomingPacketSize Maximum allowed incoming packet size (default 2MB)
-   * @param options.maxOutGoingPacketSize Maximum allowed outgoing packet size (default 2MB)
+   * @param options.maxConnectPacketSize Maximum allowed connect packet size
+   * @param options.maxIncomingPacketSize Maximum allowed incoming packet size
+   * @param options.maxOutGoingPacketSize Maximum allowed outgoing packet size
    * @param options.protocolLevel Supported protocolLevel
    */
-  constructor({
-    conn,
-    maxIncomingPacketSize = DEFAULT_MAX_PACKETSIZE,
-    protocolLevel = MQTTLevel.unknown,
-  }: {
-    conn: SockConn;
-    maxIncomingPacketSize?: number;
-    protocolLevel?: ProtocolLevel;
-  }) {
-    this.conn = new Conn(conn);
+  constructor(
+    options:
+      & { conn: SockConn; maxConnectPacketSize?: number }
+      & Partial<CodecOpts>,
+  ) {
+    this.conn = new Conn(options.conn);
+    this.maxConnectPacketSize = options.maxConnectPacketSize ??
+      DEFAULT_MAX_PACKETSIZE;
     this.codecOpts = {
-      maxIncomingPacketSize,
-      maxOutgoingPacketSize: maxIncomingPacketSize,
-      protocolLevel,
+      maxIncomingPacketSize: options.maxIncomingPacketSize ??
+        DEFAULT_MAX_PACKETSIZE,
+      maxOutgoingPacketSize: options.maxOutgoingPacketSize ??
+        DEFAULT_MAX_PACKETSIZE,
+      protocolLevel: options.protocolLevel ?? MQTTLevel.unknown,
     };
   }
 
@@ -174,7 +172,11 @@ export class MqttConn implements IMqttConn {
   async #receive(): Promise<AnyPacket | undefined> {
     if (!this._isClosed) {
       try {
-        return await readPacket(this.conn, this.codecOpts);
+        return await readPacket(
+          this.conn,
+          this.codecOpts,
+          this.maxConnectPacketSize,
+        );
       } catch (err) {
         if (err instanceof Error) {
           if (err.name === "PartialReadError") {
@@ -190,6 +192,7 @@ export class MqttConn implements IMqttConn {
     return Promise.resolve(undefined);
   }
 
+  /** Return next packet. */
   async next(): Promise<IteratorResult<AnyPacket>> {
     const packet = await this.#receive();
 
@@ -200,23 +203,36 @@ export class MqttConn implements IMqttConn {
     return { value: undefined, done: true };
   }
 
+  /** Async iterator. */
   [Symbol.asyncIterator](): this {
     return this;
   }
 
-  /**
+  /*
    * Send an MQTT packet
    * @param data Packet to send
    */
-  async send(data: AnyPacket): Promise<void> {
-    try {
-      await this.conn.write(encode(data, this.codecOpts));
-    } catch (err) {
-      if (err instanceof Error) {
-        this._reason = err.message;
+  async send(data: AnyPacket): Promise<SendResult> {
+    if (!this._isClosed) {
+      try {
+        const encoded = encode(data, this.codecOpts);
+        if (encoded.byteLength > this.codecOpts.maxOutgoingPacketSize) {
+          return {
+            sent: false,
+            reason: "packetTooLarge",
+            size: encoded.byteLength,
+          };
+        }
+        await this.conn.write(encoded);
+        return { sent: true };
+      } catch (err) {
+        if (err instanceof Error) {
+          this._reason = err.message;
+        }
+        this.close();
       }
-      this.close();
     }
+    return { sent: false, reason: "connectionClosed" };
   }
 
   /** Whether connection is closed */

package/mqttPacket/AuthenticationResult.ts

@@ -10,6 +10,12 @@ export const AuthenticationResult = {
   notAuthorized: 5,
 } as const;
 
+/**
+ * Type to limit authentication result to valid values
+ */
+export type AuthenticationResult =
+  typeof AuthenticationResult[keyof typeof AuthenticationResult];
+
 /**
  * Reverse lookup for AuthenticationResult
  */

package/mqttPacket/PacketType.ts

@@ -26,3 +26,8 @@ export const PacketType = {
 export const PacketNameByType: Record<number, string> = Object.fromEntries(
   Object.entries(PacketType).map(([k, v]) => [v, k]),
 );
+
+/**
+ * Type to limit packet type to valid values
+ */
+export type PacketType = typeof PacketType[keyof typeof PacketType];

package/mqttPacket/Properties.ts

@@ -6,7 +6,7 @@ export const PropertySetType = {
   will: 100,
 } as const;
 
-export type TPropertySetType =
+export type PropertySetType =
   typeof PropertySetType[keyof typeof PropertySetType];
 
 export const propertyKind = {
@@ -117,11 +117,14 @@ export type ValidPropertyNumber =
   (typeof propertyToId)[keyof typeof propertyToId];
 // Then, create a mapped type that iterates over the property names
 // and uses propertyToKind to look up the correct TypeScript type.
+// properties are optional, but can also hold "undefined"
 
 type AllMqttv5Properties = {
-  [K in PropertyNames]?: Mqttv5PropertyType<
-    typeof propertyToKind[typeof propertyToId[K]]
-  >;
+  [K in PropertyNames]?:
+    | Mqttv5PropertyType<
+      typeof propertyToKind[typeof propertyToId[K]]
+    >
+    | undefined;
 };
 
 type PropertyByNumberType = InvertRecord<typeof propertyToId>;
@@ -242,7 +245,7 @@ type GeneratePacketTypeProperties<T extends readonly ValidPropertyNumber[]> =
   Pick<AllMqttv5Properties, PropertyIdsToKeys<T>>;
 
 export type PropsByPacketSetType = {
-  [K in TPropertySetType]: GeneratePacketTypeProperties<
+  [K in PropertySetType]: GeneratePacketTypeProperties<
     typeof PropertyByPropertySetType[K]
   >;
 };