@seriousme/opifex 2.3.0 → 2.4.1

package/examples/mqtts.test.ts

@@ -13,7 +13,7 @@ test("Test pubSub using TLS client and server and sqlitePersistence", async func
   // the localhost certs generate here are great for testing but lack securiity
   // use something like LetsEncrypt for any serious server
   // generateLocalhostCerts is a dev tool and relies on node-forge as a (dev)dependency
-  const { key, cert, caCert } = generateLocalhostCerts();
+  const { key, cert, caCert } = await generateLocalhostCerts();
   const dbFile = ":memory:";
   // .start() combines instance creation and async initialization
   const persistence = await SqlitePersistence.start(dbFile);

package/node/nodeSocketToNext.ts

@@ -1,187 +1,124 @@
 import type { Socket } from "node:net";
-import type { ConnAddr, ConnNext } from "../connNext/connNext.ts"; // Custom types
-
-/** wrapper class */
-export class NodeSocketConnNextWrapper {
-  private endPoint: Socket;
-  private closed: boolean;
-  private leftover: Uint8Array = new Uint8Array(0);
-
-  // Store bound event listeners so we can cleanly remove them later
-  private onClose = () => {
-    this.closed = true;
-  };
-  private onError = () => {
-    this.closed = true;
-  };
-
-  /** Create a new ConnNext object from a Webstream */
-  constructor(endPoint: Socket) {
-    this.endPoint = endPoint;
-    this.closed = false;
-    this.endPoint;
-  }
-
-  async read(length?: number): Promise<Uint8Array | null> {
-    if (this.closed || (length ?? 0) < 0) {
-      return null;
-    }
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+import { BaseConnNext } from "../connNext/baseConnNext.ts";
 
-    // If no specific byte length is requested, read whatever is available
-    if (length === undefined) {
-      const chunk = this.endPoint.read();
-      return chunk
-        ? Promise.resolve(
-          new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
-        )
-        : Promise.resolve(null);
-    }
+export class NodeSocketConnNext extends BaseConnNext {
+  private socket: Socket;
 
-    if (length === 0) {
-      return new Uint8Array(0);
-    }
+  constructor(socket: Socket) {
+    super();
+    this.socket = socket;
 
-    const result = new Uint8Array(length);
-    let bytesAccumulated = 0;
+    const markClosed = () => {
+      this.closed = true;
+    };
 
-    // 1. Process leftover data from the previous read operation
-    if (this.leftover.byteLength > 0) {
-      const take = Math.min(this.leftover.byteLength, length);
-      result.set(this.leftover.subarray(0, take), bytesAccumulated);
-      bytesAccumulated += take;
-      this.leftover = this.leftover.subarray(take);
-    }
+    this.socket.on("end", markClosed);
+    this.socket.on("close", markClosed);
+    this.socket.on("error", markClosed);
+  }
 
-    // 2. Loop until the requested number of bytes is accumulated
-    while (bytesAccumulated < length) {
-      if (this.closed || !this.endPoint.readable) {
-        break;
+  get localAddr(): ConnAddr | undefined {
+    return this.socket.localAddress
+      ? {
+        transport: "tcp",
+        hostname: this.socket.localAddress,
+        port: this.socket.localPort ?? 0,
       }
+      : undefined;
+  }
 
-      const chunk = this.endPoint.read();
-      if (chunk === null) {
-        await new Promise<void>((resolve) => {
-          const cleanup = () => {
-            this.endPoint.off("readable", onEvent);
-            this.endPoint.off("end", onEvent);
-            this.endPoint.off("error", onEvent);
-            this.endPoint.off("close", onEvent);
-          };
-          const onEvent = () => {
-            cleanup();
-            resolve();
-          };
-
-          this.endPoint.on("readable", onEvent);
-          this.endPoint.on("end", onEvent);
-          this.endPoint.on("error", onEvent);
-          this.endPoint.on("close", onEvent);
-        });
-
-        continue;
+  get remoteAddr(): ConnAddr | undefined {
+    return this.socket.remoteAddress
+      ? {
+        transport: "tcp",
+        hostname: this.socket.remoteAddress,
+        port: this.socket.remotePort ?? 0,
       }
+      : undefined;
+  }
 
-      // 3. Process the retrieved chunk
-      const chunkLength = chunk.byteLength;
-      const needed = length - bytesAccumulated;
-
-      if (chunkLength <= needed) {
-        result.set(chunk, bytesAccumulated);
-        bytesAccumulated += chunkLength;
-      } else {
-        result.set(chunk.subarray(0, needed), bytesAccumulated);
-        bytesAccumulated += needed;
-        this.leftover = chunk.subarray(needed);
-      }
+  /**
+   * Haalt direct op verzoek een chunk op via Node's ingebouwde socket.read() mechanisme.
+   */
+  protected pullNextChunk(): Promise<Uint8Array | null> {
+    if (this.closed || !this.socket.readable) {
+      return Promise.resolve(null);
     }
 
-    if (bytesAccumulated < length) {
-      return null;
+    // Vraag direct om data uit de Node socket buffer
+    const chunk = this.socket.read();
+
+    if (chunk !== null) {
+      return Promise.resolve(
+        new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+      );
     }
 
-    return result;
+    // Als er nog geen data is ('null'), wachten we op het 'readable' (of 'end'/'close') event van Node
+    return new Promise<Uint8Array | null>((resolve) => {
+      const cleanup = () => {
+        this.socket.off("readable", onReadable);
+        this.socket.off("end", onEnd);
+        this.socket.off("error", onClose);
+        this.socket.off("close", onClose);
+      };
+
+      const onReadable = () => {
+        cleanup();
+        // Roep zichzelf direct weer aan om de data op te halen
+        resolve(this.pullNextChunk());
+      };
+
+      const onEnd = () => {
+        cleanup();
+        this.closed = true;
+        resolve(null);
+      };
+
+      const onClose = () => {
+        cleanup();
+        this.closed = true;
+        resolve(null);
+      };
+
+      this.socket.on("readable", onReadable);
+      this.socket.on("end", onEnd);
+      this.socket.on("error", onClose);
+      this.socket.on("close", onClose);
+    });
   }
 
   write(data: Uint8Array): Promise<void> {
-    if (this.closed || !this.endPoint.writable) {
-      return Promise.reject(new Error("Stream is closed or not writable"));
+    if (this.isClosed() || !this.socket.writable) {
+      return Promise.reject(new Error("Socket is closed or not writable"));
     }
 
     return new Promise((resolve, reject) => {
-      // 1. Fire the write command and catch any immediate/async errors via the callback
-      const canWriteMore = this.endPoint.write(data, (err) => {
-        if (err) {
-          reject(err);
-        }
+      const canWriteMore = this.socket.write(data, (err) => {
+        if (err) reject(err);
       });
 
-      // 2. Determine when to resolve based on the return value of write()
       if (canWriteMore) {
-        // If the buffer is not full, we can resolve immediately
         resolve();
       } else {
-        // If backpressure is triggered, we must wait for the 'drain' event
-        this.endPoint.once("drain", () => {
-          resolve();
-        });
+        this.socket.once("drain", resolve);
       }
     });
   }
 
-  close() {
+  close(): void {
     if (!this.closed) {
       this.closed = true;
-
-      // Clean up permanent event listeners to prevent memory leaks
-      this.endPoint.off("close", this.onClose);
-      this.endPoint.off("error", this.onError);
-
-      // Safely destroy the stream first, then trigger the closer callback
       try {
-        this.endPoint.destroy();
-        // deno-coverage-ignore
-      } catch (_) {
-        // Suppress stream destruction errors
-        // deno-coverage-ignore
+        this.socket.destroy();
+      } catch {
+        /* swallow */
       }
     }
   }
-
-  async *[Symbol.asyncIterator]() {
-    while (true) {
-      const chunk = await this.read();
-      if (chunk === null) {
-        break;
-      }
-      yield chunk;
-    }
-  }
-
-  // Convert addresses to custom ConnAddr format
-  get localAddr(): ConnAddr | undefined {
-    if (this.endPoint.localAddress) {
-      return {
-        transport: "tcp",
-        hostname: this.endPoint.localAddress,
-        port: this.endPoint.localPort ?? 0,
-      };
-    }
-    return undefined;
-  }
-
-  get remoteAddr(): ConnAddr | undefined {
-    if (this.endPoint.remoteAddress) {
-      return {
-        transport: "tcp",
-        hostname: this.endPoint.remoteAddress,
-        port: this.endPoint.remotePort ?? 0,
-      };
-    }
-    return undefined;
-  }
 }
 
 export function nodeSocketToNext(socket: Socket): ConnNext {
-  const connNext = new NodeSocketConnNextWrapper(socket);
-  return connNext;
+  return new NodeSocketConnNext(socket);
 }

package/node/tlsServer.test.ts

@@ -9,7 +9,7 @@ import { MqttServer } from "../server/mod.ts";
 
 logger.level(LogLevel.info);
 
-const { key, cert, caCert } = generateLocalhostCerts();
+const { key, cert, caCert } = await generateLocalhostCerts();
 
 test("Test pubSub using client and server", async () => {
   const server = new TlsServer({ port: 0, key, cert }, {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seriousme/opifex",
-  "version": "2.3.0",
+  "version": "2.4.1",
   "description": "MQTT client & server for Deno & NodeJS and Bun",
   "author": "Hans Klunder",
   "license": "MIT",
@@ -100,8 +100,6 @@
     }
   },
   "devDependencies": {
-    "@types/node": "^26.6.2",
-    "@types/node-forge": "^1.3.14",
-    "node-forge": "^1.4.0"
+    "@types/node": "^26.6.5"
   }
 }

package/web/webSocketToNext.ts

@@ -1,28 +1,77 @@
 import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+import { BaseConnNext } from "../connNext/baseConnNext.ts";
+import { delay } from "../utils/utils.ts";
+
+export class WebSocketConnNext extends BaseConnNext {
+  localAddr = undefined;
+  remoteAddr?: ConnAddr;
+
+  private ws: WebSocket;
+  private messageQueue: Uint8Array[] = [];
+  private maxBufferSize: number;
+  private currentBufferSize = 0;
+  private readResolvers: Array<() => void> = [];
+
+  constructor(
+    ws: WebSocket,
+    remoteAddr?: ConnAddr,
+    maxBufferSize = 1 * 1024 * 1024,
+  ) {
+    super();
+    this.ws = ws;
+    this.remoteAddr = remoteAddr;
+    this.maxBufferSize = maxBufferSize;
+    this.ws.binaryType = "arraybuffer";
+
+    this.ws.addEventListener("message", (event: MessageEvent) => {
+      let chunk: Uint8Array;
+      if (typeof event.data === "string") {
+        chunk = new TextEncoder().encode(event.data);
+      } else if (event.data instanceof ArrayBuffer) {
+        chunk = new Uint8Array(event.data);
+      } else {
+        return;
+      }
 
-export async function webSocketToNext(
-  ws: WebSocket,
-  remoteAddr?: ConnAddr,
-): Promise<ConnNext> {
-  ws.binaryType = "arraybuffer";
+      // Check the bufferlimit (Throttling / Overflow protection)
+      if (this.currentBufferSize + chunk.byteLength > this.maxBufferSize) {
+        this.close();
+        throw new Error(
+          "WebSocket buffer overflow: Too much unread data from push source",
+        );
+      }
 
-  const messageQueue: Uint8Array[] = [];
-  let queueOffset = 0;
-  const readResolvers: Array<() => void> = [];
+      this.messageQueue.push(chunk);
+      this.currentBufferSize += chunk.byteLength;
+      this.notifyWaiters();
+    });
 
-  const notifyWaiters = () => {
-    while (readResolvers.length > 0) {
-      const resolve = readResolvers.shift();
+    const onClose = () => {
+      this.closed = true;
+      this.notifyWaiters();
+    };
+    this.ws.addEventListener("close", onClose);
+    this.ws.addEventListener("error", onClose);
+  }
+
+  private notifyWaiters() {
+    while (this.readResolvers.length > 0) {
+      const resolve = this.readResolvers.shift();
       resolve?.();
     }
-  };
+  }
+
+  private waitForData(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.readResolvers.push(resolve);
+    });
+  }
 
-  // helper to check if websocket is Open
-  const ensureOpen = (): Promise<void> => {
-    if (ws.readyState === WebSocket.OPEN) {
+  ensureOpen(): Promise<void> {
+    if (this.ws.readyState === WebSocket.OPEN) {
       return Promise.resolve();
     }
-    if (ws.readyState >= WebSocket.CLOSING) {
+    if (this.ws.readyState >= WebSocket.CLOSING) {
       return Promise.reject(new Error("WebSocket is closing or closed"));
     }
     return new Promise((resolve, reject) => {
@@ -39,145 +88,59 @@ export async function webSocketToNext(
         reject(new Error("WebSocket closed before opening"));
       };
       const cleanup = () => {
-        ws.removeEventListener("open", onOpen);
-        ws.removeEventListener("error", onError);
-        ws.removeEventListener("close", onClose);
+        this.ws.removeEventListener("open", onOpen);
+        this.ws.removeEventListener("error", onError);
+        this.ws.removeEventListener("close", onClose);
       };
 
-      ws.addEventListener("open", onOpen);
-      ws.addEventListener("error", onError);
-      ws.addEventListener("close", onClose);
+      this.ws.addEventListener("open", onOpen);
+      this.ws.addEventListener("error", onError);
+      this.ws.addEventListener("close", onClose);
     });
-  };
-
-  ws.addEventListener("message", (event: MessageEvent) => {
-    let chunk: Uint8Array;
-    if (typeof event.data === "string") {
-      chunk = new TextEncoder().encode(event.data);
-    } else if (event.data instanceof ArrayBuffer) {
-      chunk = new Uint8Array(event.data);
-    } else {
-      return;
+  }
+
+  protected async pullNextChunk(): Promise<Uint8Array | null> {
+    if (this.messageQueue.length === 0) {
+      await this.waitForData();
     }
-    messageQueue.push(chunk);
-    notifyWaiters();
-  });
-
-  ws.addEventListener("close", notifyWaiters);
-  ws.addEventListener("error", notifyWaiters);
-
-  const getBufferedBytes = (): number => {
-    let total = 0;
-    for (let i = 0; i < messageQueue.length; i++) {
-      total += i === 0
-        ? messageQueue[i]!.byteLength - queueOffset
-        : messageQueue[i]!.byteLength;
+    if (this.closed) {
+      return null;
     }
-    return total;
-  };
-
-  const waitForData = (): Promise<void> => {
-    return new Promise<void>((resolve) => {
-      readResolvers.push(resolve);
-    });
-  };
-
-  const connNext = {
-    remoteAddr,
-    localAddr: undefined,
-
-    async read(length?: number): Promise<Uint8Array | null> {
-      try {
-        await ensureOpen();
-      } catch {
-        return null;
-      }
-
-      if (getBufferedBytes() === 0 && ws.readyState >= WebSocket.CLOSING) {
-        return null;
-      }
-
-      while (getBufferedBytes() === 0) {
-        if (ws.readyState >= WebSocket.CLOSING) return null;
-        await waitForData();
-      }
-
-      if (length === undefined) {
-        const firstChunk = messageQueue.shift()!;
-        const result = firstChunk.subarray(queueOffset);
-        queueOffset = 0;
-        return result;
-      }
-
-      while (getBufferedBytes() < length) {
-        if (ws.readyState >= WebSocket.CLOSING) {
-          return null;
-        }
-        await waitForData();
-      }
-
-      const result = new Uint8Array(length);
-      let bytesCopied = 0;
-
-      while (bytesCopied < length && messageQueue.length > 0) {
-        const currentChunk = messageQueue[0]!;
-        const bytesNeeded = length - bytesCopied;
-        const bytesAvailable = currentChunk.byteLength - queueOffset;
-
-        if (bytesAvailable <= bytesNeeded) {
-          result.set(currentChunk.subarray(queueOffset), bytesCopied);
-          bytesCopied += bytesAvailable;
-          messageQueue.shift();
-          queueOffset = 0;
-        } else {
-          result.set(
-            currentChunk.subarray(queueOffset, queueOffset + bytesNeeded),
-            bytesCopied,
-          );
-          bytesCopied += bytesNeeded;
-          queueOffset += bytesNeeded;
-        }
-      }
-
-      return result;
-    },
-
-    async write(data: Uint8Array): Promise<void> {
-      await ensureOpen();
-
-      if (data instanceof Uint8Array) {
-        ws.send(data.buffer as ArrayBuffer);
-      } else {
-        ws.send(data);
-      }
-
-      const HIGH_WATER_MARK = 64 * 1024;
-      while (ws.bufferedAmount > HIGH_WATER_MARK) {
-        if (ws.readyState >= WebSocket.CLOSING) {
-          throw new Error("WebSocket closed while writing");
-        }
-        await new Promise((resolve) => setTimeout(resolve, 10));
-      }
-    },
+    const chunk = this.messageQueue.shift()!;
+    this.currentBufferSize -= chunk.byteLength;
+    return chunk;
+  }
+
+  async write(data: Uint8Array): Promise<void> {
+    if (this.isClosed()) throw new Error("WebSocket is closed");
+    this.ws.send(data.buffer as ArrayBuffer);
+
+    // Check WebSocket bufferedAmount for write-backpressure
+    const HIGH_WATER_MARK = 64 * 1024;
+    while (this.ws.bufferedAmount > HIGH_WATER_MARK) {
+      if (this.isClosed()) throw new Error("WebSocket closed while writing");
+      await delay(10);
+    }
+  }
 
-    close(): void {
+  close(): void {
+    if (!this.closed) {
+      this.closed = true;
       try {
-        ws.close();
+        this.ws.close();
       } catch {
         /* swallow */
       }
-      notifyWaiters();
-    },
-
-    async *[Symbol.asyncIterator]() {
-      while (true) {
-        const chunk = await this.read();
-        if (chunk === null) break;
-        yield chunk;
-      }
-    },
-  };
+    }
+  }
+}
 
-  await ensureOpen();
+export async function webSocketToNext(
+  ws: WebSocket,
+  remoteAddr?: ConnAddr,
+  maxBufferSize?: number,
+): Promise<ConnNext> {
+  const connNext = new WebSocketConnNext(ws, remoteAddr, maxBufferSize);
+  await connNext.ensureOpen();
   return connNext;
 }

package/web/wsClient.ts

@@ -16,7 +16,10 @@ import { webSocketToNext } from "./webSocketToNext.ts";
  */
 export class WsClient extends Client {
   /** create a websocket and wrap it in a ConnNext */
-  private async establishSocket(url: string): Promise<ConnNext> {
+  private async establishSocket(
+    url: string,
+    maxBufferSize?: number,
+  ): Promise<ConnNext> {
     try {
       const urlObj = new URL(url);
       const remoteAddr: NetAddr = {
@@ -25,7 +28,7 @@ export class WsClient extends Client {
         port: Number(urlObj.port || 0),
       };
       const ws = new WebSocket(url);
-      const connNext = await webSocketToNext(ws, remoteAddr);
+      const connNext = await webSocketToNext(ws, remoteAddr, maxBufferSize);
       return connNext;
     } catch (error) {
       const message = error instanceof ErrorEvent
@@ -40,10 +43,14 @@ export class WsClient extends Client {
     const { protocol, hostname, port: portStr, pathname } = this.connectUrl;
     const port = portStr ? Number(portStr) : (protocol === "wss:" ? 443 : 80);
     const path = pathname || "/mqtt";
+    const maxBufferSize = this.maxBufferSize;
 
     logger.debug({ hostname, port, path });
     if (protocol === "ws:" || protocol === "wss:") {
-      return this.establishSocket(`${protocol}//${hostname}:${port}${path}`);
+      return this.establishSocket(
+        `${protocol}//${hostname}:${port}${path}`,
+        maxBufferSize,
+      );
     }
     throw new Error(`Unsupported protocol: ${protocol}`);
   }