@seriousme/opifex 2.3.0 → 2.4.0

package/client/client.ts

@@ -51,6 +51,8 @@ export type ConnectParameters = {
   key?: string | undefined;
   /** number of retries before stopping connection attempts */
   numberOfRetries?: number | undefined;
+  /** maximum size of receive buffer, only used by webSockets for now */
+  maxBufferSize?: number | undefined;
   /** connect options */
   options?: ConnectOptions | undefined;
 };
@@ -159,6 +161,8 @@ export class Client {
   protected key?: string | undefined;
   /** the MQTT clientId */
   protected clientId: string;
+  /** the maximum number of bytes buffered by a ws/wss client */
+  protected maxBufferSize: number | undefined;
   private ctx: Context;
   private connectPacket?: ConnectPacket;
 
@@ -310,6 +314,7 @@ export class Client {
     this.caCerts = params?.caCerts;
     this.cert = params?.cert;
     this.key = params?.key;
+    this.maxBufferSize = params?.maxBufferSize;
     const options = Object.assign(
       {
         keepAlive: this.keepAlive,

package/connNext/baseConnNext.ts

@@ -0,0 +1,123 @@
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+
+export abstract class BaseConnNext implements ConnNext {
+  abstract remoteAddr?: ConnAddr;
+  abstract localAddr?: ConnAddr;
+
+  protected chunks: Uint8Array[] = [];
+  protected queueOffset = 0;
+  protected closed = false;
+  private isPulling = false;
+
+  abstract write(data: Uint8Array): Promise<void>;
+  abstract close(): void;
+
+  /** Specifiek per bron: haal pas daadwerkelijk 1 chunk op als de basisklasse erom vraagt */
+  protected abstract pullNextChunk(): Promise<Uint8Array | null>;
+
+  isClosed(): boolean {
+    return this.closed;
+  }
+
+  protected getBufferedBytes(): number {
+    let total = 0;
+    for (let i = 0; i < this.chunks.length; i++) {
+      total += i === 0
+        ? this.chunks[i]!.byteLength - this.queueOffset
+        : this.chunks[i]!.byteLength;
+    }
+    return total;
+  }
+
+  /** Zorgt ervoor dat we data ophalen op exact het moment dat het nodig is */
+  private async ensureData(length?: number): Promise<void> {
+    while (
+      !this.closed &&
+      (this.getBufferedBytes() === 0 ||
+        (length !== undefined && this.getBufferedBytes() < length))
+    ) {
+      if (this.isPulling) {
+        // Voorkom gelijktijdige pulls als er al eentje loopt
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        continue;
+      }
+
+      this.isPulling = true;
+      try {
+        const chunk = await this.pullNextChunk();
+        if (chunk === null || chunk.byteLength === 0) {
+          this.closed = true;
+          break;
+        }
+        this.chunks.push(chunk);
+      } catch {
+        this.closed = true;
+      } finally {
+        this.isPulling = false;
+      }
+    }
+  }
+
+  async read(length?: number): Promise<Uint8Array | null> {
+    if (this.closed && this.getBufferedBytes() === 0) {
+      return null;
+    }
+
+    if (length === 0) {
+      return new Uint8Array(0);
+    }
+
+    // Pull exact zolang tot we genoeg data hebben voor deze read()
+    await this.ensureData(length);
+
+    if (this.getBufferedBytes() === 0) {
+      return null;
+    }
+
+    // Onbegrensd lezen (geen lengte opgegeven)
+    if (length === undefined) {
+      const firstChunk = this.chunks.shift()!;
+      const result = firstChunk.subarray(this.queueOffset);
+      this.queueOffset = 0;
+      return result;
+    }
+
+    // Exacte lengte uitlezen uit de buffer
+    const bytesToTake = Math.min(length, this.getBufferedBytes());
+    const result = new Uint8Array(bytesToTake);
+    let bytesCopied = 0;
+
+    while (bytesCopied < bytesToTake && this.chunks.length > 0) {
+      const currentChunk = this.chunks[0]!;
+      const bytesNeeded = bytesToTake - bytesCopied;
+      const bytesAvailable = currentChunk.byteLength - this.queueOffset;
+
+      if (bytesAvailable <= bytesNeeded) {
+        result.set(currentChunk.subarray(this.queueOffset), bytesCopied);
+        bytesCopied += bytesAvailable;
+        this.chunks.shift();
+        this.queueOffset = 0;
+      } else {
+        result.set(
+          currentChunk.subarray(
+            this.queueOffset,
+            this.queueOffset + bytesNeeded,
+          ),
+          bytesCopied,
+        );
+        bytesCopied += bytesNeeded;
+        this.queueOffset += bytesNeeded;
+      }
+    }
+
+    return bytesCopied === 0 ? null : result;
+  }
+
+  async *[Symbol.asyncIterator]() {
+    while (true) {
+      const chunk = await this.read();
+      if (chunk === null) break;
+      yield chunk;
+    }
+  }
+}

package/deno/wsServer.ts

@@ -23,8 +23,9 @@ async function handleWsClient(
   socket: WebSocket,
   mqttServer: MqttServer,
   remoteAddr: NetAddr,
+  maxBufferSize: number | undefined,
 ) {
-  const conn = await webSocketToNext(socket, remoteAddr);
+  const conn = await webSocketToNext(socket, remoteAddr, maxBufferSize);
   mqttServer.serve(conn);
   logger.debug("serving mqtt");
 }
@@ -33,6 +34,9 @@ export function mqttOverWS(
   mqttServer: MqttServer,
   pathName: string,
   next: RequestHandler,
+  opts?: {
+    maxBufferSize?: number | undefined;
+  },
 ): RequestHandler {
   const wsRequestHandler: RequestHandler = (req, info) => {
     const url = new URL(req.url);
@@ -45,7 +49,7 @@ export function mqttOverWS(
       const { socket, response } = Deno.upgradeWebSocket(req);
       logger.debug("upgraded to websocket");
       // do not await on purpose
-      handleWsClient(socket, mqttServer, remoteAddr);
+      handleWsClient(socket, mqttServer, remoteAddr, opts?.maxBufferSize);
       return response;
     }
     return next(req, info);
@@ -65,6 +69,7 @@ export class WsServer {
     hostname?: string;
   };
   private pathName: string;
+  private maxBufferSize: number | undefined;
 
   /**
    * Create a new WebSocket server
@@ -74,12 +79,14 @@ export class WsServer {
       port: number;
       hostname?: string;
       pathName?: string;
+      maxBufferSize?: number;
     },
     mqttOptions: MqttServerOptions | MqttServer,
   ) {
-    const { pathName, ...listenOptions } = serverOptions;
+    const { pathName, maxBufferSize, ...listenOptions } = serverOptions;
     this.listenOptions = listenOptions;
     this.pathName = pathName ?? "/mqtt";
+    this.maxBufferSize = maxBufferSize;
     if (mqttOptions instanceof MqttServer) {
       this.mqttServer = mqttOptions;
     } else {
@@ -95,6 +102,7 @@ export class WsServer {
       this.mqttServer,
       this.pathName,
       pageNotFound,
+      { maxBufferSize: this.maxBufferSize },
     );
     this.server = Deno.serve(this.listenOptions, requestHandler);
     await this.server.finished;

package/dev_utils/WebStreamEndpoint.ts

@@ -0,0 +1,5 @@
+export interface WebStreamEndpoint {
+  readable: ReadableStream<Uint8Array>;
+  writable: WritableStream<Uint8Array>;
+  close: () => void;
+}

package/dev_utils/asyncStreamEndpoint.ts

@@ -0,0 +1,5 @@
+export interface AsyncStreamEndpoint {
+  readable: AsyncIterable<Uint8Array>;
+  write: (chunk: Uint8Array) => Promise<void>;
+  close: () => void;
+}

package/dev_utils/asyncStreamPair.ts

@@ -2,14 +2,9 @@
  *  Duplex AsyncStream
  */
 
-export interface AsyncStreamPairEndpoint {
-  readable: AsyncIterable<Uint8Array>;
-  write: (chunk: Uint8Array) => Promise<void>;
-  close: () => void;
-}
-
+import type { AsyncStreamEndpoint } from "./asyncStreamEndpoint.ts";
 /**
- * Interne push/pull queue to support AsyncIterable iteration.
+ * Internal push/pull queue to support AsyncIterable iteration.
  */
 class AsyncQueue<T> implements AsyncIterable<T> {
   private queue: T[] = [];
@@ -75,8 +70,8 @@ class AsyncQueue<T> implements AsyncIterable<T> {
 }
 
 export function createAsyncStreamPair(): {
-  input: AsyncStreamPairEndpoint;
-  output: AsyncStreamPairEndpoint;
+  input: AsyncStreamEndpoint;
+  output: AsyncStreamEndpoint;
 } {
   const queueA = new AsyncQueue<Uint8Array>();
   const queueB = new AsyncQueue<Uint8Array>();

package/dev_utils/asyncStreamPairToNext.ts

@@ -0,0 +1,45 @@
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+import { createAsyncStreamPair } from "./asyncStreamPair.ts";
+import { asyncStreamToNext } from "./asyncStreamToNext.ts";
+
+export function createConnNextPair(): {
+  input: ConnNext;
+  output: ConnNext;
+} {
+  const { input: asyncInput, output: asyncOutput } = createAsyncStreamPair();
+
+  const mockAddr: ConnAddr = {
+    transport: "tcp",
+    hostname: "127.0.0.1",
+    port: 1883,
+  };
+
+  const inputWrapper = asyncStreamToNext(asyncInput);
+  const outputWrapper = asyncStreamToNext(asyncOutput);
+
+  const input: ConnNext = {
+    remoteAddr: mockAddr,
+    localAddr: mockAddr,
+    read: (length: number | undefined) => inputWrapper.read(length),
+    write: (data: Uint8Array<ArrayBufferLike>) => inputWrapper.write(data),
+    [Symbol.asyncIterator]: () => inputWrapper[Symbol.asyncIterator](),
+    close: () => {
+      inputWrapper.close();
+      outputWrapper.close();
+    },
+  };
+
+  const output: ConnNext = {
+    remoteAddr: mockAddr,
+    localAddr: mockAddr,
+    read: (length: number | undefined) => outputWrapper.read(length),
+    write: (data: Uint8Array<ArrayBufferLike>) => outputWrapper.write(data),
+    [Symbol.asyncIterator]: () => outputWrapper[Symbol.asyncIterator](),
+    close: () => {
+      outputWrapper.close();
+      inputWrapper.close();
+    },
+  };
+
+  return { input, output };
+}

package/dev_utils/asyncStreamToNext.ts

@@ -1,171 +1,64 @@
-import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
-import { createAsyncStreamPair } from "./asyncStreamPair.ts";
-import type { AsyncStreamPairEndpoint } from "./asyncStreamPair.ts";
+import { BaseConnNext } from "../connNext/baseConnNext.ts";
+import type { AsyncStreamEndpoint } from "./asyncStreamEndpoint.ts";
 
-/** wrapper class */
-export class AsyncStreamConnNextWrapper {
-  closed: boolean;
-  private endPoint: AsyncStreamPairEndpoint;
-  private leftover: Uint8Array = new Uint8Array(0);
+export class AsyncStreamConnNext extends BaseConnNext {
+  remoteAddr = undefined;
+  localAddr = undefined;
   private reader: AsyncIterator<Uint8Array>;
+  private endPoint: AsyncStreamEndpoint;
 
-  /** Create a new Endpoint*/
-  constructor(endPoint: AsyncStreamPairEndpoint) {
+  constructor(endPoint: AsyncStreamEndpoint) {
+    super();
     this.endPoint = endPoint;
-    this.closed = false;
+    // Haal de async iterator op uit de readable stream
     this.reader = endPoint.readable[Symbol.asyncIterator]();
   }
 
   /**
-   * Writes data chunk to the underlying AsyncStream
-   */ async write(chunk: Uint8Array): Promise<void> {
-    if (this.closed) {
-      return Promise.reject(new Error("Pipe closed"));
-    }
-    try {
-      await this.endPoint.write(chunk);
-      return;
-    } catch (err) {
-      this.closed = true;
-      throw err;
-    }
-  }
-
-  /**
-   * Reads data from the stream.
-   * If `length` is provided, waits until exact number of bytes are buffered.
+   * Haalt op exact moment van aanvragen één nieuwe chunk op via de iterator.
    */
-  async read(length?: number): Promise<Uint8Array | null> {
-    if (this.closed || (length !== undefined && length < 0)) {
+  protected async pullNextChunk(): Promise<Uint8Array | null> {
+    if (this.closed) {
       return null;
     }
 
-    if (length === 0) {
-      return new Uint8Array(0);
-    }
-
-    // Helper to fetch the next chunk from the underlying iterable
-    const fetchNextChunk = async (): Promise<Uint8Array | null> => {
+    try {
       const { value, done } = await this.reader.next();
-      if (done) {
+      if (done || !value) {
         this.closed = true;
         return null;
       }
       return value;
-    };
-
-    // Unbounded read (length === undefined)
-    if (length === undefined) {
-      if (this.leftover.length > 0) {
-        const result = this.leftover;
-        this.leftover = new Uint8Array(0);
-        return result;
-      }
-      return await fetchNextChunk();
+    } catch {
+      this.closed = true;
+      return null;
     }
+  }
 
-    // Exact length read
-    const bytesRequested = length;
-    const result = new Uint8Array(bytesRequested);
-    let bytesAccumulated = 0;
-
-    // Use leftover data first if available
-    if (this.leftover.length > 0) {
-      const take = Math.min(this.leftover.length, bytesRequested);
-      result.set(this.leftover.subarray(0, take), bytesAccumulated);
-      bytesAccumulated += take;
-      this.leftover = this.leftover.subarray(take);
+  async write(chunk: Uint8Array): Promise<void> {
+    if (this.closed) {
+      return Promise.reject(new Error("Pipe closed"));
     }
-
-    // Read remaining required bytes
-    while (bytesAccumulated < bytesRequested) {
-      const value = await fetchNextChunk();
-
-      if (value === null) {
-        // Stream ended before collecting required bytes
-        if (bytesAccumulated === 0) return null;
-        return result.subarray(0, bytesAccumulated);
-      }
-
-      const needed = bytesRequested - bytesAccumulated;
-
-      if (value.length <= needed) {
-        result.set(value, bytesAccumulated);
-        bytesAccumulated += value.length;
-      } else {
-        result.set(value.subarray(0, needed), bytesAccumulated);
-        bytesAccumulated += needed;
-        this.leftover = value.subarray(needed);
-      }
+    try {
+      await this.endPoint.write(chunk);
+    } catch (err) {
+      this.closed = true;
+      throw err;
     }
-
-    return result;
   }
 
-  /**
-   * Closes the reader, writer, and underlying AsyncStream pair endpoint.
-   */
   close(): void {
     if (!this.closed) {
       this.closed = true;
       try {
         this.endPoint.close();
-      } catch (_err) { /* swallow */ }
-    }
-  }
-  async *[Symbol.asyncIterator]() {
-    while (true) {
-      const chunk = await this.read();
-      if (chunk === null) {
-        break;
+      } catch (_err) {
+        /* swallow */
       }
-      yield chunk;
     }
   }
 }
 
-/**
- * Creates a pair of cross-connected ConnNext sockets backed by AsyncStreams.
- * Offers the exact same API as `createConnNextPair`.
- */
-export function createConnNextPair(): {
-  input: ConnNext;
-  output: ConnNext;
-} {
-  const { input: asyncInput, output: asyncOutput } = createAsyncStreamPair();
-
-  const mockAddr: ConnAddr = {
-    transport: "tcp",
-    hostname: "127.0.0.1",
-    port: 1883,
-  };
-
-  const inputWrapper = new AsyncStreamConnNextWrapper(asyncInput);
-  const outputWrapper = new AsyncStreamConnNextWrapper(asyncOutput);
-
-  const input: ConnNext = {
-    remoteAddr: mockAddr,
-    localAddr: mockAddr,
-    read: (length) => inputWrapper.read(length),
-    write: (data) => inputWrapper.write(data),
-    [Symbol.asyncIterator]: () => inputWrapper[Symbol.asyncIterator](),
-    close: () => {
-      inputWrapper.close();
-      outputWrapper.close();
-    },
-  };
-
-  const output: ConnNext = {
-    remoteAddr: mockAddr,
-    localAddr: mockAddr,
-    read: (length) => outputWrapper.read(length),
-    write: (data) => outputWrapper.write(data),
-    [Symbol.asyncIterator]: () => outputWrapper[Symbol.asyncIterator](),
-    close: () => {
-      outputWrapper.close();
-      inputWrapper.close();
-    },
-  };
-
-  return { input, output };
+export function asyncStreamToNext(endPoint: AsyncStreamEndpoint) {
+  return new AsyncStreamConnNext(endPoint);
 }

package/dev_utils/connNextPair.test.ts

@@ -1,7 +1,7 @@
 import { test } from "node:test";
 import assert from "node:assert";
-import { createConnNextPair as asyncPair } from "./asyncStreamToNext.ts";
-import { createConnNextPair as webPair } from "./webStreamToNext.ts";
+import { createConnNextPair as asyncPair } from "./asyncStreamPairToNext.ts";
+import { createConnNextPair as webPair } from "./webStreamPairToNext.ts";
 
 const pairs = {
   asyncStream: asyncPair,

package/dev_utils/mockConn.ts

@@ -1,4 +1,6 @@
 import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+import { createAsyncStreamPair } from "./asyncStreamPair.ts";
+import { AsyncStreamConnNext } from "./asyncStreamToNext.ts";
 
 /**
  * Creates an in-memory mock ConnNext object for unit testing.
@@ -12,21 +14,8 @@ export function createMockConn(
   writerBuf: Uint8Array,
   close = () => {},
 ): ConnNext {
-  let readIndex = 0;
-  let readOffset = 0;
-  let writerPos = 0;
-  let isClosed = false;
-
-  // Helper function to calculate total remaining bytes across readerBuffs
-  const getRemainingBytes = (): number => {
-    let total = 0;
-    for (let i = readIndex; i < readerBuffs.length; i++) {
-      total += i === readIndex
-        ? readerBuffs[i].byteLength - readOffset
-        : readerBuffs[i].byteLength;
-    }
-    return total;
-  };
+  // Maak een async stream pair aan
+  const { input, output } = createAsyncStreamPair();
 
   const mockAddr: ConnAddr = {
     transport: "tcp",
@@ -34,89 +23,44 @@ export function createMockConn(
     port: 1883,
   };
 
-  return {
-    remoteAddr: mockAddr,
-    localAddr: mockAddr,
-
-    /**
-     * Read exactly 'length' bytes from the provided readerBuffs
-     */
-    read(length?: number): Promise<Uint8Array | null> {
-      if (isClosed) return Promise.resolve(null);
-
-      // No specific length requested: return the next available chunk immediately
-      if (length === undefined) {
-        if (readIndex >= readerBuffs.length) Promise.resolve(null);
-        const chunk = readerBuffs[readIndex].subarray(readOffset);
-        readIndex++;
-        readOffset = 0;
-        Promise.resolve(chunk);
-      }
-
-      length = length ?? 0;
-
-      // EOF reached / not enough remaining bytes in the mock buffers
-      if (getRemainingBytes() < length) {
-        return Promise.resolve(null);
-      }
-
-      const result = new Uint8Array(length);
-      let bytesCopied = 0;
-
-      while (bytesCopied < length && readIndex < readerBuffs.length) {
-        const current = readerBuffs[readIndex];
-        const available = current.byteLength - readOffset;
-        const needed = length - bytesCopied;
-
-        if (available <= needed) {
-          result.set(current.subarray(readOffset), bytesCopied);
-          bytesCopied += available;
-          readIndex++;
-          readOffset = 0;
-        } else {
-          result.set(
-            current.subarray(readOffset, readOffset + needed),
-            bytesCopied,
-          );
-          bytesCopied += needed;
-          readOffset += needed;
-        }
+  // Push all input directly to the input of the StreamPair
+  (async () => {
+    try {
+      for (const buf of readerBuffs) {
+        await input.write(buf);
       }
+      input.close();
+    } catch {
+      // ignore errors
+    }
+  })();
 
-      return Promise.resolve(result);
-    },
-
-    /**
-     * Write data directly at the current offset in writerBuf
-     */
-    write(data: Uint8Array): Promise<void> {
-      if (isClosed) {
-        throw new Error("Connection closed");
+  // Capture written data and store it in the write buffer
+  (async () => {
+    try {
+      let writerPos = 0;
+      for await (const chunk of input.readable) {
+        writerBuf.set(chunk, writerPos);
+        writerPos += chunk.byteLength;
       }
-      writerBuf.set(data, writerPos);
-      writerPos += data.byteLength;
-      return Promise.resolve();
-    },
+    } catch {
+      // ignore errors
+    }
+  })();
 
-    /**
-     * Close the mock connection
-     */
-    close(): void {
-      if (!isClosed) {
-        isClosed = true;
-        close();
-      }
-    },
+  // Return a ConnNext wrapper connected to the output side
+  const wrapper = new AsyncStreamConnNext(output);
 
-    /**
-     * Async iterator for streaming test scenarios
-     */
-    async *[Symbol.asyncIterator]() {
-      while (true) {
-        const chunk = await this.read();
-        if (chunk === null) break;
-        yield chunk;
-      }
+  return {
+    remoteAddr: mockAddr,
+    localAddr: mockAddr,
+    read: (length) => wrapper.read(length),
+    write: (data) => wrapper.write(data),
+    [Symbol.asyncIterator]: () => wrapper[Symbol.asyncIterator](),
+    close: () => {
+      wrapper.close();
+      output.close();
+      close();
     },
   };
 }

package/dev_utils/mockServer.ts

@@ -1,4 +1,4 @@
-import { createConnNextPair } from "./webStreamToNext.ts";
+import { createConnNextPair } from "./asyncStreamPairToNext.ts";
 import { MqttConn } from "../mqttConn/mqttConn.ts";
 import { Context, MqttServer } from "../server/mod.ts";
 import { handlers } from "./test-handlers.ts";

package/dev_utils/webStreamPair.ts

@@ -1,12 +1,8 @@
-export interface WebStreamPairEndpoint {
-  readable: ReadableStream<Uint8Array>;
-  writable: WritableStream<Uint8Array>;
-  close: () => void;
-}
+import type { WebStreamEndpoint } from "./WebStreamEndpoint.ts";
 
 export function createWebStreamPair(): {
-  input: WebStreamPairEndpoint;
-  output: WebStreamPairEndpoint;
+  input: WebStreamEndpoint;
+  output: WebStreamEndpoint;
 } {
   let aController: ReadableStreamDefaultController | null = null;
   let bController: ReadableStreamDefaultController | null = null;