@seriousme/opifex 2.2.0 → 2.4.0

package/README.md

@@ -111,10 +111,11 @@ can be found in the [examples](/examples/) folder.
    encode and decode packets.
 
 2. On top of mqttPacket sits the MQTT connection module
-   ([mqttConn/mod.ts](mqttConn/mod.ts)) that reads packets from a Readable
-   stream and writes them to Writable stream. It will take care of incomplete
-   and/or malformed packets. mqttConn provides an async iterable that can be
-   awaited for new packets.
+   ([mqttConn/mod.ts](mqttConn/mod.ts)) that reads packets from and writes them
+   to a [ConnNext](connNext) adapter. It will take care of incomplete and/or
+   malformed packets. mqttConn provides an async iterable that can be awaited
+   for new packets. A ConnNext adapter per transport provides async
+   read/write/close/address functionality on top of the various transports.
 
 3. On top of mqttConn live the MQTT server ([server/mod.ts](server/mod.ts)) and
    MQTT client ([client/mod.ts](client/mod.ts)) that take care of the MQTT
@@ -122,8 +123,8 @@ can be found in the [examples](/examples/) folder.
    another type of packet will be accepted. Both follow a similar model of
    implementation where for each packet that is received a handler is invoked
    which then triggers the next step. Server and client are totally independent
-   of the technical implementation of the connection and only need a socketConn
-   ([socket](socket)) to be able to work.
+   of the technical implementation of the connection and only need a
+   [ConnNext](connNext) connection to be able to work.
 
 4. As the server needs to be able to serve multiple clients at the same time, it
    maintains a context ([server/context.ts](server/context.ts)) per client to

package/bin/mqtt.ts

@@ -278,7 +278,7 @@ async function publish() {
       qos: parseQos(publishArgs.qos),
     });
     logger.debug("Published!");
-    client.disconnect();
+    await client.disconnect();
     logger.debug("Disconnected !");
   } catch (err) {
     if (err instanceof Error) {

package/client/client.ts

@@ -10,9 +10,9 @@ import {
 import type {
   AuthenticationResult,
   ConnectPacket,
+  ConnNext,
   ProtocolLevel,
   PublishPacket,
-  SockConn,
   Store,
   SubscribePacket,
 } from "./deps.ts";
@@ -51,6 +51,8 @@ export type ConnectParameters = {
   key?: string | undefined;
   /** number of retries before stopping connection attempts */
   numberOfRetries?: number | undefined;
+  /** maximum size of receive buffer, only used by webSockets for now */
+  maxBufferSize?: number | undefined;
   /** connect options */
   options?: ConnectOptions | undefined;
 };
@@ -159,6 +161,8 @@ export class Client {
   protected key?: string | undefined;
   /** the MQTT clientId */
   protected clientId: string;
+  /** the maximum number of bytes buffered by a ws/wss client */
+  protected maxBufferSize: number | undefined;
   private ctx: Context;
   private connectPacket?: ConnectPacket;
 
@@ -169,7 +173,6 @@ export class Client {
     const cStore = store ? store : new MemoryStore();
     this.ctx = new Context(cStore, this);
     this.clientId = generateClientId(this.clientIdPrefix);
-    this.numberOfRetries = DEFAULT_RETRIES;
   }
 
   /**
@@ -188,14 +191,58 @@ export class Client {
     return this.connectUrl;
   }
 
+  /**
+   * Placeholder for MQTT, override in subclass
+   * @param _hostname
+   * @param _port
+   */
+  protected connectMQTT(
+    _hostname: string,
+    _port: number,
+  ): Promise<ConnNext> {
+    throw new Error(`Unsupported protocol: mqtt`);
+  }
+
+  /**
+   * Connect using MQTT over TLS
+   * @param hostname the name of the host to connect to
+   * @param port the port number
+   * @param caCerts the CA certificates to trust
+   * @param cert the clients certificate
+   * @param key the clients private key
+   * @returns a ConnNext object
+   */
+  protected connectMQTTS(
+    _hostname: string,
+    _port: number,
+    _caCerts?: string[],
+    _cert?: string,
+    _key?: string,
+  ): Promise<ConnNext> {
+    throw new Error(`Unsupported protocol: mqtts`);
+  }
+
   /**
    * Creates a new connection to the MQTT broker
-   * @returns Promise resolving to a SockConn connection
+   * Override this in a subclass if you need other protocols
+   * @returns Promise resolving to a ConnNext connection
    */
-  protected createConn(): Promise<SockConn> {
+  protected createConn(): Promise<ConnNext> {
     // if you need to support alternative connection types just
     // overload this method in your subclass
-    throw new Error(`Unsupported protocol: ${this.url.protocol}`);
+    const { protocol, hostname, port: portStr } = this.connectUrl;
+    const port = Number(portStr);
+    const caCerts = this.caCerts;
+    const cert = this.cert;
+    const key = this.key;
+
+    if (protocol === "mqtts:") {
+      return this.connectMQTTS(hostname, port, caCerts, cert, key);
+    }
+    if (protocol === "mqtt:") {
+      return this.connectMQTT(hostname, port);
+    }
+    throw new Error(`Unsupported protocol: ${protocol}`);
   }
 
   /**
@@ -233,7 +280,14 @@ export class Client {
         logger.debug({ lastMessage });
       }
 
-      if (tryConnect && (isReconnect || attempt < this.numberOfRetries)) {
+      logger.debug("tryConnect", {
+        tryConnect,
+        isReconnect,
+        attempt,
+        retries: this.numberOfRetries,
+      });
+
+      if (tryConnect && (isReconnect || (attempt < this.numberOfRetries))) {
         await backOffSleep(true, attempt);
         if (!isReconnect) {
           attempt++;
@@ -256,10 +310,11 @@ export class Client {
    */
   connect(params: ConnectParameters = {}): Promise<AuthenticationResult> {
     this.connectUrl = params?.url || this.connectUrl;
-    this.numberOfRetries = params.numberOfRetries || this.numberOfRetries;
+    this.numberOfRetries = params.numberOfRetries ?? this.numberOfRetries;
     this.caCerts = params?.caCerts;
     this.cert = params?.cert;
     this.key = params?.key;
+    this.maxBufferSize = params?.maxBufferSize;
     const options = Object.assign(
       {
         keepAlive: this.keepAlive,

package/client/context.ts

@@ -10,12 +10,12 @@ import type {
   AnyPacket,
   AuthenticationResult,
   ConnectPacket,
+  ConnNext,
   PacketId,
   ProtocolLevel,
   PublishPacket,
   ReasonCode,
   ReturnCodes,
-  SockConn,
   Store,
   SubscribePacket,
   UnsubscribePacket,
@@ -169,7 +169,7 @@ export class Context {
   }
 
   async handleConnection(
-    conn: SockConn,
+    conn: ConnNext,
     connectPacket: ConnectPacket,
   ): Promise<boolean> {
     this.mqttConn = new MqttConn({ conn });

package/client/deps.ts

@@ -36,12 +36,12 @@ export {
 
 export { MqttConn } from "../mqttConn/mqttConn.ts";
 export type {
+  ConnAddr,
+  ConnNext,
   NetAddr,
-  SockAddr,
-  SockConn,
   UnixAddr,
   VsockAddr,
-} from "../socket/socket.ts";
+} from "../connNext/connNext.ts";
 export { Timer } from "../timer/timer.ts";
 export { BufferedAsyncIterable, Deferred, logger } from "../utils/mod.ts";
 export { MemoryStore } from "./store/memoryStore.ts";

package/client/mod.ts

@@ -11,11 +11,11 @@ export {
 export { logger, MQTTLevel, ReasonCode } from "./deps.ts";
 export type {
   AuthenticationResult,
+  ConnAddr,
+  ConnNext,
   NetAddr,
   ProtocolLevel,
   QoS,
-  SockAddr,
-  SockConn,
   UnixAddr,
   VsockAddr,
 } from "./deps.ts";

package/connNext/baseConnNext.ts

@@ -0,0 +1,123 @@
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+
+export abstract class BaseConnNext implements ConnNext {
+  abstract remoteAddr?: ConnAddr;
+  abstract localAddr?: ConnAddr;
+
+  protected chunks: Uint8Array[] = [];
+  protected queueOffset = 0;
+  protected closed = false;
+  private isPulling = false;
+
+  abstract write(data: Uint8Array): Promise<void>;
+  abstract close(): void;
+
+  /** Specifiek per bron: haal pas daadwerkelijk 1 chunk op als de basisklasse erom vraagt */
+  protected abstract pullNextChunk(): Promise<Uint8Array | null>;
+
+  isClosed(): boolean {
+    return this.closed;
+  }
+
+  protected getBufferedBytes(): number {
+    let total = 0;
+    for (let i = 0; i < this.chunks.length; i++) {
+      total += i === 0
+        ? this.chunks[i]!.byteLength - this.queueOffset
+        : this.chunks[i]!.byteLength;
+    }
+    return total;
+  }
+
+  /** Zorgt ervoor dat we data ophalen op exact het moment dat het nodig is */
+  private async ensureData(length?: number): Promise<void> {
+    while (
+      !this.closed &&
+      (this.getBufferedBytes() === 0 ||
+        (length !== undefined && this.getBufferedBytes() < length))
+    ) {
+      if (this.isPulling) {
+        // Voorkom gelijktijdige pulls als er al eentje loopt
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        continue;
+      }
+
+      this.isPulling = true;
+      try {
+        const chunk = await this.pullNextChunk();
+        if (chunk === null || chunk.byteLength === 0) {
+          this.closed = true;
+          break;
+        }
+        this.chunks.push(chunk);
+      } catch {
+        this.closed = true;
+      } finally {
+        this.isPulling = false;
+      }
+    }
+  }
+
+  async read(length?: number): Promise<Uint8Array | null> {
+    if (this.closed && this.getBufferedBytes() === 0) {
+      return null;
+    }
+
+    if (length === 0) {
+      return new Uint8Array(0);
+    }
+
+    // Pull exact zolang tot we genoeg data hebben voor deze read()
+    await this.ensureData(length);
+
+    if (this.getBufferedBytes() === 0) {
+      return null;
+    }
+
+    // Onbegrensd lezen (geen lengte opgegeven)
+    if (length === undefined) {
+      const firstChunk = this.chunks.shift()!;
+      const result = firstChunk.subarray(this.queueOffset);
+      this.queueOffset = 0;
+      return result;
+    }
+
+    // Exacte lengte uitlezen uit de buffer
+    const bytesToTake = Math.min(length, this.getBufferedBytes());
+    const result = new Uint8Array(bytesToTake);
+    let bytesCopied = 0;
+
+    while (bytesCopied < bytesToTake && this.chunks.length > 0) {
+      const currentChunk = this.chunks[0]!;
+      const bytesNeeded = bytesToTake - bytesCopied;
+      const bytesAvailable = currentChunk.byteLength - this.queueOffset;
+
+      if (bytesAvailable <= bytesNeeded) {
+        result.set(currentChunk.subarray(this.queueOffset), bytesCopied);
+        bytesCopied += bytesAvailable;
+        this.chunks.shift();
+        this.queueOffset = 0;
+      } else {
+        result.set(
+          currentChunk.subarray(
+            this.queueOffset,
+            this.queueOffset + bytesNeeded,
+          ),
+          bytesCopied,
+        );
+        bytesCopied += bytesNeeded;
+        this.queueOffset += bytesNeeded;
+      }
+    }
+
+    return bytesCopied === 0 ? null : result;
+  }
+
+  async *[Symbol.asyncIterator]() {
+    while (true) {
+      const chunk = await this.read();
+      if (chunk === null) break;
+      yield chunk;
+    }
+  }
+}

package/connNext/connNext.ts

@@ -0,0 +1,44 @@
+/** Net addresses */
+export type NetAddr = {
+  /** the IP-based transport protocol */
+  transport: "tcp" | "udp";
+  /** the hostname */
+  hostname: string;
+  /** the port number */
+  port: number;
+};
+/** Unix addresses */
+export type UnixAddr = {
+  /** the Unix domain socket transport protocol */
+  transport: "unix" | "unixpacket";
+  /** the file system path to the socket to connect to */
+  path: string;
+};
+/** vSock addresses */
+export type VsockAddr = {
+  /** the VSOCK transport protocol */
+  transport: "vsock";
+  /** the context identifier (CID) of the peer */
+  cid: number;
+  /** the port number */
+  port: number;
+};
+
+/** connection addresses */
+export type ConnAddr = NetAddr | UnixAddr | VsockAddr | undefined;
+/** connection descriptor. */
+
+export type ConnNext = {
+  /** the "nextStream" to read from */
+  read(length?: number): Promise<Uint8Array | null>;
+  /** the "nextStream" to write to */
+  write(data: Uint8Array): Promise<void>;
+  /** close the connection */
+  close(): void;
+  /** Async Iterator for streaming per chunck */
+  [Symbol.asyncIterator](): AsyncIterableIterator<Uint8Array>;
+  /** The local address of the connection */
+  localAddr?: ConnAddr;
+  /** The remote address of the connection */
+  remoteAddr?: ConnAddr;
+};

package/deno/README.md

@@ -1,6 +1,6 @@
 # Running Opifex server/client on Deno
 
-These instructions specifify how to run [Opifex](README.md) on Deno. To run
+These instructions specifify how to run [Opifex](../README.md) on Deno. To run
 Opifex on NodeJS see the [instructions](../node/README.md)
 
 ## Playing around

package/deno/denoConnToNext.ts

@@ -0,0 +1,104 @@
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+
+export function denoConnToNext(conn: Deno.Conn): ConnNext {
+  // Convert Deno network addresses to custom ConnAddr format
+  const formatAddr = (addr: Deno.Addr): ConnAddr => {
+    if (addr.transport === "tcp" || addr.transport === "udp") {
+      return {
+        transport: addr.transport,
+        hostname: addr.hostname,
+        port: addr.port,
+      };
+    }
+    const unixAddr = addr as Deno.UnixAddr;
+    return {
+      transport: "unix",
+      path: unixAddr.path,
+    };
+  };
+
+  // Utilize a BYOB (Bring Your Own Buffer) reader on Deno's readable stream
+  // This enables exact sizing and zero-copy reads on the kernel buffer
+  const reader = conn.readable.getReader({ mode: "byob" });
+  const writer = conn.writable.getWriter();
+
+  return {
+    remoteAddr: formatAddr(conn.remoteAddr),
+    localAddr: formatAddr(conn.localAddr),
+
+    /**
+     * Read exactly 'length' bytes directly into a Uint8Array buffer
+     */
+    async read(length?: number): Promise<Uint8Array | null> {
+      try {
+        // If no length is provided, read a default chunk size (e.g., 64KB)
+        const bytesToRead = length ?? 65536;
+        let buffer = new Uint8Array(bytesToRead);
+        let bytesRead = 0;
+
+        while (bytesRead < bytesToRead) {
+          // Pass the unfilled section of the buffer to Deno's kernel reader
+          const view = new Uint8Array(
+            buffer.buffer,
+            buffer.byteOffset + bytesRead,
+            bytesToRead - bytesRead,
+          );
+
+          const { value, done } = await reader.read(view);
+
+          if (done) {
+            // Connection closed before all data could be read
+            if (bytesRead === 0) return null;
+            // Return any partial data read prior to closure
+            return buffer.subarray(0, bytesRead);
+          }
+
+          // BYOB returns the populated ArrayBufferView
+          buffer = new Uint8Array(value.buffer);
+          bytesRead += value.byteLength;
+
+          // If no explicit length was requested, return the first chunk immediately
+          if (length === undefined) {
+            return buffer.subarray(0, bytesRead);
+          }
+        }
+
+        return buffer;
+      } catch {
+        return null;
+      }
+    },
+
+    /**
+     * Write via Deno's WritableStream with built-in backpressure handling
+     */
+    async write(data: Uint8Array): Promise<void> {
+      // writer.write automatically pauses the promise when the TCP pipe is full
+      await writer.write(data);
+    },
+
+    /**
+     * Gracefully close reader, writer, and socket connection
+     */
+    close(): void {
+      try {
+        writer.releaseLock();
+        reader.releaseLock();
+        conn.close();
+      } catch {
+        /* swallow */
+      }
+    },
+
+    /**
+     * Async Iterator for Deno
+     */
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        const chunk = await this.read();
+        if (chunk === null) break;
+        yield chunk;
+      }
+    },
+  };
+}

package/deno/tcpClient.ts

@@ -5,7 +5,8 @@
  */
 
 import { Client, logger } from "../client/mod.ts";
-import type { SockConn } from "../client/mod.ts";
+import type { ConnNext } from "../client/mod.ts";
+import { denoConnToNext } from "./denoConnToNext.ts";
 
 /**
  * Fetches data from a file and returns it as a string
@@ -39,12 +40,12 @@ export class TcpClient extends Client {
    * @param hostname the name of the host to connect to
    * @param port the port number
    */
-  protected async connectMQTT(
+  protected override async connectMQTT(
     hostname: string,
     port = 1883,
-  ): Promise<Deno.TcpConn> {
+  ): Promise<ConnNext> {
     logger.debug({ hostname, port });
-    return await Deno.connect({ hostname, port });
+    return denoConnToNext(await Deno.connect({ hostname, port }));
   }
 
   /**
@@ -56,13 +57,13 @@ export class TcpClient extends Client {
    * @param key the clients private key
    * @returns a Deno.TlsConn object
    */
-  protected async connectMQTTS(
+  protected override async connectMQTTS(
     hostname: string,
     port = 8883,
     caCerts?: string[],
     cert?: string,
     key?: string,
-  ): Promise<Deno.TlsConn> {
+  ): Promise<ConnNext> {
     logger.debug({ hostname, port, caCerts, cert });
     const connectOpts = {
       hostname,
@@ -71,23 +72,6 @@ export class TcpClient extends Client {
       ...(cert !== undefined && { cert }),
       ...(key !== undefined && { key }),
     };
-    return await Deno.connectTls(connectOpts);
-  }
-
-  /** createConn is used by the base client class */
-  protected override createConn(): Promise<SockConn> {
-    const { protocol, hostname, port: portStr } = this.connectUrl;
-    const port = Number(portStr);
-    const caCerts = this.caCerts;
-    const cert = this.cert;
-    const key = this.key;
-
-    if (protocol === "mqtts:") {
-      return this.connectMQTTS(hostname, port, caCerts, cert, key);
-    }
-    if (protocol === "mqtt:") {
-      return this.connectMQTT(hostname, port);
-    }
-    throw new Error(`Unsupported protocol: ${protocol}`);
+    return denoConnToNext(await Deno.connectTls(connectOpts));
   }
 }