@seriousme/opifex 2.2.0 → 2.3.0

package/README.md

@@ -111,10 +111,11 @@ can be found in the [examples](/examples/) folder.
    encode and decode packets.
 
 2. On top of mqttPacket sits the MQTT connection module
-   ([mqttConn/mod.ts](mqttConn/mod.ts)) that reads packets from a Readable
-   stream and writes them to Writable stream. It will take care of incomplete
-   and/or malformed packets. mqttConn provides an async iterable that can be
-   awaited for new packets.
+   ([mqttConn/mod.ts](mqttConn/mod.ts)) that reads packets from and writes them
+   to a [ConnNext](connNext) adapter. It will take care of incomplete and/or
+   malformed packets. mqttConn provides an async iterable that can be awaited
+   for new packets. A ConnNext adapter per transport provides async
+   read/write/close/address functionality on top of the various transports.
 
 3. On top of mqttConn live the MQTT server ([server/mod.ts](server/mod.ts)) and
    MQTT client ([client/mod.ts](client/mod.ts)) that take care of the MQTT
@@ -122,8 +123,8 @@ can be found in the [examples](/examples/) folder.
    another type of packet will be accepted. Both follow a similar model of
    implementation where for each packet that is received a handler is invoked
    which then triggers the next step. Server and client are totally independent
-   of the technical implementation of the connection and only need a socketConn
-   ([socket](socket)) to be able to work.
+   of the technical implementation of the connection and only need a
+   [ConnNext](connNext) connection to be able to work.
 
 4. As the server needs to be able to serve multiple clients at the same time, it
    maintains a context ([server/context.ts](server/context.ts)) per client to

package/bin/mqtt.ts

@@ -278,7 +278,7 @@ async function publish() {
       qos: parseQos(publishArgs.qos),
     });
     logger.debug("Published!");
-    client.disconnect();
+    await client.disconnect();
     logger.debug("Disconnected !");
   } catch (err) {
     if (err instanceof Error) {

package/client/client.ts

@@ -10,9 +10,9 @@ import {
 import type {
   AuthenticationResult,
   ConnectPacket,
+  ConnNext,
   ProtocolLevel,
   PublishPacket,
-  SockConn,
   Store,
   SubscribePacket,
 } from "./deps.ts";
@@ -169,7 +169,6 @@ export class Client {
     const cStore = store ? store : new MemoryStore();
     this.ctx = new Context(cStore, this);
     this.clientId = generateClientId(this.clientIdPrefix);
-    this.numberOfRetries = DEFAULT_RETRIES;
   }
 
   /**
@@ -188,14 +187,58 @@ export class Client {
     return this.connectUrl;
   }
 
+  /**
+   * Placeholder for MQTT, override in subclass
+   * @param _hostname
+   * @param _port
+   */
+  protected connectMQTT(
+    _hostname: string,
+    _port: number,
+  ): Promise<ConnNext> {
+    throw new Error(`Unsupported protocol: mqtt`);
+  }
+
+  /**
+   * Connect using MQTT over TLS
+   * @param hostname the name of the host to connect to
+   * @param port the port number
+   * @param caCerts the CA certificates to trust
+   * @param cert the clients certificate
+   * @param key the clients private key
+   * @returns a ConnNext object
+   */
+  protected connectMQTTS(
+    _hostname: string,
+    _port: number,
+    _caCerts?: string[],
+    _cert?: string,
+    _key?: string,
+  ): Promise<ConnNext> {
+    throw new Error(`Unsupported protocol: mqtts`);
+  }
+
   /**
    * Creates a new connection to the MQTT broker
-   * @returns Promise resolving to a SockConn connection
+   * Override this in a subclass if you need other protocols
+   * @returns Promise resolving to a ConnNext connection
    */
-  protected createConn(): Promise<SockConn> {
+  protected createConn(): Promise<ConnNext> {
     // if you need to support alternative connection types just
     // overload this method in your subclass
-    throw new Error(`Unsupported protocol: ${this.url.protocol}`);
+    const { protocol, hostname, port: portStr } = this.connectUrl;
+    const port = Number(portStr);
+    const caCerts = this.caCerts;
+    const cert = this.cert;
+    const key = this.key;
+
+    if (protocol === "mqtts:") {
+      return this.connectMQTTS(hostname, port, caCerts, cert, key);
+    }
+    if (protocol === "mqtt:") {
+      return this.connectMQTT(hostname, port);
+    }
+    throw new Error(`Unsupported protocol: ${protocol}`);
   }
 
   /**
@@ -233,7 +276,14 @@ export class Client {
         logger.debug({ lastMessage });
       }
 
-      if (tryConnect && (isReconnect || attempt < this.numberOfRetries)) {
+      logger.debug("tryConnect", {
+        tryConnect,
+        isReconnect,
+        attempt,
+        retries: this.numberOfRetries,
+      });
+
+      if (tryConnect && (isReconnect || (attempt < this.numberOfRetries))) {
         await backOffSleep(true, attempt);
         if (!isReconnect) {
           attempt++;
@@ -256,7 +306,7 @@ export class Client {
    */
   connect(params: ConnectParameters = {}): Promise<AuthenticationResult> {
     this.connectUrl = params?.url || this.connectUrl;
-    this.numberOfRetries = params.numberOfRetries || this.numberOfRetries;
+    this.numberOfRetries = params.numberOfRetries ?? this.numberOfRetries;
     this.caCerts = params?.caCerts;
     this.cert = params?.cert;
     this.key = params?.key;

package/client/context.ts

@@ -10,12 +10,12 @@ import type {
   AnyPacket,
   AuthenticationResult,
   ConnectPacket,
+  ConnNext,
   PacketId,
   ProtocolLevel,
   PublishPacket,
   ReasonCode,
   ReturnCodes,
-  SockConn,
   Store,
   SubscribePacket,
   UnsubscribePacket,
@@ -169,7 +169,7 @@ export class Context {
   }
 
   async handleConnection(
-    conn: SockConn,
+    conn: ConnNext,
     connectPacket: ConnectPacket,
   ): Promise<boolean> {
     this.mqttConn = new MqttConn({ conn });

package/client/deps.ts

@@ -36,12 +36,12 @@ export {
 
 export { MqttConn } from "../mqttConn/mqttConn.ts";
 export type {
+  ConnAddr,
+  ConnNext,
   NetAddr,
-  SockAddr,
-  SockConn,
   UnixAddr,
   VsockAddr,
-} from "../socket/socket.ts";
+} from "../connNext/connNext.ts";
 export { Timer } from "../timer/timer.ts";
 export { BufferedAsyncIterable, Deferred, logger } from "../utils/mod.ts";
 export { MemoryStore } from "./store/memoryStore.ts";

package/client/mod.ts

@@ -11,11 +11,11 @@ export {
 export { logger, MQTTLevel, ReasonCode } from "./deps.ts";
 export type {
   AuthenticationResult,
+  ConnAddr,
+  ConnNext,
   NetAddr,
   ProtocolLevel,
   QoS,
-  SockAddr,
-  SockConn,
   UnixAddr,
   VsockAddr,
 } from "./deps.ts";

package/connNext/connNext.ts

@@ -0,0 +1,44 @@
+/** Net addresses */
+export type NetAddr = {
+  /** the IP-based transport protocol */
+  transport: "tcp" | "udp";
+  /** the hostname */
+  hostname: string;
+  /** the port number */
+  port: number;
+};
+/** Unix addresses */
+export type UnixAddr = {
+  /** the Unix domain socket transport protocol */
+  transport: "unix" | "unixpacket";
+  /** the file system path to the socket to connect to */
+  path: string;
+};
+/** vSock addresses */
+export type VsockAddr = {
+  /** the VSOCK transport protocol */
+  transport: "vsock";
+  /** the context identifier (CID) of the peer */
+  cid: number;
+  /** the port number */
+  port: number;
+};
+
+/** connection addresses */
+export type ConnAddr = NetAddr | UnixAddr | VsockAddr | undefined;
+/** connection descriptor. */
+
+export type ConnNext = {
+  /** the "nextStream" to read from */
+  read(length?: number): Promise<Uint8Array | null>;
+  /** the "nextStream" to write to */
+  write(data: Uint8Array): Promise<void>;
+  /** close the connection */
+  close(): void;
+  /** Async Iterator for streaming per chunck */
+  [Symbol.asyncIterator](): AsyncIterableIterator<Uint8Array>;
+  /** The local address of the connection */
+  localAddr?: ConnAddr;
+  /** The remote address of the connection */
+  remoteAddr?: ConnAddr;
+};

package/deno/README.md

@@ -1,6 +1,6 @@
 # Running Opifex server/client on Deno
 
-These instructions specifify how to run [Opifex](README.md) on Deno. To run
+These instructions specifify how to run [Opifex](../README.md) on Deno. To run
 Opifex on NodeJS see the [instructions](../node/README.md)
 
 ## Playing around

package/deno/denoConnToNext.ts

@@ -0,0 +1,104 @@
+import type { ConnAddr, ConnNext } from "../connNext/connNext.ts";
+
+export function denoConnToNext(conn: Deno.Conn): ConnNext {
+  // Convert Deno network addresses to custom ConnAddr format
+  const formatAddr = (addr: Deno.Addr): ConnAddr => {
+    if (addr.transport === "tcp" || addr.transport === "udp") {
+      return {
+        transport: addr.transport,
+        hostname: addr.hostname,
+        port: addr.port,
+      };
+    }
+    const unixAddr = addr as Deno.UnixAddr;
+    return {
+      transport: "unix",
+      path: unixAddr.path,
+    };
+  };
+
+  // Utilize a BYOB (Bring Your Own Buffer) reader on Deno's readable stream
+  // This enables exact sizing and zero-copy reads on the kernel buffer
+  const reader = conn.readable.getReader({ mode: "byob" });
+  const writer = conn.writable.getWriter();
+
+  return {
+    remoteAddr: formatAddr(conn.remoteAddr),
+    localAddr: formatAddr(conn.localAddr),
+
+    /**
+     * Read exactly 'length' bytes directly into a Uint8Array buffer
+     */
+    async read(length?: number): Promise<Uint8Array | null> {
+      try {
+        // If no length is provided, read a default chunk size (e.g., 64KB)
+        const bytesToRead = length ?? 65536;
+        let buffer = new Uint8Array(bytesToRead);
+        let bytesRead = 0;
+
+        while (bytesRead < bytesToRead) {
+          // Pass the unfilled section of the buffer to Deno's kernel reader
+          const view = new Uint8Array(
+            buffer.buffer,
+            buffer.byteOffset + bytesRead,
+            bytesToRead - bytesRead,
+          );
+
+          const { value, done } = await reader.read(view);
+
+          if (done) {
+            // Connection closed before all data could be read
+            if (bytesRead === 0) return null;
+            // Return any partial data read prior to closure
+            return buffer.subarray(0, bytesRead);
+          }
+
+          // BYOB returns the populated ArrayBufferView
+          buffer = new Uint8Array(value.buffer);
+          bytesRead += value.byteLength;
+
+          // If no explicit length was requested, return the first chunk immediately
+          if (length === undefined) {
+            return buffer.subarray(0, bytesRead);
+          }
+        }
+
+        return buffer;
+      } catch {
+        return null;
+      }
+    },
+
+    /**
+     * Write via Deno's WritableStream with built-in backpressure handling
+     */
+    async write(data: Uint8Array): Promise<void> {
+      // writer.write automatically pauses the promise when the TCP pipe is full
+      await writer.write(data);
+    },
+
+    /**
+     * Gracefully close reader, writer, and socket connection
+     */
+    close(): void {
+      try {
+        writer.releaseLock();
+        reader.releaseLock();
+        conn.close();
+      } catch {
+        /* swallow */
+      }
+    },
+
+    /**
+     * Async Iterator for Deno
+     */
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        const chunk = await this.read();
+        if (chunk === null) break;
+        yield chunk;
+      }
+    },
+  };
+}

package/deno/tcpClient.ts

@@ -5,7 +5,8 @@
  */
 
 import { Client, logger } from "../client/mod.ts";
-import type { SockConn } from "../client/mod.ts";
+import type { ConnNext } from "../client/mod.ts";
+import { denoConnToNext } from "./denoConnToNext.ts";
 
 /**
  * Fetches data from a file and returns it as a string
@@ -39,12 +40,12 @@ export class TcpClient extends Client {
    * @param hostname the name of the host to connect to
    * @param port the port number
    */
-  protected async connectMQTT(
+  protected override async connectMQTT(
     hostname: string,
     port = 1883,
-  ): Promise<Deno.TcpConn> {
+  ): Promise<ConnNext> {
     logger.debug({ hostname, port });
-    return await Deno.connect({ hostname, port });
+    return denoConnToNext(await Deno.connect({ hostname, port }));
   }
 
   /**
@@ -56,13 +57,13 @@ export class TcpClient extends Client {
    * @param key the clients private key
    * @returns a Deno.TlsConn object
    */
-  protected async connectMQTTS(
+  protected override async connectMQTTS(
     hostname: string,
     port = 8883,
     caCerts?: string[],
     cert?: string,
     key?: string,
-  ): Promise<Deno.TlsConn> {
+  ): Promise<ConnNext> {
     logger.debug({ hostname, port, caCerts, cert });
     const connectOpts = {
       hostname,
@@ -71,23 +72,6 @@ export class TcpClient extends Client {
       ...(cert !== undefined && { cert }),
       ...(key !== undefined && { key }),
     };
-    return await Deno.connectTls(connectOpts);
-  }
-
-  /** createConn is used by the base client class */
-  protected override createConn(): Promise<SockConn> {
-    const { protocol, hostname, port: portStr } = this.connectUrl;
-    const port = Number(portStr);
-    const caCerts = this.caCerts;
-    const cert = this.cert;
-    const key = this.key;
-
-    if (protocol === "mqtts:") {
-      return this.connectMQTTS(hostname, port, caCerts, cert, key);
-    }
-    if (protocol === "mqtt:") {
-      return this.connectMQTT(hostname, port);
-    }
-    throw new Error(`Unsupported protocol: ${protocol}`);
+    return denoConnToNext(await Deno.connectTls(connectOpts));
   }
 }

package/deno/tcpServer.test.ts

@@ -2,8 +2,10 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { TcpClient } from "./tcpClient.ts";
 import { TcpServer } from "./tcpServer.ts";
-import { delay, logger, LogLevel } from "../utils/mod.ts";
-import type { ProtocolLevel, PublishPacket, QoS } from "../mqttPacket/mod.ts";
+import { logger, LogLevel } from "../utils/mod.ts";
+import type { ProtocolLevel } from "../mqttPacket/mod.ts";
+import { testClient } from "../dev_utils/testClient.ts";
+import { MqttServer } from "../server/mod.ts";
 
 logger.level(LogLevel.info);
 
@@ -24,64 +26,10 @@ test("Deno: Test pubSub using client and server", async () => {
   };
 
   const client = new TcpClient();
-
-  await client.connect(params);
-  assert(true, "Client connected to server");
-
-  const publishSet: { topic: string; qos: QoS }[] = [
-    { topic: "t0@q0", qos: 0 },
-    { topic: "t1@q0", qos: 0 },
-    { topic: "t2@q0", qos: 0 },
-    { topic: "t0@q1", qos: 1 },
-    { topic: "t1@q1", qos: 1 },
-    { topic: "t2@q1", qos: 1 },
-    { topic: "t0@q2", qos: 2 },
-    { topic: "t1@q2", qos: 2 },
-    { topic: "t2@q2", qos: 2 },
-  ];
-
-  const subscriptions = publishSet.map((item) => ({
-    topicFilter: item.topic,
-    qos: item.qos,
-  }));
-  await client.subscribe({
-    subscriptions,
-  });
-
-  // the IIFE ensures message reception runs in parallel
-  logger.verbose("Start receiving");
-  const received: PublishPacket[] = [];
-  (async () => {
-    for await (const item of client.messages()) {
-      logger.verbose("Receiving:", item.topic, "--", item.qos);
-      received.push(item);
-    }
-  })();
-  // end of IIFE
-  for (const item of publishSet) {
-    logger.verbose("Publishing:", item.topic, "--", item.qos);
-    await client.publish({
-      topic: item.topic,
-      qos: item.qos,
-      payload: new Uint8Array([0x01]),
-    });
-  }
-
-  await delay(100);
-  logger.verbose("Disconnect client");
-  await client.disconnect();
-
-  logger.verbose("Check completeness");
-  for (const item of publishSet) {
-    const found = received.find((f) =>
-      f.topic == item.topic && f.qos === item.qos
-    );
-    logger.verbose("Found:", item.topic, "--", item.qos);
-    assert(found, `${item.topic} -- ${item.qos}`);
-  }
+  await testClient(client, params);
 
   logger.verbose("Stop server");
-  server.stop();
+  await server.stop();
 });
 
 test("Deno: Test subscription persistence after reconnect", async () => {
@@ -99,51 +47,14 @@ test("Deno: Test subscription persistence after reconnect", async () => {
   };
 
   const client = new TcpClient();
-  const testTopic = "test/topic";
-  const received: PublishPacket[] = [];
-
-  // First connection and subscription
-  await client.connect(params);
-  await client.subscribe({
-    subscriptions: [{
-      topicFilter: testTopic,
-      qos: 0,
-    }],
-  });
-
-  // Start receiving messages
-  (async () => {
-    for await (const item of client.messages()) {
-      received.push(item);
-    }
-  })();
+  await testClient(client, params);
 
-  // Disconnect client
-  await client.disconnect();
-  await delay(100);
-
-  // Reconnect client
-  await client.connect(params);
-  await delay(100);
-
-  // Publish test message
-  await client.publish({
-    topic: testTopic,
-    qos: 0,
-    payload: new Uint8Array([0x01]),
-  });
-
-  await delay(100);
-  logger.verbose("Disconnect client");
-  await client.disconnect();
-
-  // Verify message was received
-  assert.equal(received.length, 1, "Should receive one message");
-  assert.equal(
-    received[0].topic,
-    testTopic,
-    "Should receive message on subscribed topic",
-  );
+  logger.verbose("Stop server");
+  await server.stop();
+});
 
-  server.stop();
+test("Deno: Passing a mqttServer works", () => {
+  const mqttServer = new MqttServer({});
+  const server = new TcpServer({ port: 0 }, mqttServer);
+  assert(server.mqttServer === mqttServer);
 });

package/deno/tcpServer.ts

@@ -5,24 +5,30 @@
  */
 
 import { MqttServer } from "../server/mod.ts";
+import { denoConnToNext } from "./denoConnToNext.ts";
 import type { MqttServerOptions } from "../server/mod.ts";
 
 /**
  * TCP server that wraps a MqttServer, see the /examples folder
  */
 export class TcpServer {
+  /** the MqttServer instance used by the server */
+  readonly mqttServer: MqttServer;
   private listener: Deno.Listener<Deno.Conn>;
-  private mqttServer: MqttServer;
 
   /**
    * Create a new TCP server
    */
   constructor(
     serverOptions: Deno.TcpListenOptions,
-    mqttOptions: MqttServerOptions,
+    mqttOptions: MqttServerOptions | MqttServer,
   ) {
     this.listener = Deno.listen(serverOptions);
-    this.mqttServer = new MqttServer(mqttOptions);
+    if (mqttOptions instanceof MqttServer) {
+      this.mqttServer = mqttOptions;
+    } else {
+      this.mqttServer = new MqttServer(mqttOptions);
+    }
   }
 
   /**
@@ -30,15 +36,17 @@ export class TcpServer {
    */
   async start(): Promise<void> {
     for await (const conn of this.listener) {
-      this.mqttServer.serve(conn);
+      this.mqttServer.serve(denoConnToNext(conn));
     }
   }
 
   /**
    * Stop listening
    */
-  stop(): void {
-    this.mqttServer.close(true);
+  async stop(opts = { closeMqtt: true }): Promise<void> {
+    if (opts.closeMqtt) {
+      await this.mqttServer.close();
+    }
     this.listener.close();
   }