@seriousme/opifex 1.11.3 → 2.1.0

package/socket/socket.ts

@@ -1,21 +1,16 @@
-export type SockAddr = NetAddr | UnixAddr | VsockAddr;
-export interface NetAddr {
+/** Net adresses */
+export type NetAddr = {
   transport: "tcp" | "udp";
   hostname: string;
   port: number;
-}
-
-export interface UnixAddr {
-  transport: "unix" | "unixpacket";
-  path: string;
-}
-
-export interface VsockAddr {
-  transport: "vsock";
-  cid: number;
-  port: number;
-}
-
+};
+/** Unix adresses */
+export type UnixAddr = { transport: "unix" | "unixpacket"; path: string };
+/** vSock adresses */
+export type VsockAddr = { transport: "vsock"; cid: number; port: number };
+/** Socket adresses */
+export type SockAddr = NetAddr | UnixAddr | VsockAddr;
+/** Socket connection descriptor. */
 export type SockConn = {
   readable: ReadableStream<Uint8Array>;
   writable: WritableStream<Uint8Array>;
@@ -25,49 +20,105 @@ export type SockConn = {
 };
 
 export class Conn {
-  reader: ReadableStreamBYOBReader;
+  reader: ReadableStreamDefaultReader<Uint8Array>;
   writer: WritableStreamDefaultWriter<Uint8Array>;
   closed: boolean;
   remoteAddr: SockAddr | undefined;
   closer: () => void;
 
+  private leftover: Uint8Array = new Uint8Array(0);
+
   constructor(sockConn: SockConn) {
     this.closed = false;
-    this.reader = sockConn.readable.getReader({ mode: "byob" });
+    this.reader = sockConn.readable.getReader();
     this.writer = sockConn.writable.getWriter();
-    // this.writer.closed.then(() => this.closed = true);
     this.closer = sockConn.close.bind(sockConn);
+    this.reader.closed.catch(() => {});
+    this.writer.closed.catch(() => {});
     this.remoteAddr = sockConn.remoteAddr;
   }
 
-  async read(buff: Uint8Array): Promise<number | null> {
-    const buff2 = new Uint8Array(buff.length);
-    const result = await this.reader.read(buff2);
-    if (!result.done) {
-      buff.set(result.value);
+  async read(length: number): Promise<Uint8Array | null> {
+    if (this.closed || length < 0) {
+      return null;
+    }
+
+    if (length === 0) {
+      return new Uint8Array(0);
+    }
+
+    const result = new Uint8Array(length);
+    let bytesAccumulated = 0;
+
+    // Do we have data from the previous read()
+    if (this.leftover.length > 0) {
+      const take = Math.min(this.leftover.length, length);
+      result.set(this.leftover.subarray(0, take), bytesAccumulated);
+      bytesAccumulated += take;
+      this.leftover = this.leftover.subarray(take);
+    }
+
+    // Keep readin until we have enough data
+    while (bytesAccumulated < length) {
+      const { value, done } = await this.reader.read();
+
+      if (value) {
+        const needed = length - bytesAccumulated;
+
+        if (value.length <= needed) {
+          // We need the whole chunk
+          result.set(value, bytesAccumulated);
+          bytesAccumulated += value.length;
+        } else {
+          // Chunk is bigger than what we need
+          result.set(value.subarray(0, needed), bytesAccumulated);
+          bytesAccumulated += needed;
+          // Save the rest for the next read() call
+          this.leftover = value.subarray(needed);
+        }
+      }
+      if (done) {
+        // if the stream closed and not enough data was returned, return null
+        return null;
+      }
     }
-    return result.value?.byteLength || null;
+    return result;
   }
 
-  write(data: Uint8Array): Promise<number> {
+  async write(data: Uint8Array): Promise<number> {
     if (this.closed) {
-      return Promise.reject();
+      return Promise.reject(new Error("Connection closed"));
+    }
+    try {
+      await this.writer.write(data);
+      return data.length;
+    } catch (err) {
+      this.closed = true;
+      throw err;
     }
-    this.writer.write(data);
-    return new Promise((resolve) => resolve(data.length));
   }
 
   close() {
     if (!this.closed) {
-      try {
-        this.writer.close();
-      } catch (_err) { /* swallow any errors on close */ }
       this.closed = true;
+
+      // deno-coverage-ignore-start
+      void this.writer.close().catch(() => {
+        // ignore duplicate/late close attempts; the stream may already be closed
+      });
+      // deno-coverage-ignore-stop
+
+      try {
+        void this.reader.cancel().catch(() => {
+          // ignore duplicate/late cancel attempts; the stream may already be closed
+        });
+        this.reader?.releaseLock();
+        // deno-coverage-ignore
+      } catch (_err) { /* swallow */ }
+
       try {
         this.closer();
-      } catch (_err) {
-        // swallow any errors on close
-      }
+      } catch (_err) { /* swallow */ }
     }
   }
 }

package/timer/timer.test.ts

@@ -1,12 +1,9 @@
 import { Timer } from "./timer.ts";
 import assert from "node:assert/strict";
 import { test } from "node:test";
+import { delay } from "../dev_utils/mod.ts";
 
-function delay(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-test("timer should ring", async () => {
+test("timer should ring", { concurrency: false }, async () => {
   const wait = 50;
   const start = Date.now();
   let end = Date.now();
@@ -18,7 +15,7 @@ test("timer should ring", async () => {
   assert.deepStrictEqual(timeDiff < 10, true);
 });
 
-test("snooze should work", async () => {
+test("snooze should work", { concurrency: false }, async () => {
   const wait = 50;
   const quarterWait = Math.floor(wait / 4);
   const halfWait = quarterWait * 2;
@@ -37,7 +34,7 @@ test("snooze should work", async () => {
   assert.deepStrictEqual(timeDiff < 15, true, `TimeDiff of ${timeDiff} < 15`);
 });
 
-test("clear should work", () => {
+test("clear should work", { concurrency: false }, () => {
   const wait = 50;
   let end = 0;
   const timer = new Timer(() => {

package/timer/timer.ts

@@ -1,11 +1,31 @@
+/**
+ * A reusable timer class that executes a callback function after a specified delay.
+ * It handles potential drifts or early triggers by checking the actual remaining time
+ * before executing the action.
+ */
 export class Timer {
+  /** The delay duration in milliseconds before the timer expires. */
   readonly delay: number; // delay in microseconds
+
+  /** The callback function to perform when the timer expires. */
   // deno-lint-ignore ban-types
   private action: Function; // function to perform when timer expires
+
+  /** The internal timeout identifier reference, if a timer is active. */
   private timer?: ReturnType<typeof setTimeout>;
+
+  /** The timestamp (in milliseconds) indicating when the timer is scheduled to end. */
   private end = 0;
+
+  /** Indicates whether the timer is currently active and running. */
   private running = false;
 
+  /**
+   * Initializes a new instance of the Timer.
+   * @param {Function} action - The callback function to execute upon expiration.
+   * @param {number} delay - The delay duration in milliseconds.
+   * @param {boolean} [wait=false] - If true, the timer will not start automatically until reset() is called.
+   */
   // deno-lint-ignore ban-types
   constructor(action: Function, delay: number, wait = false) {
     this.delay = delay;
@@ -15,11 +35,20 @@ export class Timer {
     }
   }
 
+  /**
+   * Starts the internal setTimeout with the specified delay.
+   * @param {number} delay - The duration in milliseconds for the timeout.
+   */
   private startTimer(delay: number): void {
     this.running = true;
     this.timer = setTimeout(() => this.ring(), delay);
   }
 
+  /**
+   * Internal handler triggered when the timeout fires.
+   * Verifies if the target end time has genuinely passed, rescheduling if necessary,
+   * or executes the action.
+   */
   private ring(): void {
     const timeLeft = this.end - Date.now();
     // check if delay has passed, else execute action
@@ -31,6 +60,10 @@ export class Timer {
     this.action();
   }
 
+  /**
+   * Resets the timer, recalculating the end time and restarting the countdown.
+   * If the timer is already running, it updates the target end time.
+   */
   reset(): void {
     this.end = Date.now() + this.delay;
     if (!this.running) {
@@ -38,6 +71,9 @@ export class Timer {
     }
   }
 
+  /**
+   * Cancels the active timeout and stops the timer from running.
+   */
   clear(): void {
     clearTimeout(this.timer);
     this.running = false;

package/tools/benchmarks/receiver.ts

@@ -0,0 +1,51 @@
+import { exit } from "node:process";
+import { getArgs, importClientClass } from "./utils.ts";
+import { logger } from "../../utils/mod.ts";
+import type { LogLevel } from "../../utils/logger.ts";
+import type { QoS } from "../../client/deps.ts";
+
+export async function runReceiver(
+  level: LogLevel,
+  url: string,
+  topic: string,
+  qos: QoS,
+) {
+  logger.level(level);
+  const { TcpClient } = await importClientClass();
+  const client = new TcpClient();
+  client.onError = (err: Error) => {
+    console.log("client error", err);
+    exit(1);
+  };
+
+  await client.connect({
+    url: new URL(url),
+  });
+
+  await client.subscribe({ subscriptions: [{ topicFilter: topic, qos }] });
+
+  let counter = 0;
+  const interval = 5000;
+
+  function count() {
+    console.log("received/s", (counter / interval) * 1000);
+    counter = 0;
+  }
+
+  setInterval(count, interval);
+
+  // // Consume using a callback
+  // client.onPacket = async (packet) => {
+  //   counter++;
+  // };
+
+  // Consume using an AsyncIterator
+  for await (const _ of client.messages()) {
+    counter++;
+  }
+}
+
+if (import.meta.main) {
+  const { level, url, topic, qos } = getArgs();
+  runReceiver(level, url, topic, qos);
+}

package/tools/benchmarks/sender.ts

@@ -0,0 +1,48 @@
+import { exit } from "node:process";
+import { runAsap } from "../../dev_utils/timers.ts";
+import { getArgs, importClientClass } from "./utils.ts";
+import { logger } from "../../utils/mod.ts";
+import type { LogLevel } from "../../utils/logger.ts";
+import type { QoS } from "../../client/deps.ts";
+
+const { TcpClient } = await importClientClass();
+const client = new TcpClient();
+client.onError = (err: Error) => {
+  console.log("client error", err);
+  exit(1);
+};
+
+export async function runSender(
+  level: LogLevel,
+  url: string,
+  topic: string,
+  qos: QoS,
+) {
+  logger.level(level);
+  await client.connect({
+    url: new URL(url),
+  });
+
+  const payload = new TextEncoder().encode("payload");
+  let sent = 0;
+  const interval = 5000;
+
+  const loop = () => {
+    sent++;
+    client.publish({ topic, qos, payload })
+      .then(() => runAsap(loop));
+  };
+
+  function count() {
+    console.log("sent/s", (sent / interval) * 1000);
+    sent = 0;
+  }
+
+  setInterval(count, interval);
+  loop();
+}
+
+if (import.meta.main) {
+  const { level, url, topic, qos } = getArgs();
+  runSender(level, url, topic, qos);
+}

package/tools/benchmarks/utils.ts

@@ -0,0 +1,31 @@
+import { parseArgs } from "node:util";
+import { LogLevel } from "../../utils/mod.ts";
+import type { QoS } from "../../client/mod.ts";
+
+export async function importClientClass() {
+  // @ts-ignore: the Deno global variable is not defined in @types/node,
+  //             therefore we need to ignore this error because this file
+  //             is meant to be used in both environments.
+  const isDeno = typeof Deno !== "undefined";
+  return isDeno
+    ? (await import("../../deno/tcpClient.ts"))
+    : (await import("../../node/tcpClient.ts"));
+}
+
+export function getArgs() {
+  const benchMarkOpts = {
+    url: { type: "string", short: "u", default: "mqtt://localhost:1883" },
+    topic: { type: "string", short: "t", default: "benchmark" },
+    qos: { type: "string", short: "q", default: "0" },
+    logLevel: { type: "string", short: "l", default: "error" },
+  } as const;
+
+  const { values } = parseArgs({ options: benchMarkOpts });
+  const url = values.url as string;
+  const topic = values.topic as string;
+  const qos = Number(values.qos ?? 0) as QoS;
+  const logLevelKey = (values.logLevel ?? "error") as keyof typeof LogLevel;
+  const level = LogLevel[logLevelKey] ?? LogLevel.error;
+
+  return { url, topic, qos, level };
+}

package/tools/mqtt-compat/README.md

@@ -0,0 +1,130 @@
+# MQTT compatibility tests
+
+Measures Opifex's **actual** MQTT 5.0 and 3.1.1 protocol compatibility by
+running the vendor-neutral [Eclipse Paho interoperability suite][paho] against a
+broker booted from this repo, and reports an honest pass percentage. Wired into
+CI by
+[`.github/workflows/mqtt-compat.yml`](../../.github/workflows/mqtt-compat.yml),
+which posts the result as a sticky PR comment and a job summary.
+
+## Files
+
+| File            | Purpose                                                                                                            |
+| --------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `broker.js`     | Boots Opifex on a plain-TCP port (`MQTT_PORT`, default 1883) with the v5 capabilities the suite exercises enabled. |
+| `run_compat.py` | Runs each Paho test in an isolated subprocess against the broker, emits `report.md` + `result.json`. Pure stdlib.  |
+
+## Running locally
+
+```bash
+# 1. Fetch the Paho suite, pinned to the same commit CI uses so local results
+#    match. The pin (PAHO_REF) lives in .github/workflows/mqtt-compat.yml.
+git clone https://github.com/eclipse-paho/paho.mqtt.testing /tmp/paho
+git -C /tmp/paho checkout "$(grep -oP 'PAHO_REF:\s*\K\S+' ../../.github/workflows/mqtt-compat.yml)"
+
+# 2. Start the broker
+node ./broker.ts &
+
+# 3. Run the suites
+python3 ./run_compat.py --paho /tmp/paho/interoperability --host localhost --port 1883 --out report.md --json result.json
+```
+
+`--protocols v5,v3` selects the suites; `--timeout` bounds each test (default
+45s).
+
+### Individual test
+
+Version 3.11:
+
+```bash
+python3 /tmp/paho/interoperability/client_test.py Test.test_retained_messages
+```
+
+Version 5:
+
+```bash
+python3 /tmp/paho/interoperability/client_test5.py Test.test_retained_messages
+```
+
+## How the score is computed
+
+Each Paho `Test` method runs in its **own subprocess** with a hard timeout. This
+matters because the Paho clients do blocking waits with no internal timeout: a
+case where the broker never sends an expected packet would otherwise hang the
+whole run, and a hung test would corrupt the class-level shared clients later
+tests reuse. A timed-out test is recorded as a bounded failure.
+
+The percentage is the raw pass rate (`passed / evaluated`). Two categories are
+handled specially:
+
+- **Expected gaps** (`EXPECTED_GAPS` in `run_compat.py`) — features Opifex
+  intentionally does not implement yet (e.g. shared subscriptions, which the
+  broker advertises as `sharedSubscriptionAvailable=false`). These still count
+  as failures; they are only annotated with the reason so the table is readable.
+  Because v5 is a work in progress, the harness also detects the reverse: when
+  an `EXPECTED_GAPS` test starts **passing** (the feature got implemented), the
+  report highlights it with a 🎉 banner and the workflow emits a CI warning,
+  prompting you to remove the now-stale entry from `EXPECTED_GAPS`.
+- **Harness-limited** (`HARNESS_LIMITED`) — tests that cannot be evaluated under
+  per-test isolation against _any_ broker (currently only `test_flow_control2`,
+  which depends on the persistent client the suite sets up only in
+  single-process mode). These are **excluded** from the denominator. The
+  capability they probe is still measured elsewhere (`test_flow_control1` for
+  receive-maximum).
+
+### Adding or moving a test category
+
+Both lists in `run_compat.py` are keyed
+`protocol -> exact Paho method name ->
+reason string`. Pick the dict by what the
+failure _means_:
+
+- A genuine Opifex gap that should drag the score down → **`EXPECTED_GAPS`**. It
+  still runs and counts as a failure; when Opifex later implements it the
+  harness flags the 🎉 xpass so you remove the entry.
+- A test the harness can't fairly evaluate against _any_ broker →
+  **`HARNESS_LIMITED`**. It is skipped before the gap logic and excluded from
+  the denominator. A method must live in **exactly one** list (`HARNESS_LIMITED`
+  is checked first, so a duplicate `EXPECTED_GAPS` entry would be dead code).
+
+## Coverage scope — read the percentage correctly
+
+The percentage is **"% of the Paho functional suite that passes"**, not "%
+MQTT-5.0-compliant". The Paho suite is a third-party **happy-path functional**
+test: it exercises the user-facing v5 features well (session expiry, will + will
+delay, message expiry, RH/RAP/NL, request/response, payload format, assigned
+client id, server keep alive, inbound topic aliases, max-packet-size rejection,
+PUBLISH user properties), but it touches only 5 reason codes and deliberately
+omits large parts of the spec:
+
+- **Error / negative paths** — most of Opifex's reason codes
+  (`lib/constants.js`): `0x87` not-authorized acks, `0x8E` session takeover,
+  `0x82` protocol errors, `0x8C` bad auth method, `0x9E`, the CONNACK reject
+  codes, broker-initiated DISCONNECT with ReasonString, the oversized-frame read
+  latch, wildcard ResponseTopic rejection. These are covered by
+  **`test/mqtt5.js`**, which is the authoritative place Opifex's v5 error
+  semantics are pinned.
+- **Whole spec chapters** — AUTH / enhanced auth (§4.12), Server Reference
+  (`0x9C`/`0x9D`), CONNACK capability flags, ReasonString round-trips, and
+  UserProperty on non-PUBLISH packets. Some of these Opifex does not implement;
+  the test gap left by _both_ suites is tracked in
+  [#1096](https://github.com/moscajs/Opifex/issues/1096).
+
+So treat this workflow as the **cross-implementation functional gauge** and
+`test/mqtt5.js` as the **gap-filler** — they are complementary, not redundant.
+
+Some Paho failures are intentional Opifex design decisions rather than bugs (see
+`EXPECTED_GAPS`), and at least one is a genuine spec bug found by this suite:
+[#1095](https://github.com/moscajs/Opifex/issues/1095) (zero-length ClientId +
+CleanSession=0 must be rejected on v3.1.1).
+
+## Harness validation
+
+The harness was validated against the Paho project's own fully-compliant
+reference broker (`startbroker.py`): it scores **100%** of evaluated v5 tests
+(with `test_flow_control2` excluded as above). Every test Opifex fails, the
+reference broker passes under the identical harness — so a failure here reflects
+a real difference between Opifex and a spec-compliant broker, not a harness
+artifact.
+
+[paho]: https://github.com/eclipse-paho/paho.mqtt.testing

package/tools/mqtt-compat/broker.ts

@@ -0,0 +1,52 @@
+// Standalone aedes broker used by the MQTT compatibility workflow.
+//
+// It boots the broker from *this branch* on a plain-TCP port so the Eclipse Paho
+// interoperability suite (tools/mqtt-compat/run_compat.py) can drive it as an
+// external broker. v5 capabilities the suite exercises are enabled here so the
+// reported score reflects real broker ability, not a broker started with features
+// switched off (e.g. inbound topic aliases are a no-op when topicAliasMaximum: 0).
+//
+// Usage: MQTT_PORT=1883 node tools/mqtt-compat/broker.js
+import { TcpServer } from "../../node/tcpServer.ts";
+import type { Context, ShareName, Topic } from "../../server/mod.ts";
+import { logger } from "../../utils/mod.ts";
+import { isAuthenticatedBroker as isAuthenticated } from "../../dev_utils/mod.ts";
+import { SqlitePersistence } from "../../persistence/sqlite/sqlitePersistence.ts";
+
+const port = Number(process.env.MQTT_PORT) || 1883;
+const persistence = new SqlitePersistence();
+
+// The Paho `test_subscribe_failure` (v3.1.1 and v5) subscribes to a topic that is
+// "not allowed to be subscribed to" and asserts the broker answers with SUBACK
+// reason code 0x80.
+
+const NO_SUBSCRIBE_TOPIC = "test/nosubscribe";
+
+function isAuthorizedToSubscribe(
+  ctx: Context,
+  topic: Topic,
+  _shareName: ShareName,
+): boolean {
+  logger.debug(
+    `Checking authorization of client '${ctx.clientId}' to subscribe to topic '${topic}'`,
+  );
+  return topic !== NO_SUBSCRIBE_TOPIC;
+}
+
+const tcpServer = new TcpServer({ port }, {
+  persistence,
+  handlers: {
+    isAuthenticated,
+    isAuthorizedToSubscribe,
+  },
+});
+tcpServer.start();
+logger.info("Server started on port", tcpServer.port);
+
+function shutdown() {
+  tcpServer.stop();
+  process.exit(0);
+}
+
+process.on("SIGTERM", shutdown);
+process.on("SIGINT", shutdown);