@seriousme/opifex 1.11.3 → 2.1.0

package/persistence/persistence.ts

@@ -2,92 +2,136 @@
  * @module persistence
  * @description Module for handling MQTT message persistence and client management
  */
-import type { ClientId, PublishPacket, QoS, Topic } from "./deps.ts";
-import type { IStore } from "./store.ts";
+import type {
+  ClientId,
+  PacketId,
+  PublishPacketV5,
+  QoS,
+  RetainHandling,
+  TopicFilter,
+} from "./deps.ts";
 
 /**
- * Maximum packet ID value for MQTT messages (0xffff/65535)
- * @constant {number}
- */
-export const maxPacketId = 0xffff;
-/**
- * Handler function type for processing publish packets
- * @callback Handler
- * @param {PublishPacket} packet - The MQTT publish packet to handle
+ * extended publish packet also contains meta data
  */
-export type Handler = (packet: PublishPacket) => void;
+export type ExtPublishPacket = PublishPacketV5 & {
+  expiresAtMs?: number;
+};
 
 /**
- * Store type for retained messages mapped by topic
- * @typedef {Map<Topic, PublishPacket>} RetainStore
+ * The MQTT topic share to subscribe/unsubscribe to
  */
-export type RetainStore = Map<Topic, PublishPacket>;
+export type ShareName = string;
+
+export type ClientSubscription = {
+  topicFilter: TopicFilter;
+  shareName: ShareName;
+  qos: QoS;
+  noLocal?: boolean;
+  retainAsPublished?: boolean;
+  retainHandling?: RetainHandling;
+  subscriptionIdentifier?: number;
+};
 
 /**
- * Client type containing message store and packet handler
- * @typedef {Object} Client
- * @property {IStore} store - The client's message store
- * @property {Handler} handler - The client's packet handler function
+ * Maximum packet ID value for MQTT messages (0xffff/65535)
  */
-export type Client = { store: IStore; handler: Handler };
+export const MAX_PACKET_ID = 0xffff;
+
+// Handler function type for processing publish packets
+export type Handler = (packet: ExtPublishPacket) => void | Promise<void>;
+
+// The result returned by client registration
+export type ClientRegistrationResult = {
+  existingSession: boolean;
+};
 
 /**
  * Interface for persistence implementations to store messages and subscriptions
- * @interface IPersistence
  */
-export interface IPersistence {
-  /**
-   * Map of connected clients by client ID
-   * @type {Map<ClientId, Client>}
-   */
-  clientList: Map<ClientId, Client>;
+export interface Persistence {
+  // initialize the persistence
+  // e.g. setting up the data store
+  initialize(): Promise<void>;
+
+  // client registration
+  registerClient(
+    clientId: ClientId,
+    handler: Handler,
+  ): Promise<ClientRegistrationResult>;
+  deregisterClient(clientId: ClientId): Promise<void>;
 
-  /**
-   * Map of retained messages by topic
-   * @type {RetainStore}
-   */
-  retained: RetainStore;
+  disconnectClient(clientId: ClientId): Promise<void>;
 
-  /**
-   * Register a new client with the persistence layer
-   * @param {ClientId} clientId - Unique identifier for the client
-   * @param {Handler} handler - Packet handler function for the client
-   * @param {boolean} clean - Whether to start with a clean session
-   * @returns {IStore} The client's message store
-   */
-  registerClient(clientId: ClientId, handler: Handler, clean: boolean): IStore;
+  // subscription management
+  subscribe(
+    clientId: ClientId,
+    subscription: ClientSubscription,
+  ): Promise<void>;
+  unsubscribe(
+    clientId: ClientId,
+    topicFilter: TopicFilter,
+    shareName: ShareName,
+  ): Promise<void>;
+  listSubscriptions(
+    clientId: ClientId,
+  ): AsyncIterableIterator<ClientSubscription>;
 
-  /**
-   * Remove a client from the persistence layer
-   * @param {ClientId} clientId - ID of client to deregister
-   */
-  deregisterClient(clientId: ClientId): void;
+  // Incoming Packet management
+  addPendingIncomingPacket(
+    clientId: ClientId,
+    packet: ExtPublishPacket,
+  ): Promise<void>;
+  getPendingIncomingPacket(
+    clientId: ClientId,
+    packetId: PacketId,
+  ): Promise<ExtPublishPacket | null>;
+  listPendingIncomingPackets(
+    clientId: ClientId,
+  ): AsyncIterableIterator<ExtPublishPacket>;
+  deletePendingIncomingPacket(
+    clientId: ClientId,
+    packetId: PacketId,
+  ): Promise<boolean>;
 
-  /**
-   * Publish a message to all subscribed clients
-   * @param {Topic} topic - Topic to publish to
-   * @param {PublishPacket} packet - Packet containing the message
-   */
-  publish(topic: Topic, packet: PublishPacket): void;
+  // Outgoing Packet management
+  addPendingOutgoingPacket(
+    clientId: ClientId,
+    packet: ExtPublishPacket,
+  ): Promise<void>;
+  getPendingOutgoingPacket(
+    clientId: ClientId,
+    packetId: PacketId,
+  ): Promise<ExtPublishPacket | null>;
+  updatePendingOutgoingPacket(
+    clientId: ClientId,
+    packetId: PacketId,
+    dup: boolean,
+  ): Promise<boolean>;
+  listPendingOutgoingPackets(
+    clientId: ClientId,
+  ): AsyncIterableIterator<ExtPublishPacket>;
+  deletePendingOutgoingPacket(
+    clientId: ClientId,
+    packetId: PacketId,
+  ): Promise<boolean>;
 
-  /**
-   * Subscribe a client to a topic
-   * @param {IStore} store - Client's message store
-   * @param {Topic} topic - Topic to subscribe to
-   * @param {QoS} qos - Quality of Service level
-   */
-  subscribe(store: IStore, topic: Topic, qos: QoS): void;
+  // Acks
+  addPendingAck(clientId: ClientId, packetId: PacketId): Promise<void>;
+  hasPendingAck(clientId: ClientId, packetId: PacketId): Promise<boolean>;
+  deletePendingAck(clientId: ClientId, packetId: PacketId): Promise<boolean>;
+  listPendingAcks(clientId: ClientId): AsyncIterableIterator<PacketId>;
 
-  /**
-   * Unsubscribe a client from a topic
-   * @param {IStore} store - Client's message store
-   * @param {Topic} topic - Topic to unsubscribe from
-   */
-  unsubscribe(store: IStore, topic: Topic): void;
+  // Message Delivery & Retained
+  publish(
+    clientId: ClientId,
+    packet: ExtPublishPacket,
+  ): Promise<void>;
+  handleRetained(
+    clientId: ClientId,
+    subscriptions: ClientSubscription[],
+  ): Promise<void>;
 
-  /**
-   * Send retained messages to a client
-   * @param {ClientId} clientId - ID of client to send retained messages to
-   */
-  handleRetained(clientId: ClientId): void;
+  // Packet ID Generation
+  nextPacketId(clientId: ClientId): Promise<PacketId>;
 }

package/persistence/sqlite/sqliteDatabase.ts

@@ -0,0 +1,294 @@
+import sqlite from "node:sqlite";
+import type { ClientId } from "../../mqttPacket/types.ts";
+
+/**
+ * Instantiates the physical relational structures and schemas required by the persistence engine.
+ * @param filename Filepath location or memory target specifier.
+ */
+export function initializeDatabase(filename: string): sqlite.DatabaseSync {
+  const db = new sqlite.DatabaseSync(filename);
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+    PRAGMA synchronous = NORMAL;
+    
+    CREATE TABLE IF NOT EXISTS client_sessions (
+      client_id    TEXT PRIMARY KEY,
+      session_data TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS subscriptions (
+      client_id         TEXT NOT NULL,
+      topic             TEXT NOT NULL,
+      share_name        TEXT NOT NULL,
+      subscription_data TEXT NOT NULL,
+      PRIMARY KEY (client_id, topic, share_name)
+    );
+
+    CREATE TABLE IF NOT EXISTS pending_incoming (
+      seq_id     INTEGER PRIMARY KEY AUTOINCREMENT, -- Guarantees absolute insertion order
+      client_id  TEXT NOT NULL,
+      packet_id  INTEGER NOT NULL,
+      packet     TEXT NOT NULL,
+      payload    BLOB,
+      created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      UNIQUE (client_id, packet_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS pending_outgoing (
+      seq_id     INTEGER PRIMARY KEY AUTOINCREMENT, -- Guarantees absolute insertion order
+      client_id  TEXT NOT NULL,
+      packet_id  INTEGER NOT NULL,
+      packet     TEXT NOT NULL,
+      payload    BLOB,
+      created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      UNIQUE (client_id, packet_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS pending_ack_outgoing (
+      client_id TEXT NOT NULL,
+      packet_id INTEGER NOT NULL,
+      PRIMARY KEY (client_id, packet_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS retained (
+      topic   TEXT PRIMARY KEY,
+      packet  TEXT NOT NULL,
+      payload BLOB
+    );
+  `);
+  return db;
+}
+
+export type AllStatements = {
+  // Sessions
+  saveSession: sqlite.StatementSync;
+  getSession: sqlite.StatementSync;
+  listAllSessions: sqlite.StatementSync;
+
+  // Subscriptions
+  saveSubscription: sqlite.StatementSync;
+  deleteSubscription: sqlite.StatementSync;
+  listSubscriptions: sqlite.StatementSync;
+  listAllSubscriptions: sqlite.StatementSync;
+
+  // Pending Incoming Packets
+  saveIncoming: sqlite.StatementSync;
+  getIncoming: sqlite.StatementSync;
+  updateIncomingDup: sqlite.StatementSync;
+  deleteIncoming: sqlite.StatementSync;
+  listIncoming: sqlite.StatementSync;
+
+  // Pending Outgoing Packets
+  saveOutgoing: sqlite.StatementSync;
+  getOutgoing: sqlite.StatementSync;
+  updateOutgoingDup: sqlite.StatementSync;
+  deleteOutgoing: sqlite.StatementSync;
+  listOutgoing: sqlite.StatementSync;
+
+  // ACKs
+  saveAck: sqlite.StatementSync;
+  hasAck: sqlite.StatementSync;
+  deleteAck: sqlite.StatementSync;
+  listAcks: sqlite.StatementSync;
+
+  // Retained
+  saveRetained: sqlite.StatementSync;
+  deleteRetained: sqlite.StatementSync;
+  getRetainedExact: sqlite.StatementSync;
+  listRetainedLike: sqlite.StatementSync;
+};
+
+/**
+ * Instantiates the physical relational structures and schemas required by the persistence engine.
+ * @param db  the database for which to prepare the statements
+ */
+export function prepareAllStatements(db: sqlite.DatabaseSync): AllStatements {
+  return {
+    // Sessions
+    saveSession: db.prepare(`
+      INSERT INTO client_sessions (client_id, session_data) 
+      VALUES (?, ?)
+      ON CONFLICT(client_id) DO UPDATE SET 
+        session_data = excluded.session_data
+    `),
+
+    getSession: db.prepare(`
+      SELECT session_data 
+      FROM client_sessions 
+      WHERE client_id = ?
+    `),
+
+    listAllSessions: db.prepare(`
+      SELECT client_id, session_data 
+      FROM client_sessions 
+    `),
+
+    // Subscriptions
+    saveSubscription: db.prepare(`
+      INSERT INTO subscriptions (client_id, topic, share_name, subscription_data) 
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(client_id, topic, share_name) DO UPDATE SET 
+        subscription_data = excluded.subscription_data
+    `),
+
+    deleteSubscription: db.prepare(`
+      DELETE FROM subscriptions 
+      WHERE client_id = ? AND topic = ? AND share_name = ?
+    `),
+
+    listSubscriptions: db.prepare(`
+      SELECT topic, share_name, subscription_data 
+      FROM subscriptions 
+      WHERE client_id = ?
+    `),
+
+    listAllSubscriptions: db.prepare(`
+      SELECT client_id, topic, share_name, subscription_data 
+      FROM subscriptions
+    `),
+
+    // Pending Incoming Packets
+    saveIncoming: db.prepare(`
+      INSERT INTO pending_incoming (client_id, packet_id, packet, payload)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(client_id, packet_id) DO UPDATE SET 
+        packet = excluded.packet, 
+        payload = excluded.payload
+    `),
+
+    getIncoming: db.prepare(`
+      SELECT packet, payload
+      FROM pending_incoming 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    updateIncomingDup: db.prepare(`
+      UPDATE pending_incoming 
+      SET packet = json_set(packet, '$.dup', json(?)) 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    deleteIncoming: db.prepare(`
+      DELETE FROM pending_incoming 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    listIncoming: db.prepare(`
+      SELECT packet, payload
+      FROM pending_incoming 
+      WHERE client_id = ? 
+      ORDER BY seq_id ASC
+    `),
+
+    // Pending Outgoing Packets
+    saveOutgoing: db.prepare(`
+      INSERT INTO pending_outgoing (client_id, packet_id, packet, payload)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(client_id, packet_id) DO UPDATE SET 
+        packet = excluded.packet, 
+        payload = excluded.payload
+    `),
+
+    getOutgoing: db.prepare(`
+      SELECT packet, payload
+      FROM pending_outgoing 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    updateOutgoingDup: db.prepare(`
+      UPDATE pending_outgoing 
+      SET packet = json_set(packet, '$.dup', json(?)) 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    deleteOutgoing: db.prepare(`
+      DELETE FROM pending_outgoing 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    listOutgoing: db.prepare(`
+      SELECT packet, payload
+      FROM pending_outgoing 
+      WHERE client_id = ? 
+      ORDER BY seq_id ASC
+    `),
+
+    // ACKs
+    saveAck: db.prepare(`
+      INSERT INTO pending_ack_outgoing (client_id, packet_id)
+      VALUES (?, ?)
+      ON CONFLICT(client_id, packet_id) DO NOTHING
+    `),
+
+    hasAck: db.prepare(`
+      SELECT 1 
+      FROM pending_ack_outgoing 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    deleteAck: db.prepare(`
+      DELETE FROM pending_ack_outgoing 
+      WHERE client_id = ? AND packet_id = ?
+    `),
+
+    listAcks: db.prepare(`
+      SELECT packet_id 
+      FROM pending_ack_outgoing 
+      WHERE client_id = ?
+    `),
+
+    // Retained Messages
+    saveRetained: db.prepare(`
+      INSERT INTO retained (topic, packet, payload)
+      VALUES (?, ?, ?)
+      ON CONFLICT(topic) DO UPDATE SET 
+        packet = excluded.packet, 
+        payload = excluded.payload
+    `),
+
+    deleteRetained: db.prepare(`
+      DELETE FROM retained 
+      WHERE topic = ?
+    `),
+
+    getRetainedExact: db.prepare(`
+      SELECT packet, payload 
+      FROM retained 
+      WHERE topic = ?
+    `),
+
+    listRetainedLike: db.prepare(`
+      SELECT topic, packet, payload 
+      FROM retained 
+      WHERE topic LIKE ?
+    `),
+  };
+}
+
+/**
+ * Truncates and drops relational state assignments under targeted clients inside atomic scopes.
+ * @param db Connection reference context.
+ * @param clientId Specific target client ID tracking token.
+ */
+export function deleteClientState(
+  db: sqlite.DatabaseSync,
+  clientId: ClientId,
+): void {
+  db.exec("begin;");
+  db.prepare(
+    "delete from subscriptions where client_id = ?",
+  ).run(clientId);
+  db.prepare(
+    "delete from pending_incoming where client_id = ?",
+  ).run(clientId);
+  db.prepare(
+    "delete from pending_outgoing where client_id = ?",
+  ).run(clientId);
+  db.prepare(
+    "delete from pending_ack_outgoing where client_id = ?",
+  ).run(clientId);
+  db.prepare(
+    "delete from client_sessions where client_id = ?",
+  ).run(clientId);
+  db.exec("commit;");
+}

package/persistence/sqlite/sqlitePersistence.test.ts

@@ -0,0 +1,48 @@
+import { test } from "node:test";
+import { initializeDatabase } from "./sqliteDatabase.ts";
+import { SqlitePersistence } from "./sqlitePersistence.ts";
+import { SqliteStorage } from "./sqliteStorage.ts";
+import { PacketDirection } from "../storage.ts";
+import { PacketType } from "../deps.ts";
+import assert from "node:assert/strict";
+
+const utf8Encoder = new TextEncoder();
+
+test("SqlitePersistence - Trie is correctly rebuilt (restored) from the database", async () => {
+  // Manually create an in-memory (:memory:) database and set up the tables
+  const sharedDb = initializeDatabase(":memory:");
+  // Simulate an existing state by directly injecting data into the sessions table
+  const storage = new SqliteStorage(sharedDb);
+  storage.saveSession("Client_A", { existingSession: true });
+  storage.saveSession("Client_B", { existingSession: true });
+  storage.saveSubscription("Client_A", {
+    topicFilter: "sensor/temperature",
+    shareName: "",
+    qos: 1,
+  });
+  storage.saveSubscription("Client_B", {
+    topicFilter: "sensor/#",
+    shareName: "",
+    qos: 2,
+  });
+  const persistence = await SqlitePersistence.start(sharedDb);
+  const topic = "sensor/temperature";
+  await persistence.publish("Client_C", {
+    type: PacketType.publish,
+    protocolLevel: 5,
+    topic,
+    payload: utf8Encoder.encode("25 degrees"),
+    qos: 2,
+  });
+  const packetsA = await Array.fromAsync(
+    storage.listPendingPackets("Client_A", PacketDirection.Outgoing),
+  );
+  assert.strictEqual(packetsA[0].topic, topic);
+  assert.strictEqual(packetsA[0].qos, 1);
+  const packetsB = await Array.fromAsync(
+    storage.listPendingPackets("Client_B", PacketDirection.Outgoing),
+  );
+  assert.strictEqual(packetsB[0].topic, topic);
+  assert.strictEqual(packetsB[0].qos, 2);
+  persistence.close();
+});

package/persistence/sqlite/sqlitePersistence.ts

@@ -0,0 +1,36 @@
+import { MqttPersistence } from "../basicPersistence.ts";
+import { SqliteStorage } from "./sqliteStorage.ts";
+import type sqlite from "node:sqlite";
+
+const SQLITE_DATABASE_URL = ":memory:";
+
+export class SqlitePersistence extends MqttPersistence {
+  private storageInstance: SqliteStorage;
+
+  constructor(db: string | sqlite.DatabaseSync = SQLITE_DATABASE_URL) {
+    const storage = new SqliteStorage(db);
+    super(storage);
+    this.storageInstance = storage;
+  }
+
+  /**
+   * Convenient lifecycle method combining instance creation and initialization.
+   */
+  static async start(
+    db: string | sqlite.DatabaseSync = SQLITE_DATABASE_URL,
+  ): Promise<SqlitePersistence> {
+    const persistence = new SqlitePersistence(db);
+    await persistence.initialize();
+    return persistence;
+  }
+
+  /**
+   * Overrides close to make sure the underlying database file
+   * handle is cleaned up correctly when the persistence is halted.
+   */
+  async close(): Promise<void> {
+    if (this.storageInstance.close) {
+      await this.storageInstance.close();
+    }
+  }
+}