@fedify/sqlite 2.0.0-dev.215 → 2.0.0-dev.226

package/dist/mq.js

@@ -0,0 +1,368 @@
+
+        import { Temporal } from "@js-temporal/polyfill";
+      
+import { SqliteDatabase } from "#sqlite";
+import { getLogger } from "@logtape/logtape";
+
+//#region src/mq.ts
+const logger = getLogger([
+	"fedify",
+	"sqlite",
+	"mq"
+]);
+var EnqueueEvent = class extends Event {
+	delayMs;
+	constructor(delayMs) {
+		super("enqueue");
+		this.delayMs = delayMs;
+	}
+};
+/**
+* A message queue that uses SQLite as the underlying storage.
+*
+* This implementation is designed for single-node deployments and uses
+* polling to check for new messages. It is not suitable for high-throughput
+* scenarios or distributed environments.
+*
+* @example
+* ```ts ignore
+* import { createFederation } from "@fedify/fedify";
+* import { SqliteMessageQueue } from "@fedify/sqlite";
+* import { DatabaseSync } from "node:sqlite";
+*
+* const db = new DatabaseSync(":memory:");
+* const federation = createFederation({
+*   // ...
+*   queue: new SqliteMessageQueue(db),
+* });
+* ```
+*/
+var SqliteMessageQueue = class SqliteMessageQueue {
+	static #defaultTableName = "fedify_message";
+	static #tableNameRegex = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
+	static #notifyChannels = /* @__PURE__ */ new Map();
+	static #activeInstances = /* @__PURE__ */ new Map();
+	static #getNotifyChannel(tableName) {
+		let channel = SqliteMessageQueue.#notifyChannels.get(tableName);
+		if (channel == null) {
+			channel = new EventTarget();
+			SqliteMessageQueue.#notifyChannels.set(tableName, channel);
+		}
+		return channel;
+	}
+	#db;
+	#tableName;
+	#pollIntervalMs;
+	#instanceId;
+	#maxRetries;
+	#retryDelayMs;
+	#journalMode;
+	#initialized;
+	/**
+	* SQLite message queue does not provide native retry mechanisms.
+	*/
+	nativeRetrial = false;
+	/**
+	* Creates a new SQLite message queue.
+	* @param db The SQLite database to use. Supports `node:sqlite`, `bun:sqlite`.
+	* @param options The options for the message queue.
+	*/
+	constructor(db, options = {}) {
+		this.db = db;
+		this.options = options;
+		this.#db = new SqliteDatabase(db);
+		this.#initialized = options.initialized ?? false;
+		this.#tableName = options.tableName ?? SqliteMessageQueue.#defaultTableName;
+		this.#instanceId = crypto.randomUUID();
+		this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
+		this.#maxRetries = options.maxRetries ?? 5;
+		this.#retryDelayMs = options.retryDelayMs ?? 100;
+		this.#journalMode = options.journalMode ?? "WAL";
+		if (!SqliteMessageQueue.#tableNameRegex.test(this.#tableName)) throw new Error(`Invalid table name for the message queue: ${this.#tableName}.\
+ Only letters, digits, and underscores are allowed.`);
+		this.#registerInstance();
+	}
+	#registerInstance() {
+		let instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
+		if (instances == null) {
+			instances = /* @__PURE__ */ new Set();
+			SqliteMessageQueue.#activeInstances.set(this.#tableName, instances);
+		}
+		instances.add(this.#instanceId);
+	}
+	/**
+	* {@inheritDoc MessageQueue.enqueue}
+	*/
+	enqueue(message, options) {
+		this.initialize();
+		const id = crypto.randomUUID();
+		const encodedMessage = this.#encodeMessage(message);
+		const now = Temporal.Now.instant().epochMilliseconds;
+		const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
+		const scheduled = now + delay.total({ unit: "milliseconds" });
+		if (options?.delay) logger.debug("Enqueuing a message with a delay of {delay}...", {
+			delay,
+			message
+		});
+		else logger.debug("Enqueuing a message...", { message });
+		return this.#retryOnBusy(() => {
+			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
+          VALUES (?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled);
+			logger.debug("Enqueued a message.", { message });
+			const delayMs = delay.total("millisecond");
+			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
+		});
+	}
+	/**
+	* {@inheritDoc MessageQueue.enqueueMany}
+	*/
+	enqueueMany(messages, options) {
+		if (messages.length === 0) return Promise.resolve();
+		this.initialize();
+		const now = Temporal.Now.instant().epochMilliseconds;
+		const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
+		const scheduled = now + delay.total({ unit: "milliseconds" });
+		if (options?.delay) logger.debug("Enqueuing messages with a delay of {delay}...", {
+			delay,
+			messages
+		});
+		else logger.debug("Enqueuing messages...", { messages });
+		return this.#withTransactionRetries(() => {
+			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
+        VALUES (?, ?, ?, ?)`);
+			for (const message of messages) {
+				const id = crypto.randomUUID();
+				const encodedMessage = this.#encodeMessage(message);
+				stmt.run(id, encodedMessage, now, scheduled);
+			}
+			logger.debug("Enqueued messages.", { messages });
+			const delayMs = delay.total("millisecond");
+			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
+		}).catch((error) => {
+			logger.error("Failed to enqueue messages to table {tableName}: {error}", {
+				tableName: this.#tableName,
+				messageCount: messages.length,
+				error
+			});
+			throw error;
+		});
+	}
+	/**
+	* {@inheritDoc MessageQueue.listen}
+	*/
+	async listen(handler, options) {
+		this.initialize();
+		const { signal } = options ?? {};
+		logger.debug("Starting to listen for messages on table {tableName}...", { tableName: this.#tableName });
+		const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
+		const timeouts = /* @__PURE__ */ new Set();
+		const poll = async () => {
+			while (signal == null || !signal.aborted) {
+				const now = Temporal.Now.instant().epochMilliseconds;
+				const result = await this.#withTransactionRetries(() => {
+					return this.#db.prepare(`DELETE FROM "${this.#tableName}"
+              WHERE id = (
+                SELECT id FROM "${this.#tableName}"
+                WHERE scheduled <= ?
+                ORDER BY scheduled
+                LIMIT 1
+              )
+              RETURNING id, message`).get(now);
+				});
+				if (result) {
+					const message = this.#decodeMessage(result.message);
+					logger.debug("Processing message {id}...", {
+						id: result.id,
+						message
+					});
+					try {
+						await handler(message);
+						logger.debug("Processed message {id}.", { id: result.id });
+					} catch (error) {
+						logger.error("Failed to process message {id} from table {tableName}: {error}", {
+							id: result.id,
+							tableName: this.#tableName,
+							message,
+							error
+						});
+					}
+					continue;
+				}
+				break;
+			}
+		};
+		const onEnqueue = (event) => {
+			const delayMs = event.delayMs;
+			if (delayMs < 1) poll();
+			else timeouts.add(setTimeout(poll, delayMs));
+		};
+		channel.addEventListener("enqueue", onEnqueue);
+		signal?.addEventListener("abort", () => {
+			channel.removeEventListener("enqueue", onEnqueue);
+			for (const timeout of timeouts) clearTimeout(timeout);
+		});
+		await poll();
+		while (signal == null || !signal.aborted) {
+			let timeout;
+			await new Promise((resolve) => {
+				const onAbort = () => {
+					signal?.removeEventListener("abort", onAbort);
+					resolve(void 0);
+				};
+				signal?.addEventListener("abort", onAbort);
+				timeout = setTimeout(() => {
+					signal?.removeEventListener("abort", onAbort);
+					resolve(0);
+				}, this.#pollIntervalMs);
+				timeouts.add(timeout);
+			});
+			if (timeout != null) timeouts.delete(timeout);
+			await poll();
+		}
+		logger.debug("Stopped listening for messages on table {tableName}.", { tableName: this.#tableName });
+	}
+	/**
+	* Creates the message queue table if it does not already exist.
+	* Does nothing if the table already exists.
+	*
+	* This method also configures the SQLite journal mode for better concurrency.
+	* WAL (Write-Ahead Logging) mode is enabled by default to improve
+	* concurrent access in multi-process environments.
+	*/
+	initialize() {
+		if (this.#initialized) return;
+		logger.debug("Initializing the message queue table {tableName}...", { tableName: this.#tableName });
+		this.#db.exec(`PRAGMA journal_mode=${this.#journalMode}`);
+		this.#withTransaction(() => {
+			this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}" (
+          id TEXT PRIMARY KEY,
+          message TEXT NOT NULL,
+          created INTEGER NOT NULL,
+          scheduled INTEGER NOT NULL
+        )
+      `);
+			this.#db.exec(`
+        CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
+        ON "${this.#tableName}" (scheduled)
+      `);
+		});
+		this.#initialized = true;
+		logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
+	}
+	/**
+	* Drops the table used by the message queue.  Does nothing if the table
+	* does not exist.
+	*/
+	drop() {
+		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
+		this.#initialized = false;
+	}
+	/**
+	* Closes the database connection.
+	*/
+	[Symbol.dispose]() {
+		try {
+			this.#db.close();
+			this.#unregisterInstance();
+		} catch (error) {
+			logger.error("Failed to close the database connection for table {tableName}: {error}", {
+				tableName: this.#tableName,
+				error
+			});
+		}
+	}
+	#unregisterInstance() {
+		const instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
+		if (instances == null) return;
+		instances.delete(this.#instanceId);
+		if (instances.size === 0) {
+			SqliteMessageQueue.#activeInstances.delete(this.#tableName);
+			SqliteMessageQueue.#notifyChannels.delete(this.#tableName);
+		}
+	}
+	/**
+	* Checks if an error is a SQLITE_BUSY error or transaction conflict.
+	* Handles different error formats from node:sqlite and bun:sqlite.
+	*/
+	#isBusyError(error) {
+		if (!(error instanceof Error)) return false;
+		if (error.message.includes("SQLITE_BUSY") || error.message.includes("database is locked") || error.message.includes("transaction within a transaction")) return true;
+		const errorWithCode = error;
+		if (errorWithCode.code === "SQLITE_BUSY") return true;
+		const errorWithErrno = error;
+		if (errorWithErrno.errno === 5) return true;
+		return false;
+	}
+	/**
+	* Retries a database operation with exponential backoff on SQLITE_BUSY errors.
+	*/
+	async #retryOnBusy(operation) {
+		let lastError;
+		for (let attempt = 0; attempt <= this.#maxRetries; attempt++) try {
+			return operation();
+		} catch (error) {
+			lastError = error;
+			if (!this.#isBusyError(error)) {
+				logger.error("Database operation failed on table {tableName}: {error}", {
+					tableName: this.#tableName,
+					error
+				});
+				throw error;
+			}
+			if (attempt === this.#maxRetries) {
+				logger.error("Max retries ({maxRetries}) reached for SQLITE_BUSY error on table {tableName}.", {
+					maxRetries: this.#maxRetries,
+					tableName: this.#tableName,
+					error
+				});
+				throw error;
+			}
+			const delayMs = this.#retryDelayMs * Math.pow(2, attempt);
+			logger.debug("SQLITE_BUSY error on table {tableName}, retrying in {delayMs}ms (attempt {attempt}/{maxRetries})...", {
+				tableName: this.#tableName,
+				delayMs,
+				attempt: attempt + 1,
+				maxRetries: this.#maxRetries
+			});
+			await new Promise((resolve) => setTimeout(resolve, delayMs));
+		}
+		throw lastError;
+	}
+	/**
+	* Executes a database operation within a transaction.
+	* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
+	*/
+	#withTransaction(operation) {
+		let transactionStarted = false;
+		try {
+			this.#db.exec("BEGIN IMMEDIATE");
+			transactionStarted = true;
+			const result = operation();
+			this.#db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (transactionStarted) try {
+				this.#db.exec("ROLLBACK");
+			} catch {}
+			throw error;
+		}
+	}
+	/**
+	* Executes a database operation within a transaction with retry logic.
+	* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
+	* Retries on SQLITE_BUSY errors with exponential backoff.
+	*/
+	async #withTransactionRetries(operation) {
+		return await this.#retryOnBusy(() => this.#withTransaction(operation));
+	}
+	#encodeMessage(message) {
+		return JSON.stringify(message);
+	}
+	#decodeMessage(message) {
+		return JSON.parse(message);
+	}
+};
+
+//#endregion
+export { SqliteMessageQueue };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fedify/sqlite",
-  "version": "2.0.0-dev.215+84aa16bb",
+  "version": "2.0.0-dev.226+c28b813d",
   "description": "SQLite drivers for Fedify",
   "keywords": [
     "fedify",
@@ -46,6 +46,16 @@
       "require": "./dist/kv.cjs",
       "default": "./dist/kv.js"
     },
+    "./mq": {
+      "types": {
+        "import": "./dist/mq.d.ts",
+        "require": "./dist/mq.d.cts",
+        "default": "./dist/mq.d.ts"
+      },
+      "import": "./dist/mq.js",
+      "require": "./dist/mq.cjs",
+      "default": "./dist/mq.js"
+    },
     "./package.json": "./package.json"
   },
   "imports": {
@@ -62,12 +72,13 @@
     "es-toolkit": "^1.31.0"
   },
   "peerDependencies": {
-    "@fedify/fedify": "^2.0.0-dev.215+84aa16bb"
+    "@fedify/fedify": "^2.0.0-dev.226+c28b813d"
   },
   "devDependencies": {
     "@std/async": "npm:@jsr/std__async@^1.0.13",
     "tsdown": "^0.12.9",
-    "typescript": "^5.9.3"
+    "typescript": "^5.9.3",
+    "@fedify/testing": "^2.0.0-dev.226+c28b813d"
   },
   "scripts": {
     "build": "tsdown",

package/src/mod.ts

@@ -3,3 +3,4 @@
  * @module
  */
 export { SqliteKvStore } from "./kv.ts";
+export { SqliteMessageQueue } from "./mq.ts";

package/src/mq.test.ts

@@ -0,0 +1,23 @@
+import { PlatformDatabase } from "#sqlite";
+import { test } from "@fedify/fixture";
+import { SqliteMessageQueue } from "@fedify/sqlite/mq";
+import { getRandomKey, testMessageQueue } from "@fedify/testing";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const dbDir = await mkdtemp(join(tmpdir(), "fedify-sqlite-"));
+const dbPath = join(dbDir, `${getRandomKey("sqlite")}.db`);
+const db = new PlatformDatabase(dbPath);
+const tableName = getRandomKey("message").replaceAll("-", "_");
+
+test("SqliteMessageQueue", () =>
+  testMessageQueue(
+    () => new SqliteMessageQueue(db, { tableName }),
+    ({ mq1, mq2, controller }) => {
+      controller.abort();
+      mq1.drop();
+      mq1[Symbol.dispose]();
+      mq2[Symbol.dispose]();
+    },
+  ));