@fedify/sqlite 2.0.0-pr.490.2 → 2.0.0-pr.559.4

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright 2024–2025 Hong Minhee
+Copyright 2024–2026 Hong Minhee
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

package/README.md

@@ -6,12 +6,12 @@
 
 This package provides a SQLite-based [`KvStore`] implementation for [Fedify].
 
-[JSR]: https://jsr.io/@fedify/sqlite
 [JSR badge]: https://jsr.io/badges/@fedify/sqlite
-[npm]: https://www.npmjs.com/package/@fedify/sqlite
+[JSR]: https://jsr.io/@fedify/sqlite
 [npm badge]: https://img.shields.io/npm/v/@fedify/sqlite?logo=npm
-[Fedify]: https://fedify.dev/
+[npm]: https://www.npmjs.com/package/@fedify/sqlite
 [`KvStore`]: https://jsr.io/@fedify/fedify/doc/federation/~/KvStore
+[Fedify]: https://fedify.dev/
 
 
 Usage

package/deno.json

@@ -1,10 +1,11 @@
 {
   "name": "@fedify/sqlite",
-  "version": "2.0.0-pr.490.2+99a396d5",
+  "version": "2.0.0-pr.559.4+6357309b",
   "license": "MIT",
   "exports": {
     ".": "./src/mod.ts",
-    "./kv": "./src/kv.ts"
+    "./kv": "./src/kv.ts",
+    "./mq": "./src/mq.ts"
   },
   "imports": {
     "#sqlite": "./src/sqlite.node.ts"
@@ -16,11 +17,13 @@
   ],
   "publish": {
     "exclude": [
-      "!dist/"
+      "**/*.test.ts",
+      "!dist/",
+      "tsdown.config.ts"
     ]
   },
   "tasks": {
     "check": "deno fmt --check && deno lint && deno check",
-    "test": "deno test --allow-net --allow-env --doc --no-check=leaks"
+    "test": "deno test --allow-net --allow-env --allow-read --allow-write --doc --no-check=leaks"
   }
 }

package/dist/kv.cjs

@@ -134,6 +134,36 @@ var SqliteKvStore = class SqliteKvStore {
 		}
 	}
 	/**
+	* {@inheritDoc KvStore.list}
+	* @since 1.10.0
+	*/
+	async *list(prefix) {
+		this.initialize();
+		const now = Temporal.Now.instant().epochMilliseconds;
+		let results;
+		if (prefix == null || prefix.length === 0) results = this.#db.prepare(`
+          SELECT key, value
+          FROM "${this.#tableName}"
+          WHERE expires IS NULL OR expires > ?
+          ORDER BY key
+        `).all(now);
+		else {
+			const pattern = JSON.stringify(prefix).slice(0, -1) + ",%";
+			const exactKey = JSON.stringify(prefix);
+			results = this.#db.prepare(`
+          SELECT key, value
+          FROM "${this.#tableName}"
+          WHERE (key LIKE ? ESCAPE '\\' OR key = ?)
+            AND (expires IS NULL OR expires > ?)
+          ORDER BY key
+        `).all(pattern, exactKey, now);
+		}
+		for (const row of results) yield {
+			key: this.#decodeKey(row.key),
+			value: this.#decodeValue(row.value)
+		};
+	}
+	/**
 	* Creates the table used by the key–value store if it does not already exist.
 	* Does nothing if the table already exists.
 	*/

package/dist/kv.d.cts

@@ -1,5 +1,5 @@
 import { PlatformDatabase } from "#sqlite";
-import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
+import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
 
 //#region src/kv.d.ts
 /**
@@ -12,12 +12,12 @@ interface SqliteKvStoreOptions {
    * `"fedify_kv"` by default.
    * @default `"fedify_kv"`
    */
-  tableName?: string;
+  readonly tableName?: string;
   /**
    * Whether the table has been initialized.  `false` by default.
    * @default `false`
    */
-  initialized?: boolean;
+  readonly initialized?: boolean;
 }
 /**
  * A key–value store that uses SQLite as the underlying storage.
@@ -61,6 +61,11 @@ declare class SqliteKvStore implements KvStore {
    * {@inheritDoc KvStore.cas}
    */
   cas(key: KvKey, expectedValue: unknown, newValue: unknown, options?: KvStoreSetOptions): Promise<boolean>;
+  /**
+   * {@inheritDoc KvStore.list}
+   * @since 1.10.0
+   */
+  list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
   /**
    * Creates the table used by the key–value store if it does not already exist.
    * Does nothing if the table already exists.

package/dist/kv.d.ts

@@ -1,6 +1,6 @@
 import { Temporal } from "@js-temporal/polyfill";
 import { PlatformDatabase } from "#sqlite";
-import { KvKey, KvStore, KvStoreSetOptions } from "@fedify/fedify";
+import { KvKey, KvStore, KvStoreListEntry, KvStoreSetOptions } from "@fedify/fedify";
 
 //#region src/kv.d.ts
 /**
@@ -13,12 +13,12 @@ interface SqliteKvStoreOptions {
    * `"fedify_kv"` by default.
    * @default `"fedify_kv"`
    */
-  tableName?: string;
+  readonly tableName?: string;
   /**
    * Whether the table has been initialized.  `false` by default.
    * @default `false`
    */
-  initialized?: boolean;
+  readonly initialized?: boolean;
 }
 /**
  * A key–value store that uses SQLite as the underlying storage.
@@ -62,6 +62,11 @@ declare class SqliteKvStore implements KvStore {
    * {@inheritDoc KvStore.cas}
    */
   cas(key: KvKey, expectedValue: unknown, newValue: unknown, options?: KvStoreSetOptions): Promise<boolean>;
+  /**
+   * {@inheritDoc KvStore.list}
+   * @since 1.10.0
+   */
+  list(prefix?: KvKey): AsyncIterable<KvStoreListEntry>;
   /**
    * Creates the table used by the key–value store if it does not already exist.
    * Does nothing if the table already exists.

package/dist/kv.js

@@ -133,6 +133,36 @@ var SqliteKvStore = class SqliteKvStore {
 		}
 	}
 	/**
+	* {@inheritDoc KvStore.list}
+	* @since 1.10.0
+	*/
+	async *list(prefix) {
+		this.initialize();
+		const now = Temporal.Now.instant().epochMilliseconds;
+		let results;
+		if (prefix == null || prefix.length === 0) results = this.#db.prepare(`
+          SELECT key, value
+          FROM "${this.#tableName}"
+          WHERE expires IS NULL OR expires > ?
+          ORDER BY key
+        `).all(now);
+		else {
+			const pattern = JSON.stringify(prefix).slice(0, -1) + ",%";
+			const exactKey = JSON.stringify(prefix);
+			results = this.#db.prepare(`
+          SELECT key, value
+          FROM "${this.#tableName}"
+          WHERE (key LIKE ? ESCAPE '\\' OR key = ?)
+            AND (expires IS NULL OR expires > ?)
+          ORDER BY key
+        `).all(pattern, exactKey, now);
+		}
+		for (const row of results) yield {
+			key: this.#decodeKey(row.key),
+			value: this.#decodeValue(row.value)
+		};
+	}
+	/**
 	* Creates the table used by the key–value store if it does not already exist.
 	* Does nothing if the table already exists.
 	*/

package/dist/mod.cjs

@@ -2,5 +2,7 @@
         const { Temporal } = require("@js-temporal/polyfill");
       
 const require_kv = require('./kv.cjs');
+const require_mq = require('./mq.cjs');
 
-exports.SqliteKvStore = require_kv.SqliteKvStore;
+exports.SqliteKvStore = require_kv.SqliteKvStore;
+exports.SqliteMessageQueue = require_mq.SqliteMessageQueue;

package/dist/mod.d.cts

@@ -1,2 +1,3 @@
 import { SqliteKvStore } from "./kv.cjs";
-export { SqliteKvStore };
+import { SqliteMessageQueue } from "./mq.cjs";
+export { SqliteKvStore, SqliteMessageQueue };

package/dist/mod.d.ts

@@ -1,3 +1,4 @@
 import { Temporal } from "@js-temporal/polyfill";
 import { SqliteKvStore } from "./kv.js";
-export { SqliteKvStore };
+import { SqliteMessageQueue } from "./mq.js";
+export { SqliteKvStore, SqliteMessageQueue };

package/dist/mod.js

@@ -2,5 +2,6 @@
         import { Temporal } from "@js-temporal/polyfill";
       
 import { SqliteKvStore } from "./kv.js";
+import { SqliteMessageQueue } from "./mq.js";
 
-export { SqliteKvStore };
+export { SqliteKvStore, SqliteMessageQueue };

package/dist/mq.cjs

@@ -0,0 +1,393 @@
+
+        const { Temporal } = require("@js-temporal/polyfill");
+      
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+const __sqlite = require_rolldown_runtime.__toESM(require("#sqlite"));
+const __logtape_logtape = require_rolldown_runtime.__toESM(require("@logtape/logtape"));
+
+//#region src/mq.ts
+const logger = (0, __logtape_logtape.getLogger)([
+	"fedify",
+	"sqlite",
+	"mq"
+]);
+var EnqueueEvent = class extends Event {
+	delayMs;
+	constructor(delayMs) {
+		super("enqueue");
+		this.delayMs = delayMs;
+	}
+};
+/**
+* A message queue that uses SQLite as the underlying storage.
+*
+* This implementation is designed for single-node deployments and uses
+* polling to check for new messages. It is not suitable for high-throughput
+* scenarios or distributed environments.
+*
+* @example
+* ```ts ignore
+* import { createFederation } from "@fedify/fedify";
+* import { SqliteMessageQueue } from "@fedify/sqlite";
+* import { DatabaseSync } from "node:sqlite";
+*
+* const db = new DatabaseSync(":memory:");
+* const federation = createFederation({
+*   // ...
+*   queue: new SqliteMessageQueue(db),
+* });
+* ```
+*/
+var SqliteMessageQueue = class SqliteMessageQueue {
+	static #defaultTableName = "fedify_message";
+	static #tableNameRegex = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
+	static #notifyChannels = /* @__PURE__ */ new Map();
+	static #activeInstances = /* @__PURE__ */ new Map();
+	static #getNotifyChannel(tableName) {
+		let channel = SqliteMessageQueue.#notifyChannels.get(tableName);
+		if (channel == null) {
+			channel = new EventTarget();
+			SqliteMessageQueue.#notifyChannels.set(tableName, channel);
+		}
+		return channel;
+	}
+	#db;
+	#tableName;
+	#pollIntervalMs;
+	#instanceId;
+	#maxRetries;
+	#retryDelayMs;
+	#journalMode;
+	#initialized;
+	/**
+	* SQLite message queue does not provide native retry mechanisms.
+	*/
+	nativeRetrial = false;
+	/**
+	* Creates a new SQLite message queue.
+	* @param db The SQLite database to use. Supports `node:sqlite`, `bun:sqlite`.
+	* @param options The options for the message queue.
+	*/
+	constructor(db, options = {}) {
+		this.db = db;
+		this.options = options;
+		this.#db = new __sqlite.SqliteDatabase(db);
+		this.#initialized = options.initialized ?? false;
+		this.#tableName = options.tableName ?? SqliteMessageQueue.#defaultTableName;
+		this.#instanceId = crypto.randomUUID();
+		this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 5 }).total("millisecond");
+		this.#maxRetries = options.maxRetries ?? 5;
+		this.#retryDelayMs = options.retryDelayMs ?? 100;
+		this.#journalMode = options.journalMode ?? "WAL";
+		if (!SqliteMessageQueue.#tableNameRegex.test(this.#tableName)) throw new Error(`Invalid table name for the message queue: ${this.#tableName}.\
+ Only letters, digits, and underscores are allowed.`);
+		this.#registerInstance();
+	}
+	#registerInstance() {
+		let instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
+		if (instances == null) {
+			instances = /* @__PURE__ */ new Set();
+			SqliteMessageQueue.#activeInstances.set(this.#tableName, instances);
+		}
+		instances.add(this.#instanceId);
+	}
+	/**
+	* {@inheritDoc MessageQueue.enqueue}
+	*/
+	enqueue(message, options) {
+		this.initialize();
+		const id = crypto.randomUUID();
+		const encodedMessage = this.#encodeMessage(message);
+		const now = Temporal.Now.instant().epochMilliseconds;
+		const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
+		const scheduled = now + delay.total({ unit: "milliseconds" });
+		if (options?.delay) logger.debug("Enqueuing a message with a delay of {delay}...", {
+			delay,
+			message
+		});
+		else logger.debug("Enqueuing a message...", { message });
+		const orderingKey = options?.orderingKey ?? null;
+		return this.#retryOnBusy(() => {
+			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+          VALUES (?, ?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled, orderingKey);
+			logger.debug("Enqueued a message.", {
+				message,
+				orderingKey
+			});
+			const delayMs = delay.total("millisecond");
+			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
+		});
+	}
+	/**
+	* {@inheritDoc MessageQueue.enqueueMany}
+	*/
+	enqueueMany(messages, options) {
+		if (messages.length === 0) return Promise.resolve();
+		this.initialize();
+		const now = Temporal.Now.instant().epochMilliseconds;
+		const delay = options?.delay ?? Temporal.Duration.from({ seconds: 0 });
+		const scheduled = now + delay.total({ unit: "milliseconds" });
+		if (options?.delay) logger.debug("Enqueuing messages with a delay of {delay}...", {
+			delay,
+			messages
+		});
+		else logger.debug("Enqueuing messages...", { messages });
+		const orderingKey = options?.orderingKey ?? null;
+		return this.#withTransactionRetries(() => {
+			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+        VALUES (?, ?, ?, ?, ?)`);
+			for (const message of messages) {
+				const id = crypto.randomUUID();
+				const encodedMessage = this.#encodeMessage(message);
+				stmt.run(id, encodedMessage, now, scheduled, orderingKey);
+			}
+			logger.debug("Enqueued messages.", { messages });
+			const delayMs = delay.total("millisecond");
+			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
+		}).catch((error) => {
+			logger.error("Failed to enqueue messages to table {tableName}: {error}", {
+				tableName: this.#tableName,
+				messageCount: messages.length,
+				error
+			});
+			throw error;
+		});
+	}
+	/**
+	* {@inheritDoc MessageQueue.listen}
+	*/
+	async listen(handler, options) {
+		this.initialize();
+		const { signal } = options ?? {};
+		logger.debug("Starting to listen for messages on table {tableName}...", { tableName: this.#tableName });
+		const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
+		const timeouts = /* @__PURE__ */ new Set();
+		const lockTableName = `${this.#tableName}_locks`;
+		const poll = async () => {
+			while (signal == null || !signal.aborted) {
+				const now = Temporal.Now.instant().epochMilliseconds;
+				const result = await this.#withTransactionRetries(() => {
+					return this.#db.prepare(`DELETE FROM "${this.#tableName}"
+              WHERE id = (
+                SELECT id FROM "${this.#tableName}"
+                WHERE scheduled <= ?
+                  AND (ordering_key IS NULL
+                       OR ordering_key NOT IN (SELECT ordering_key FROM "${lockTableName}"))
+                ORDER BY scheduled
+                LIMIT 1
+              )
+              RETURNING id, message, ordering_key`).get(now);
+				});
+				if (result) {
+					const message = this.#decodeMessage(result.message);
+					const orderingKey = result.ordering_key;
+					if (orderingKey != null) await this.#retryOnBusy(() => {
+						this.#db.prepare(`INSERT OR IGNORE INTO "${lockTableName}" (ordering_key) VALUES (?)`).run(orderingKey);
+					});
+					logger.debug("Processing message {id}...", {
+						id: result.id,
+						message,
+						orderingKey
+					});
+					try {
+						await handler(message);
+						logger.debug("Processed message {id}.", { id: result.id });
+					} catch (error) {
+						logger.error("Failed to process message {id} from table {tableName}: {error}", {
+							id: result.id,
+							tableName: this.#tableName,
+							message,
+							error
+						});
+					} finally {
+						if (orderingKey != null) await this.#retryOnBusy(() => {
+							this.#db.prepare(`DELETE FROM "${lockTableName}" WHERE ordering_key = ?`).run(orderingKey);
+						});
+					}
+					continue;
+				}
+				break;
+			}
+		};
+		const onEnqueue = (event) => {
+			const delayMs = event.delayMs;
+			if (delayMs < 1) poll();
+			else timeouts.add(setTimeout(poll, delayMs));
+		};
+		channel.addEventListener("enqueue", onEnqueue);
+		signal?.addEventListener("abort", () => {
+			channel.removeEventListener("enqueue", onEnqueue);
+			for (const timeout of timeouts) clearTimeout(timeout);
+		});
+		await poll();
+		while (signal == null || !signal.aborted) {
+			let timeout;
+			await new Promise((resolve) => {
+				const onAbort = () => {
+					signal?.removeEventListener("abort", onAbort);
+					resolve(void 0);
+				};
+				signal?.addEventListener("abort", onAbort);
+				timeout = setTimeout(() => {
+					signal?.removeEventListener("abort", onAbort);
+					resolve(0);
+				}, this.#pollIntervalMs);
+				timeouts.add(timeout);
+			});
+			if (timeout != null) timeouts.delete(timeout);
+			await poll();
+		}
+		logger.debug("Stopped listening for messages on table {tableName}.", { tableName: this.#tableName });
+	}
+	/**
+	* Creates the message queue table if it does not already exist.
+	* Does nothing if the table already exists.
+	*
+	* This method also configures the SQLite journal mode for better concurrency.
+	* WAL (Write-Ahead Logging) mode is enabled by default to improve
+	* concurrent access in multi-process environments.
+	*/
+	initialize() {
+		if (this.#initialized) return;
+		logger.debug("Initializing the message queue table {tableName}...", { tableName: this.#tableName });
+		this.#db.exec(`PRAGMA journal_mode=${this.#journalMode}`);
+		this.#withTransaction(() => {
+			this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}" (
+          id TEXT PRIMARY KEY,
+          message TEXT NOT NULL,
+          created INTEGER NOT NULL,
+          scheduled INTEGER NOT NULL,
+          ordering_key TEXT
+        )
+      `);
+			this.#db.exec(`
+        CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
+        ON "${this.#tableName}" (scheduled)
+      `);
+			this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}_locks" (
+          ordering_key TEXT PRIMARY KEY
+        )
+      `);
+		});
+		this.#initialized = true;
+		logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
+	}
+	/**
+	* Drops the tables used by the message queue.  Does nothing if the tables
+	* do not exist.
+	*/
+	drop() {
+		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
+		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}_locks"`);
+		this.#initialized = false;
+	}
+	/**
+	* Closes the database connection.
+	*/
+	[Symbol.dispose]() {
+		try {
+			this.#db.close();
+			this.#unregisterInstance();
+		} catch (error) {
+			logger.error("Failed to close the database connection for table {tableName}: {error}", {
+				tableName: this.#tableName,
+				error
+			});
+		}
+	}
+	#unregisterInstance() {
+		const instances = SqliteMessageQueue.#activeInstances.get(this.#tableName);
+		if (instances == null) return;
+		instances.delete(this.#instanceId);
+		if (instances.size === 0) {
+			SqliteMessageQueue.#activeInstances.delete(this.#tableName);
+			SqliteMessageQueue.#notifyChannels.delete(this.#tableName);
+		}
+	}
+	/**
+	* Checks if an error is a SQLITE_BUSY error or transaction conflict.
+	* Handles different error formats from node:sqlite and bun:sqlite.
+	*/
+	#isBusyError(error) {
+		if (!(error instanceof Error)) return false;
+		if (error.message.includes("SQLITE_BUSY") || error.message.includes("database is locked") || error.message.includes("transaction within a transaction")) return true;
+		const errorWithCode = error;
+		if (errorWithCode.code === "SQLITE_BUSY") return true;
+		const errorWithErrno = error;
+		if (errorWithErrno.errno === 5) return true;
+		return false;
+	}
+	/**
+	* Retries a database operation with exponential backoff on SQLITE_BUSY errors.
+	*/
+	async #retryOnBusy(operation) {
+		let lastError;
+		for (let attempt = 0; attempt <= this.#maxRetries; attempt++) try {
+			return operation();
+		} catch (error) {
+			lastError = error;
+			if (!this.#isBusyError(error)) {
+				logger.error("Database operation failed on table {tableName}: {error}", {
+					tableName: this.#tableName,
+					error
+				});
+				throw error;
+			}
+			if (attempt === this.#maxRetries) {
+				logger.error("Max retries ({maxRetries}) reached for SQLITE_BUSY error on table {tableName}.", {
+					maxRetries: this.#maxRetries,
+					tableName: this.#tableName,
+					error
+				});
+				throw error;
+			}
+			const delayMs = this.#retryDelayMs * Math.pow(2, attempt);
+			logger.debug("SQLITE_BUSY error on table {tableName}, retrying in {delayMs}ms (attempt {attempt}/{maxRetries})...", {
+				tableName: this.#tableName,
+				delayMs,
+				attempt: attempt + 1,
+				maxRetries: this.#maxRetries
+			});
+			await new Promise((resolve) => setTimeout(resolve, delayMs));
+		}
+		throw lastError;
+	}
+	/**
+	* Executes a database operation within a transaction.
+	* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
+	*/
+	#withTransaction(operation) {
+		let transactionStarted = false;
+		try {
+			this.#db.exec("BEGIN IMMEDIATE");
+			transactionStarted = true;
+			const result = operation();
+			this.#db.exec("COMMIT");
+			return result;
+		} catch (error) {
+			if (transactionStarted) try {
+				this.#db.exec("ROLLBACK");
+			} catch {}
+			throw error;
+		}
+	}
+	/**
+	* Executes a database operation within a transaction with retry logic.
+	* Automatically handles BEGIN IMMEDIATE, COMMIT, and ROLLBACK.
+	* Retries on SQLITE_BUSY errors with exponential backoff.
+	*/
+	async #withTransactionRetries(operation) {
+		return await this.#retryOnBusy(() => this.#withTransaction(operation));
+	}
+	#encodeMessage(message) {
+		return JSON.stringify(message);
+	}
+	#decodeMessage(message) {
+		return JSON.parse(message);
+	}
+};
+
+//#endregion
+exports.SqliteMessageQueue = SqliteMessageQueue;