@fedify/sqlite 2.0.0-dev.237 → 2.0.0-dev.279

package/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@fedify/sqlite",
-  "version": "2.0.0-dev.237+7f2bb1de",
+  "version": "2.0.0-dev.279+ce1bdc22",
   "license": "MIT",
   "exports": {
     ".": "./src/mod.ts",
@@ -18,7 +18,8 @@
   "publish": {
     "exclude": [
       "**/*.test.ts",
-      "!dist/"
+      "!dist/",
+      "tsdown.config.ts"
     ]
   },
   "tasks": {

package/dist/mq.cjs

@@ -106,10 +106,14 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 			message
 		});
 		else logger.debug("Enqueuing a message...", { message });
+		const orderingKey = options?.orderingKey ?? null;
 		return this.#retryOnBusy(() => {
-			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-          VALUES (?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled);
-			logger.debug("Enqueued a message.", { message });
+			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+          VALUES (?, ?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled, orderingKey);
+			logger.debug("Enqueued a message.", {
+				message,
+				orderingKey
+			});
 			const delayMs = delay.total("millisecond");
 			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
 		});
@@ -128,13 +132,14 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 			messages
 		});
 		else logger.debug("Enqueuing messages...", { messages });
+		const orderingKey = options?.orderingKey ?? null;
 		return this.#withTransactionRetries(() => {
-			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-        VALUES (?, ?, ?, ?)`);
+			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+        VALUES (?, ?, ?, ?, ?)`);
 			for (const message of messages) {
 				const id = crypto.randomUUID();
 				const encodedMessage = this.#encodeMessage(message);
-				stmt.run(id, encodedMessage, now, scheduled);
+				stmt.run(id, encodedMessage, now, scheduled, orderingKey);
 			}
 			logger.debug("Enqueued messages.", { messages });
 			const delayMs = delay.total("millisecond");
@@ -157,6 +162,7 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 		logger.debug("Starting to listen for messages on table {tableName}...", { tableName: this.#tableName });
 		const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
 		const timeouts = /* @__PURE__ */ new Set();
+		const lockTableName = `${this.#tableName}_locks`;
 		const poll = async () => {
 			while (signal == null || !signal.aborted) {
 				const now = Temporal.Now.instant().epochMilliseconds;
@@ -165,16 +171,23 @@ var SqliteMessageQueue = class SqliteMessageQueue {
               WHERE id = (
                 SELECT id FROM "${this.#tableName}"
                 WHERE scheduled <= ?
+                  AND (ordering_key IS NULL
+                       OR ordering_key NOT IN (SELECT ordering_key FROM "${lockTableName}"))
                 ORDER BY scheduled
                 LIMIT 1
               )
-              RETURNING id, message`).get(now);
+              RETURNING id, message, ordering_key`).get(now);
 				});
 				if (result) {
 					const message = this.#decodeMessage(result.message);
+					const orderingKey = result.ordering_key;
+					if (orderingKey != null) await this.#retryOnBusy(() => {
+						this.#db.prepare(`INSERT OR IGNORE INTO "${lockTableName}" (ordering_key) VALUES (?)`).run(orderingKey);
+					});
 					logger.debug("Processing message {id}...", {
 						id: result.id,
-						message
+						message,
+						orderingKey
 					});
 					try {
 						await handler(message);
@@ -186,6 +199,10 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 							message,
 							error
 						});
+					} finally {
+						if (orderingKey != null) await this.#retryOnBusy(() => {
+							this.#db.prepare(`DELETE FROM "${lockTableName}" WHERE ordering_key = ?`).run(orderingKey);
+						});
 					}
 					continue;
 				}
@@ -240,23 +257,30 @@ var SqliteMessageQueue = class SqliteMessageQueue {
           id TEXT PRIMARY KEY,
           message TEXT NOT NULL,
           created INTEGER NOT NULL,
-          scheduled INTEGER NOT NULL
+          scheduled INTEGER NOT NULL,
+          ordering_key TEXT
         )
       `);
 			this.#db.exec(`
         CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
         ON "${this.#tableName}" (scheduled)
+      `);
+			this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}_locks" (
+          ordering_key TEXT PRIMARY KEY
+        )
       `);
 		});
 		this.#initialized = true;
 		logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
 	}
 	/**
-	* Drops the table used by the message queue.  Does nothing if the table
-	* does not exist.
+	* Drops the tables used by the message queue.  Does nothing if the tables
+	* do not exist.
 	*/
 	drop() {
 		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
+		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}_locks"`);
 		this.#initialized = false;
 	}
 	/**

package/dist/mq.d.cts

@@ -99,8 +99,8 @@ declare class SqliteMessageQueue implements MessageQueue, Disposable {
    */
   initialize(): void;
   /**
-   * Drops the table used by the message queue.  Does nothing if the table
-   * does not exist.
+   * Drops the tables used by the message queue.  Does nothing if the tables
+   * do not exist.
    */
   drop(): void;
   /**

package/dist/mq.d.ts

@@ -100,8 +100,8 @@ declare class SqliteMessageQueue implements MessageQueue, Disposable {
    */
   initialize(): void;
   /**
-   * Drops the table used by the message queue.  Does nothing if the table
-   * does not exist.
+   * Drops the tables used by the message queue.  Does nothing if the tables
+   * do not exist.
    */
   drop(): void;
   /**

package/dist/mq.js

@@ -105,10 +105,14 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 			message
 		});
 		else logger.debug("Enqueuing a message...", { message });
+		const orderingKey = options?.orderingKey ?? null;
 		return this.#retryOnBusy(() => {
-			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-          VALUES (?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled);
-			logger.debug("Enqueued a message.", { message });
+			this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+          VALUES (?, ?, ?, ?, ?)`).run(id, encodedMessage, now, scheduled, orderingKey);
+			logger.debug("Enqueued a message.", {
+				message,
+				orderingKey
+			});
 			const delayMs = delay.total("millisecond");
 			SqliteMessageQueue.#getNotifyChannel(this.#tableName).dispatchEvent(new EnqueueEvent(delayMs));
 		});
@@ -127,13 +131,14 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 			messages
 		});
 		else logger.debug("Enqueuing messages...", { messages });
+		const orderingKey = options?.orderingKey ?? null;
 		return this.#withTransactionRetries(() => {
-			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-        VALUES (?, ?, ?, ?)`);
+			const stmt = this.#db.prepare(`INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+        VALUES (?, ?, ?, ?, ?)`);
 			for (const message of messages) {
 				const id = crypto.randomUUID();
 				const encodedMessage = this.#encodeMessage(message);
-				stmt.run(id, encodedMessage, now, scheduled);
+				stmt.run(id, encodedMessage, now, scheduled, orderingKey);
 			}
 			logger.debug("Enqueued messages.", { messages });
 			const delayMs = delay.total("millisecond");
@@ -156,6 +161,7 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 		logger.debug("Starting to listen for messages on table {tableName}...", { tableName: this.#tableName });
 		const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
 		const timeouts = /* @__PURE__ */ new Set();
+		const lockTableName = `${this.#tableName}_locks`;
 		const poll = async () => {
 			while (signal == null || !signal.aborted) {
 				const now = Temporal.Now.instant().epochMilliseconds;
@@ -164,16 +170,23 @@ var SqliteMessageQueue = class SqliteMessageQueue {
               WHERE id = (
                 SELECT id FROM "${this.#tableName}"
                 WHERE scheduled <= ?
+                  AND (ordering_key IS NULL
+                       OR ordering_key NOT IN (SELECT ordering_key FROM "${lockTableName}"))
                 ORDER BY scheduled
                 LIMIT 1
               )
-              RETURNING id, message`).get(now);
+              RETURNING id, message, ordering_key`).get(now);
 				});
 				if (result) {
 					const message = this.#decodeMessage(result.message);
+					const orderingKey = result.ordering_key;
+					if (orderingKey != null) await this.#retryOnBusy(() => {
+						this.#db.prepare(`INSERT OR IGNORE INTO "${lockTableName}" (ordering_key) VALUES (?)`).run(orderingKey);
+					});
 					logger.debug("Processing message {id}...", {
 						id: result.id,
-						message
+						message,
+						orderingKey
 					});
 					try {
 						await handler(message);
@@ -185,6 +198,10 @@ var SqliteMessageQueue = class SqliteMessageQueue {
 							message,
 							error
 						});
+					} finally {
+						if (orderingKey != null) await this.#retryOnBusy(() => {
+							this.#db.prepare(`DELETE FROM "${lockTableName}" WHERE ordering_key = ?`).run(orderingKey);
+						});
 					}
 					continue;
 				}
@@ -239,23 +256,30 @@ var SqliteMessageQueue = class SqliteMessageQueue {
           id TEXT PRIMARY KEY,
           message TEXT NOT NULL,
           created INTEGER NOT NULL,
-          scheduled INTEGER NOT NULL
+          scheduled INTEGER NOT NULL,
+          ordering_key TEXT
         )
       `);
 			this.#db.exec(`
         CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
         ON "${this.#tableName}" (scheduled)
+      `);
+			this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}_locks" (
+          ordering_key TEXT PRIMARY KEY
+        )
       `);
 		});
 		this.#initialized = true;
 		logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
 	}
 	/**
-	* Drops the table used by the message queue.  Does nothing if the table
-	* does not exist.
+	* Drops the tables used by the message queue.  Does nothing if the tables
+	* do not exist.
 	*/
 	drop() {
 		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
+		this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}_locks"`);
 		this.#initialized = false;
 	}
 	/**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fedify/sqlite",
-  "version": "2.0.0-dev.237+7f2bb1de",
+  "version": "2.0.0-dev.279+ce1bdc22",
   "description": "SQLite drivers for Fedify",
   "keywords": [
     "fedify",
@@ -72,19 +72,22 @@
     "es-toolkit": "^1.31.0"
   },
   "peerDependencies": {
-    "@fedify/fedify": "^2.0.0-dev.237+7f2bb1de"
+    "@fedify/fedify": "^2.0.0-dev.279+ce1bdc22"
   },
   "devDependencies": {
     "@std/async": "npm:@jsr/std__async@^1.0.13",
     "tsdown": "^0.12.9",
     "typescript": "^5.9.3",
-    "@fedify/testing": "^2.0.0-dev.237+7f2bb1de"
+    "@fedify/testing": "^2.0.0-dev.279+ce1bdc22"
   },
   "scripts": {
-    "build": "tsdown",
-    "prepublish": "tsdown",
-    "test": "tsdown && node --experimental-transform-types --test",
-    "test:bun": "tsdown && bun test --timeout=10000",
+    "build:self": "tsdown",
+    "build": "pnpm --filter @fedify/sqlite... run build:self",
+    "prepublish": "pnpm build",
+    "pretest": "pnpm build",
+    "test": "node --experimental-transform-types --test",
+    "pretest:bun": "pnpm build",
+    "test:bun": "bun test --timeout=10000",
     "test:deno": "deno task test"
   }
 }

package/src/mq.test.ts

@@ -20,4 +20,5 @@ test("SqliteMessageQueue", () =>
       mq1[Symbol.dispose]();
       mq2[Symbol.dispose]();
     },
+    { testOrderingKey: true },
   ));

package/src/mq.ts

@@ -181,15 +181,17 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
       logger.debug("Enqueuing a message...", { message });
     }
 
+    const orderingKey = options?.orderingKey ?? null;
+
     return this.#retryOnBusy(() => {
       this.#db
         .prepare(
-          `INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-          VALUES (?, ?, ?, ?)`,
+          `INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+          VALUES (?, ?, ?, ?, ?)`,
         )
-        .run(id, encodedMessage, now, scheduled);
+        .run(id, encodedMessage, now, scheduled, orderingKey);
 
-      logger.debug("Enqueued a message.", { message });
+      logger.debug("Enqueued a message.", { message, orderingKey });
 
       // Notify listeners that a message has been enqueued
       const delayMs = delay.total("millisecond");
@@ -224,16 +226,18 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
       logger.debug("Enqueuing messages...", { messages });
     }
 
+    const orderingKey = options?.orderingKey ?? null;
+
     return this.#withTransactionRetries(() => {
       const stmt = this.#db.prepare(
-        `INSERT INTO "${this.#tableName}" (id, message, created, scheduled)
-        VALUES (?, ?, ?, ?)`,
+        `INSERT INTO "${this.#tableName}" (id, message, created, scheduled, ordering_key)
+        VALUES (?, ?, ?, ?, ?)`,
       );
 
       for (const message of messages) {
         const id = crypto.randomUUID();
         const encodedMessage = this.#encodeMessage(message);
-        stmt.run(id, encodedMessage, now, scheduled);
+        stmt.run(id, encodedMessage, now, scheduled, orderingKey);
       }
 
       logger.debug("Enqueued messages.", { messages });
@@ -274,6 +278,7 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
 
     const channel = SqliteMessageQueue.#getNotifyChannel(this.#tableName);
     const timeouts = new Set<ReturnType<typeof setTimeout>>();
+    const lockTableName = `${this.#tableName}_locks`;
 
     const poll = async () => {
       while (signal == null || !signal.aborted) {
@@ -283,6 +288,7 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
         // processed using DELETE ... RETURNING (SQLite >= 3.35.0)
         // Wrapped in BEGIN IMMEDIATE transaction to ensure proper locking
         // and prevent race conditions in multi-process scenarios
+        // Exclude messages with ordering keys currently being processed (DB-level lock)
         const result = await this.#withTransactionRetries(() => {
           return this.#db
             .prepare(
@@ -290,19 +296,37 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
               WHERE id = (
                 SELECT id FROM "${this.#tableName}"
                 WHERE scheduled <= ?
+                  AND (ordering_key IS NULL
+                       OR ordering_key NOT IN (SELECT ordering_key FROM "${lockTableName}"))
                 ORDER BY scheduled
                 LIMIT 1
               )
-              RETURNING id, message`,
+              RETURNING id, message, ordering_key`,
             )
-            .get(now) as { id: string; message: string } | undefined;
+            .get(now) as
+              | { id: string; message: string; ordering_key: string | null }
+              | undefined;
         });
 
         if (result) {
           const message = this.#decodeMessage(result.message);
+          const orderingKey = result.ordering_key;
+
+          // Acquire DB-level lock for this ordering key
+          if (orderingKey != null) {
+            await this.#retryOnBusy(() => {
+              this.#db
+                .prepare(
+                  `INSERT OR IGNORE INTO "${lockTableName}" (ordering_key) VALUES (?)`,
+                )
+                .run(orderingKey);
+            });
+          }
+
           logger.debug("Processing message {id}...", {
             id: result.id,
             message,
+            orderingKey,
           });
           try {
             await handler(message);
@@ -317,6 +341,17 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
                 error,
               },
             );
+          } finally {
+            // Release DB-level lock for this ordering key
+            if (orderingKey != null) {
+              await this.#retryOnBusy(() => {
+                this.#db
+                  .prepare(
+                    `DELETE FROM "${lockTableName}" WHERE ordering_key = ?`,
+                  )
+                  .run(orderingKey);
+              });
+            }
           }
 
           // Check for next message immediately
@@ -397,7 +432,8 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
           id TEXT PRIMARY KEY,
           message TEXT NOT NULL,
           created INTEGER NOT NULL,
-          scheduled INTEGER NOT NULL
+          scheduled INTEGER NOT NULL,
+          ordering_key TEXT
         )
       `);
 
@@ -405,6 +441,13 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
         CREATE INDEX IF NOT EXISTS "idx_${this.#tableName}_scheduled"
         ON "${this.#tableName}" (scheduled)
       `);
+
+      // Create lock table for distributed ordering key locks
+      this.#db.exec(`
+        CREATE TABLE IF NOT EXISTS "${this.#tableName}_locks" (
+          ordering_key TEXT PRIMARY KEY
+        )
+      `);
     });
 
     this.#initialized = true;
@@ -414,11 +457,12 @@ export class SqliteMessageQueue implements MessageQueue, Disposable {
   }
 
   /**
-   * Drops the table used by the message queue.  Does nothing if the table
-   * does not exist.
+   * Drops the tables used by the message queue.  Does nothing if the tables
+   * do not exist.
    */
   drop(): void {
     this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}"`);
+    this.#db.exec(`DROP TABLE IF EXISTS "${this.#tableName}_locks"`);
     this.#initialized = false;
   }