@fedify/mysql 2.1.0-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +48 -0
- package/dist/_virtual/rolldown_runtime.cjs +33 -0
- package/dist/kv.cjs +235 -0
- package/dist/kv.d.cts +103 -0
- package/dist/kv.d.ts +104 -0
- package/dist/kv.js +234 -0
- package/dist/mod.cjs +8 -0
- package/dist/mod.d.cts +3 -0
- package/dist/mod.d.ts +4 -0
- package/dist/mod.js +7 -0
- package/dist/mq.cjs +395 -0
- package/dist/mq.d.cts +126 -0
- package/dist/mq.d.ts +127 -0
- package/dist/mq.js +394 -0
- package/package.json +95 -0
package/dist/kv.js
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
|
|
2
|
+
import { Temporal } from "@js-temporal/polyfill";
|
|
3
|
+
|
|
4
|
+
import { isEqual } from "es-toolkit";
|
|
5
|
+
import { getLogger } from "@logtape/logtape";
|
|
6
|
+
|
|
7
|
+
//#region src/kv.ts
|
|
8
|
+
const logger = getLogger([
|
|
9
|
+
"fedify",
|
|
10
|
+
"mysql",
|
|
11
|
+
"kv"
|
|
12
|
+
]);
|
|
13
|
+
/**
|
|
14
|
+
* A key-value store that uses MySQL (or MariaDB) as the underlying storage.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```ts
|
|
18
|
+
* import { createFederation } from "@fedify/fedify";
|
|
19
|
+
* import { MysqlKvStore } from "@fedify/mysql";
|
|
20
|
+
* import mysql from "mysql2/promise";
|
|
21
|
+
*
|
|
22
|
+
* const pool = mysql.createPool("mysql://user:pass@localhost/db");
|
|
23
|
+
*
|
|
24
|
+
* const federation = createFederation({
|
|
25
|
+
* // ...
|
|
26
|
+
* kv: new MysqlKvStore(pool),
|
|
27
|
+
* });
|
|
28
|
+
* ```
|
|
29
|
+
*
|
|
30
|
+
* @since 2.1.0
|
|
31
|
+
*/
|
|
32
|
+
var MysqlKvStore = class {
|
|
33
|
+
#pool;
|
|
34
|
+
#tableName;
|
|
35
|
+
#expireCleanupRate;
|
|
36
|
+
#initialized;
|
|
37
|
+
/**
|
|
38
|
+
* Creates a new MySQL key-value store.
|
|
39
|
+
* @param pool The MySQL connection pool to use.
|
|
40
|
+
* @param options The options for the key-value store.
|
|
41
|
+
* @since 2.1.0
|
|
42
|
+
*/
|
|
43
|
+
constructor(pool, options = {}) {
|
|
44
|
+
this.#pool = pool;
|
|
45
|
+
const tableName = options.tableName ?? "fedify_kv";
|
|
46
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must start with a letter or underscore and contain only letters, digits, and underscores.`);
|
|
47
|
+
if (tableName.length > 50) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must be at most 50 characters long (MySQL identifier limit is 64 chars; the derived index "idx_<name>_expires" uses 12 more).`);
|
|
48
|
+
this.#tableName = tableName;
|
|
49
|
+
const expireCleanupRate = options.expireCleanupRate ?? 1;
|
|
50
|
+
if (expireCleanupRate < 0 || expireCleanupRate > 1) throw new RangeError(`Invalid expireCleanupRate: ${expireCleanupRate}. Must be a number between 0 and 1 inclusive.`);
|
|
51
|
+
this.#expireCleanupRate = expireCleanupRate;
|
|
52
|
+
this.#initialized = options.initialized ?? false;
|
|
53
|
+
}
|
|
54
|
+
async #expire() {
|
|
55
|
+
if (this.#expireCleanupRate <= 0) return;
|
|
56
|
+
if (this.#expireCleanupRate < 1 && Math.random() >= this.#expireCleanupRate) return;
|
|
57
|
+
await this.#pool.query(`DELETE FROM \`${this.#tableName}\`
|
|
58
|
+
WHERE \`expires\` IS NOT NULL AND \`expires\` < NOW(6)`);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* {@inheritDoc KvStore.get}
|
|
62
|
+
* @since 2.1.0
|
|
63
|
+
*/
|
|
64
|
+
async get(key) {
|
|
65
|
+
await this.initialize();
|
|
66
|
+
const serializedKey = JSON.stringify([...key]);
|
|
67
|
+
const [rows] = await this.#pool.query(`SELECT \`value\` FROM \`${this.#tableName}\`
|
|
68
|
+
WHERE \`key\` = ?
|
|
69
|
+
AND (\`expires\` IS NULL OR \`expires\` > NOW(6))`, [serializedKey]);
|
|
70
|
+
if (rows.length < 1) return void 0;
|
|
71
|
+
return rows[0].value;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* {@inheritDoc KvStore.set}
|
|
75
|
+
* @since 2.1.0
|
|
76
|
+
*/
|
|
77
|
+
async set(key, value, options) {
|
|
78
|
+
if (value === void 0) return;
|
|
79
|
+
await this.initialize();
|
|
80
|
+
const serializedKey = JSON.stringify([...key]);
|
|
81
|
+
const jsonValue = JSON.stringify(value);
|
|
82
|
+
if (options?.ttl != null) {
|
|
83
|
+
const ttlSeconds = durationToSeconds(options.ttl);
|
|
84
|
+
await this.#pool.query(`INSERT INTO \`${this.#tableName}\` (\`key\`, \`value\`, \`expires\`)
|
|
85
|
+
VALUES (?, CAST(? AS JSON),
|
|
86
|
+
DATE_ADD(NOW(6), INTERVAL ? SECOND))
|
|
87
|
+
ON DUPLICATE KEY UPDATE
|
|
88
|
+
\`value\` = VALUES(\`value\`),
|
|
89
|
+
\`expires\` = VALUES(\`expires\`)`, [
|
|
90
|
+
serializedKey,
|
|
91
|
+
jsonValue,
|
|
92
|
+
ttlSeconds
|
|
93
|
+
]);
|
|
94
|
+
} else await this.#pool.query(`INSERT INTO \`${this.#tableName}\` (\`key\`, \`value\`, \`expires\`)
|
|
95
|
+
VALUES (?, CAST(? AS JSON), NULL)
|
|
96
|
+
ON DUPLICATE KEY UPDATE
|
|
97
|
+
\`value\` = VALUES(\`value\`),
|
|
98
|
+
\`expires\` = NULL`, [serializedKey, jsonValue]);
|
|
99
|
+
await this.#expire();
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* {@inheritDoc KvStore.delete}
|
|
103
|
+
* @since 2.1.0
|
|
104
|
+
*/
|
|
105
|
+
async delete(key) {
|
|
106
|
+
await this.initialize();
|
|
107
|
+
const serializedKey = JSON.stringify([...key]);
|
|
108
|
+
await this.#pool.query(`DELETE FROM \`${this.#tableName}\` WHERE \`key\` = ?`, [serializedKey]);
|
|
109
|
+
await this.#expire();
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* {@inheritDoc KvStore.cas}
|
|
113
|
+
* @since 2.1.0
|
|
114
|
+
*/
|
|
115
|
+
async cas(key, expectedValue, newValue, options) {
|
|
116
|
+
await this.initialize();
|
|
117
|
+
const serializedKey = JSON.stringify([...key]);
|
|
118
|
+
let conn;
|
|
119
|
+
try {
|
|
120
|
+
conn = await this.#pool.getConnection();
|
|
121
|
+
await conn.beginTransaction();
|
|
122
|
+
const [rows] = await conn.query(`SELECT
|
|
123
|
+
\`value\`,
|
|
124
|
+
(\`expires\` IS NOT NULL AND \`expires\` <= NOW(6)) AS \`is_expired\`
|
|
125
|
+
FROM \`${this.#tableName}\`
|
|
126
|
+
WHERE \`key\` = ?
|
|
127
|
+
FOR UPDATE`, [serializedKey]);
|
|
128
|
+
const row = rows[0];
|
|
129
|
+
const currentValue = !row || row.is_expired ? void 0 : row.value;
|
|
130
|
+
if (!isEqual(currentValue, expectedValue)) {
|
|
131
|
+
await conn.rollback();
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
if (newValue === void 0) await conn.query(`DELETE FROM \`${this.#tableName}\` WHERE \`key\` = ?`, [serializedKey]);
|
|
135
|
+
else {
|
|
136
|
+
const jsonValue = JSON.stringify(newValue);
|
|
137
|
+
if (options?.ttl != null) {
|
|
138
|
+
const ttlSeconds = durationToSeconds(options.ttl);
|
|
139
|
+
await conn.query(`INSERT INTO \`${this.#tableName}\`
|
|
140
|
+
(\`key\`, \`value\`, \`expires\`)
|
|
141
|
+
VALUES (?, CAST(? AS JSON),
|
|
142
|
+
DATE_ADD(NOW(6), INTERVAL ? SECOND))
|
|
143
|
+
ON DUPLICATE KEY UPDATE
|
|
144
|
+
\`value\` = VALUES(\`value\`),
|
|
145
|
+
\`expires\` = VALUES(\`expires\`)`, [
|
|
146
|
+
serializedKey,
|
|
147
|
+
jsonValue,
|
|
148
|
+
ttlSeconds
|
|
149
|
+
]);
|
|
150
|
+
} else await conn.query(`INSERT INTO \`${this.#tableName}\`
|
|
151
|
+
(\`key\`, \`value\`, \`expires\`)
|
|
152
|
+
VALUES (?, CAST(? AS JSON), NULL)
|
|
153
|
+
ON DUPLICATE KEY UPDATE
|
|
154
|
+
\`value\` = VALUES(\`value\`),
|
|
155
|
+
\`expires\` = NULL`, [serializedKey, jsonValue]);
|
|
156
|
+
}
|
|
157
|
+
await conn.commit();
|
|
158
|
+
await this.#expire();
|
|
159
|
+
return true;
|
|
160
|
+
} catch (e) {
|
|
161
|
+
if (conn) await conn.rollback();
|
|
162
|
+
throw e;
|
|
163
|
+
} finally {
|
|
164
|
+
if (conn) conn.release();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* {@inheritDoc KvStore.list}
|
|
169
|
+
* @since 2.1.0
|
|
170
|
+
*/
|
|
171
|
+
async *list(prefix) {
|
|
172
|
+
await this.initialize();
|
|
173
|
+
let rows;
|
|
174
|
+
if (prefix == null || prefix.length === 0) [rows] = await this.#pool.query(`SELECT \`key\`, \`value\` FROM \`${this.#tableName}\`
|
|
175
|
+
WHERE \`expires\` IS NULL OR \`expires\` > NOW(6)
|
|
176
|
+
ORDER BY \`key\``);
|
|
177
|
+
else {
|
|
178
|
+
const serializedPrefix = JSON.stringify([...prefix]);
|
|
179
|
+
const likePrefix = serializedPrefix.slice(0, -1).replace(/[%_\\]/g, "\\$&") + ",%";
|
|
180
|
+
[rows] = await this.#pool.query(`SELECT \`key\`, \`value\` FROM \`${this.#tableName}\`
|
|
181
|
+
WHERE (\`key\` = ? OR \`key\` LIKE ? ESCAPE '\\\\')
|
|
182
|
+
AND (\`expires\` IS NULL OR \`expires\` > NOW(6))
|
|
183
|
+
ORDER BY \`key\``, [serializedPrefix, likePrefix]);
|
|
184
|
+
}
|
|
185
|
+
for (const row of rows) yield {
|
|
186
|
+
key: JSON.parse(row.key),
|
|
187
|
+
value: row.value
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Creates the table used by the key-value store if it does not already exist.
|
|
192
|
+
* Does nothing if the table already exists.
|
|
193
|
+
*
|
|
194
|
+
* @since 2.1.0
|
|
195
|
+
*/
|
|
196
|
+
async initialize() {
|
|
197
|
+
if (this.#initialized) return;
|
|
198
|
+
logger.debug("Initializing the key-value store table {tableName}...", { tableName: this.#tableName });
|
|
199
|
+
await this.#pool.query(`CREATE TABLE IF NOT EXISTS \`${this.#tableName}\` (
|
|
200
|
+
\`key\` VARCHAR(768) NOT NULL,
|
|
201
|
+
\`value\` JSON NOT NULL,
|
|
202
|
+
\`expires\` DATETIME(6) NULL DEFAULT NULL,
|
|
203
|
+
PRIMARY KEY (\`key\`)
|
|
204
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`);
|
|
205
|
+
try {
|
|
206
|
+
await this.#pool.query(`CREATE INDEX \`idx_${this.#tableName}_expires\`
|
|
207
|
+
ON \`${this.#tableName}\` (\`expires\`)`);
|
|
208
|
+
} catch (e) {
|
|
209
|
+
if (e.code !== "ER_DUP_KEYNAME") throw e;
|
|
210
|
+
}
|
|
211
|
+
this.#initialized = true;
|
|
212
|
+
logger.debug("Initialized the key-value store table {tableName}.", { tableName: this.#tableName });
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Drops the table used by the key-value store. Does nothing if the table
|
|
216
|
+
* does not exist. Resets the initialized flag so that
|
|
217
|
+
* {@link MysqlKvStore.initialize} can recreate the table on the next call.
|
|
218
|
+
*
|
|
219
|
+
* @since 2.1.0
|
|
220
|
+
*/
|
|
221
|
+
async drop() {
|
|
222
|
+
await this.#pool.query(`DROP TABLE IF EXISTS \`${this.#tableName}\``);
|
|
223
|
+
this.#initialized = false;
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
function durationToSeconds(duration) {
|
|
227
|
+
return duration.total({
|
|
228
|
+
unit: "second",
|
|
229
|
+
relativeTo: Temporal.Now.plainDateTimeISO()
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
//#endregion
|
|
234
|
+
export { MysqlKvStore };
|
package/dist/mod.cjs
ADDED
package/dist/mod.d.cts
ADDED
package/dist/mod.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { Temporal } from "@js-temporal/polyfill";
|
|
2
|
+
import { MysqlKvStore, MysqlKvStoreOptions } from "./kv.js";
|
|
3
|
+
import { MysqlMessageQueue, MysqlMessageQueueOptions } from "./mq.js";
|
|
4
|
+
export { MysqlKvStore, MysqlKvStoreOptions, MysqlMessageQueue, MysqlMessageQueueOptions };
|
package/dist/mod.js
ADDED
package/dist/mq.cjs
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
|
|
2
|
+
const { Temporal } = require("@js-temporal/polyfill");
|
|
3
|
+
|
|
4
|
+
const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
|
|
5
|
+
const __logtape_logtape = require_rolldown_runtime.__toESM(require("@logtape/logtape"));
|
|
6
|
+
|
|
7
|
+
//#region src/mq.ts
|
|
8
|
+
const logger = (0, __logtape_logtape.getLogger)([
|
|
9
|
+
"fedify",
|
|
10
|
+
"mysql",
|
|
11
|
+
"mq"
|
|
12
|
+
]);
|
|
13
|
+
const INITIALIZE_MAX_ATTEMPTS = 5;
|
|
14
|
+
const INITIALIZE_BACKOFF_MS = 10;
|
|
15
|
+
const ORDERING_KEY_CANDIDATE_LIMIT = 10;
|
|
16
|
+
function sleep(ms) {
|
|
17
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
18
|
+
}
|
|
19
|
+
function withTimeout(result, timeoutMs) {
|
|
20
|
+
const resolved = Promise.resolve(result);
|
|
21
|
+
if (timeoutMs <= 0) return resolved;
|
|
22
|
+
let timer;
|
|
23
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
24
|
+
timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`Message handler timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
25
|
+
});
|
|
26
|
+
return Promise.race([resolved, timeoutPromise]).finally(() => clearTimeout(timer));
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Computes a MySQL advisory lock name for the given table name and ordering
|
|
30
|
+
* key. The result is always at most 64 characters, which is well within
|
|
31
|
+
* MySQL's advisory lock name length limit.
|
|
32
|
+
*/
|
|
33
|
+
function getMysqlLockName(tableName, orderingKey) {
|
|
34
|
+
const raw = `${tableName}:${orderingKey}`;
|
|
35
|
+
if (raw.length <= 64) return raw;
|
|
36
|
+
let h1 = 0;
|
|
37
|
+
let h2 = 5381;
|
|
38
|
+
for (let i = 0; i < raw.length; i++) {
|
|
39
|
+
const c = raw.charCodeAt(i);
|
|
40
|
+
h1 = (h1 << 5) - h1 + c | 0;
|
|
41
|
+
h2 = (h2 << 5) + h2 + c | 0;
|
|
42
|
+
}
|
|
43
|
+
return `fdy:${(h1 >>> 0).toString(16).padStart(8, "0")}${(h2 >>> 0).toString(16).padStart(8, "0")}`;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* A message queue that uses MySQL or MariaDB as the underlying storage.
|
|
47
|
+
* Messages are delivered via periodic polling, since MySQL and MariaDB do not
|
|
48
|
+
* provide a `LISTEN`/`NOTIFY` equivalent.
|
|
49
|
+
*
|
|
50
|
+
* Concurrent workers are supported via `SELECT … FOR UPDATE SKIP LOCKED`
|
|
51
|
+
* (requires MySQL 8.0+ or MariaDB 10.6+) and MySQL advisory locks
|
|
52
|
+
* (`GET_LOCK`/`RELEASE_LOCK`) for ordering-key serialization.
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```ts
|
|
56
|
+
* import { createFederation } from "@fedify/fedify";
|
|
57
|
+
* import { MysqlKvStore, MysqlMessageQueue } from "@fedify/mysql";
|
|
58
|
+
* import mysql from "mysql2/promise";
|
|
59
|
+
*
|
|
60
|
+
* const pool = mysql.createPool("mysql://user:pass@localhost/db");
|
|
61
|
+
*
|
|
62
|
+
* const federation = createFederation({
|
|
63
|
+
* kv: new MysqlKvStore(pool),
|
|
64
|
+
* queue: new MysqlMessageQueue(pool),
|
|
65
|
+
* });
|
|
66
|
+
* ```
|
|
67
|
+
*
|
|
68
|
+
* @since 2.1.0
|
|
69
|
+
*/
|
|
70
|
+
var MysqlMessageQueue = class {
|
|
71
|
+
/**
|
|
72
|
+
* MySQL/MariaDB does not provide native retry mechanisms; Fedify handles
|
|
73
|
+
* retries itself.
|
|
74
|
+
* @since 2.1.0
|
|
75
|
+
*/
|
|
76
|
+
nativeRetrial = false;
|
|
77
|
+
#pool;
|
|
78
|
+
#tableName;
|
|
79
|
+
#pollIntervalMs;
|
|
80
|
+
#handlerTimeoutMs;
|
|
81
|
+
#initialized;
|
|
82
|
+
#initPromise;
|
|
83
|
+
/**
|
|
84
|
+
* Creates a new MySQL message queue.
|
|
85
|
+
* @param pool The MySQL connection pool to use.
|
|
86
|
+
* @param options Options for the message queue.
|
|
87
|
+
* @since 2.1.0
|
|
88
|
+
*/
|
|
89
|
+
constructor(pool, options = {}) {
|
|
90
|
+
this.#pool = pool;
|
|
91
|
+
const tableName = options.tableName ?? "fedify_mq";
|
|
92
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must start with a letter or underscore and contain only letters, digits, and underscores.`);
|
|
93
|
+
if (tableName.length > 46) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must be at most 46 characters long (MySQL identifier limit is 64 chars; the derived index "idx_<name>_deliver_after" uses 18 more).`);
|
|
94
|
+
this.#tableName = tableName;
|
|
95
|
+
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 1 }).total("millisecond");
|
|
96
|
+
this.#handlerTimeoutMs = Temporal.Duration.from(options.handlerTimeout ?? { seconds: 60 }).total("millisecond");
|
|
97
|
+
this.#initialized = options.initialized ?? false;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* {@inheritDoc MessageQueue.enqueue}
|
|
101
|
+
* @since 2.1.0
|
|
102
|
+
*/
|
|
103
|
+
async enqueue(message, options) {
|
|
104
|
+
await this.initialize();
|
|
105
|
+
const delayMs = options?.delay == null ? 0 : Math.max(Math.round(options.delay.total("millisecond")), 0);
|
|
106
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
107
|
+
if (options?.delay) logger.debug("Enqueuing a message with a delay of {delayMs}ms...", {
|
|
108
|
+
delayMs,
|
|
109
|
+
message,
|
|
110
|
+
orderingKey
|
|
111
|
+
});
|
|
112
|
+
else logger.debug("Enqueuing a message...", {
|
|
113
|
+
message,
|
|
114
|
+
orderingKey
|
|
115
|
+
});
|
|
116
|
+
await this.#pool.query(`INSERT INTO \`${this.#tableName}\`
|
|
117
|
+
(\`id\`, \`message\`, \`deliver_after\`, \`ordering_key\`)
|
|
118
|
+
VALUES (
|
|
119
|
+
UUID(),
|
|
120
|
+
?,
|
|
121
|
+
DATE_ADD(NOW(6), INTERVAL ? MICROSECOND),
|
|
122
|
+
?
|
|
123
|
+
)`, [
|
|
124
|
+
JSON.stringify(message),
|
|
125
|
+
delayMs * 1e3,
|
|
126
|
+
orderingKey
|
|
127
|
+
]);
|
|
128
|
+
logger.debug("Enqueued a message.", {
|
|
129
|
+
message,
|
|
130
|
+
orderingKey
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* {@inheritDoc MessageQueue.enqueueMany}
|
|
135
|
+
* @since 2.1.0
|
|
136
|
+
*/
|
|
137
|
+
async enqueueMany(messages, options) {
|
|
138
|
+
if (messages.length === 0) return;
|
|
139
|
+
await this.initialize();
|
|
140
|
+
const delayMs = options?.delay == null ? 0 : Math.max(Math.round(options.delay.total("millisecond")), 0);
|
|
141
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
142
|
+
if (options?.delay) logger.debug("Enqueuing {count} messages with a delay of {delayMs}ms...", {
|
|
143
|
+
count: messages.length,
|
|
144
|
+
delayMs,
|
|
145
|
+
orderingKey
|
|
146
|
+
});
|
|
147
|
+
else logger.debug("Enqueuing {count} messages...", {
|
|
148
|
+
count: messages.length,
|
|
149
|
+
orderingKey
|
|
150
|
+
});
|
|
151
|
+
const placeholders = messages.map(() => "(UUID(), ?, DATE_ADD(NOW(6), INTERVAL ? MICROSECOND), ?)").join(", ");
|
|
152
|
+
const values = messages.flatMap((message, index) => [
|
|
153
|
+
JSON.stringify(message),
|
|
154
|
+
delayMs * 1e3 + index,
|
|
155
|
+
orderingKey
|
|
156
|
+
]);
|
|
157
|
+
let conn;
|
|
158
|
+
try {
|
|
159
|
+
conn = await this.#pool.getConnection();
|
|
160
|
+
await conn.beginTransaction();
|
|
161
|
+
await conn.query(`INSERT INTO \`${this.#tableName}\`
|
|
162
|
+
(\`id\`, \`message\`, \`deliver_after\`, \`ordering_key\`)
|
|
163
|
+
VALUES ${placeholders}`, values);
|
|
164
|
+
await conn.commit();
|
|
165
|
+
} catch (e) {
|
|
166
|
+
if (conn != null) await conn.rollback();
|
|
167
|
+
throw e;
|
|
168
|
+
} finally {
|
|
169
|
+
conn?.release();
|
|
170
|
+
}
|
|
171
|
+
logger.debug("Enqueued {count} messages.", {
|
|
172
|
+
count: messages.length,
|
|
173
|
+
orderingKey
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* {@inheritDoc MessageQueue.listen}
|
|
178
|
+
* @since 2.1.0
|
|
179
|
+
*/
|
|
180
|
+
async listen(handler, options = {}) {
|
|
181
|
+
await this.initialize();
|
|
182
|
+
const { signal } = options;
|
|
183
|
+
const poll = async () => {
|
|
184
|
+
while (!signal?.aborted) {
|
|
185
|
+
let processed = false;
|
|
186
|
+
const noKeyMsg = await this.#dequeueWithoutOrderingKey();
|
|
187
|
+
if (noKeyMsg !== void 0) {
|
|
188
|
+
if (signal?.aborted) return;
|
|
189
|
+
await withTimeout(handler(noKeyMsg), this.#handlerTimeoutMs);
|
|
190
|
+
processed = true;
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
const candidates = await this.#findOrderingKeyCandidates();
|
|
194
|
+
for (const orderingKey of candidates) {
|
|
195
|
+
if (signal?.aborted) break;
|
|
196
|
+
const lockName = getMysqlLockName(this.#tableName, orderingKey);
|
|
197
|
+
let conn;
|
|
198
|
+
try {
|
|
199
|
+
conn = await this.#pool.getConnection();
|
|
200
|
+
const [lockResult] = await conn.query(`SELECT GET_LOCK(?, 0) AS acquired`, [lockName]);
|
|
201
|
+
if (lockResult[0].acquired === null) logger.warn("GET_LOCK({lockName}) returned NULL (server error); skipping ordering key {orderingKey}.", {
|
|
202
|
+
lockName,
|
|
203
|
+
orderingKey
|
|
204
|
+
});
|
|
205
|
+
else if (lockResult[0].acquired === 1) {
|
|
206
|
+
try {
|
|
207
|
+
const msg = await this.#dequeueOrderedMessage(conn, orderingKey);
|
|
208
|
+
if (msg !== void 0) {
|
|
209
|
+
if (signal?.aborted) return;
|
|
210
|
+
await withTimeout(handler(msg), this.#handlerTimeoutMs);
|
|
211
|
+
processed = true;
|
|
212
|
+
}
|
|
213
|
+
} finally {
|
|
214
|
+
await conn.query(`SELECT RELEASE_LOCK(?)`, [lockName]);
|
|
215
|
+
}
|
|
216
|
+
if (processed) break;
|
|
217
|
+
}
|
|
218
|
+
} finally {
|
|
219
|
+
conn?.release();
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (!processed) break;
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
const safePoll = async (trigger) => {
|
|
226
|
+
try {
|
|
227
|
+
await poll();
|
|
228
|
+
} catch (error) {
|
|
229
|
+
logger.error("Error while polling for messages ({trigger}); will retry on next poll: {error}", {
|
|
230
|
+
trigger,
|
|
231
|
+
error
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
await safePoll("initial");
|
|
236
|
+
while (!signal?.aborted) {
|
|
237
|
+
await new Promise((resolve) => {
|
|
238
|
+
const timeoutId = setTimeout(() => {
|
|
239
|
+
signal?.removeEventListener("abort", onAbort);
|
|
240
|
+
resolve(0);
|
|
241
|
+
}, this.#pollIntervalMs);
|
|
242
|
+
function onAbort() {
|
|
243
|
+
clearTimeout(timeoutId);
|
|
244
|
+
resolve(void 0);
|
|
245
|
+
}
|
|
246
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
247
|
+
});
|
|
248
|
+
if (signal?.aborted) break;
|
|
249
|
+
await safePoll("interval");
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Atomically dequeues the oldest ready message that has no ordering key,
|
|
254
|
+
* using `FOR UPDATE SKIP LOCKED` within a transaction.
|
|
255
|
+
* Returns `undefined` when no such message is available.
|
|
256
|
+
*/
|
|
257
|
+
async #dequeueWithoutOrderingKey() {
|
|
258
|
+
let conn;
|
|
259
|
+
try {
|
|
260
|
+
conn = await this.#pool.getConnection();
|
|
261
|
+
await conn.beginTransaction();
|
|
262
|
+
const [rows] = await conn.query(`SELECT \`id\`, \`message\`
|
|
263
|
+
FROM \`${this.#tableName}\`
|
|
264
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` IS NULL
|
|
265
|
+
ORDER BY \`deliver_after\`
|
|
266
|
+
LIMIT 1
|
|
267
|
+
FOR UPDATE SKIP LOCKED`);
|
|
268
|
+
if (rows.length === 0) {
|
|
269
|
+
await conn.rollback();
|
|
270
|
+
return void 0;
|
|
271
|
+
}
|
|
272
|
+
const { id, message } = rows[0];
|
|
273
|
+
await conn.query(`DELETE FROM \`${this.#tableName}\` WHERE \`id\` = ?`, [id]);
|
|
274
|
+
await conn.commit();
|
|
275
|
+
return message;
|
|
276
|
+
} catch (e) {
|
|
277
|
+
if (conn != null) await conn.rollback();
|
|
278
|
+
throw e;
|
|
279
|
+
} finally {
|
|
280
|
+
conn?.release();
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Returns up to {@link ORDERING_KEY_CANDIDATE_LIMIT} distinct ordering keys
|
|
285
|
+
* that have at least one ready message, ordered by their earliest
|
|
286
|
+
* `deliver_after`. Fetching a batch in one query avoids the growing
|
|
287
|
+
* `NOT IN (…)` list that results from repeatedly calling a single-result
|
|
288
|
+
* version after each failed lock attempt.
|
|
289
|
+
*/
|
|
290
|
+
async #findOrderingKeyCandidates() {
|
|
291
|
+
const [rows] = await this.#pool.query(`SELECT \`ordering_key\`
|
|
292
|
+
FROM \`${this.#tableName}\`
|
|
293
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` IS NOT NULL
|
|
294
|
+
GROUP BY \`ordering_key\`
|
|
295
|
+
ORDER BY MIN(\`deliver_after\`)
|
|
296
|
+
LIMIT ?`, [ORDERING_KEY_CANDIDATE_LIMIT]);
|
|
297
|
+
return rows.map((r) => r.ordering_key);
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Dequeues the oldest ready message for the given ordering key using
|
|
301
|
+
* the supplied (dedicated) connection. The caller MUST hold the advisory
|
|
302
|
+
* lock for `orderingKey` before calling this method.
|
|
303
|
+
* Returns `undefined` when no ready message exists for the ordering key.
|
|
304
|
+
*/
|
|
305
|
+
async #dequeueOrderedMessage(conn, orderingKey) {
|
|
306
|
+
await conn.beginTransaction();
|
|
307
|
+
try {
|
|
308
|
+
const [rows] = await conn.query(`SELECT \`id\`, \`message\`
|
|
309
|
+
FROM \`${this.#tableName}\`
|
|
310
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` = ?
|
|
311
|
+
ORDER BY \`deliver_after\`
|
|
312
|
+
LIMIT 1`, [orderingKey]);
|
|
313
|
+
if (rows.length === 0) {
|
|
314
|
+
await conn.rollback();
|
|
315
|
+
return void 0;
|
|
316
|
+
}
|
|
317
|
+
const { id, message } = rows[0];
|
|
318
|
+
await conn.query(`DELETE FROM \`${this.#tableName}\` WHERE \`id\` = ?`, [id]);
|
|
319
|
+
await conn.commit();
|
|
320
|
+
return message;
|
|
321
|
+
} catch (e) {
|
|
322
|
+
await conn.rollback();
|
|
323
|
+
throw e;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Initializes the message queue table if it does not already exist.
|
|
328
|
+
* Concurrent calls are coalesced — only one initialization runs at a time.
|
|
329
|
+
*
|
|
330
|
+
* @since 2.1.0
|
|
331
|
+
*/
|
|
332
|
+
initialize() {
|
|
333
|
+
if (this.#initialized) return Promise.resolve();
|
|
334
|
+
return this.#initPromise ??= this.#doInitialize();
|
|
335
|
+
}
|
|
336
|
+
async #doInitialize() {
|
|
337
|
+
logger.debug("Initializing the message queue table {tableName}...", { tableName: this.#tableName });
|
|
338
|
+
for (let attempt = 1; attempt <= INITIALIZE_MAX_ATTEMPTS; attempt++) try {
|
|
339
|
+
await this.#pool.query(`CREATE TABLE IF NOT EXISTS \`${this.#tableName}\` (
|
|
340
|
+
\`id\` CHAR(36) NOT NULL,
|
|
341
|
+
\`message\` JSON NOT NULL,
|
|
342
|
+
\`deliver_after\` DATETIME(6) NOT NULL DEFAULT NOW(6),
|
|
343
|
+
\`ordering_key\` TEXT NULL DEFAULT NULL,
|
|
344
|
+
PRIMARY KEY (\`id\`)
|
|
345
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`);
|
|
346
|
+
try {
|
|
347
|
+
await this.#pool.query(`CREATE INDEX \`idx_${this.#tableName}_deliver_after\`
|
|
348
|
+
ON \`${this.#tableName}\` (\`deliver_after\`)`);
|
|
349
|
+
} catch (e) {
|
|
350
|
+
if (e.code !== "ER_DUP_KEYNAME") throw e;
|
|
351
|
+
}
|
|
352
|
+
try {
|
|
353
|
+
await this.#pool.query(`CREATE INDEX \`idx_${this.#tableName}_ok_da\`
|
|
354
|
+
ON \`${this.#tableName}\` (\`ordering_key\`(766), \`deliver_after\`)`);
|
|
355
|
+
} catch (e) {
|
|
356
|
+
if (e.code !== "ER_DUP_KEYNAME") throw e;
|
|
357
|
+
}
|
|
358
|
+
break;
|
|
359
|
+
} catch (error) {
|
|
360
|
+
if (attempt >= INITIALIZE_MAX_ATTEMPTS) {
|
|
361
|
+
logger.error("Failed to initialize the message queue table: {error}", { error });
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
364
|
+
const backoffMs = INITIALIZE_BACKOFF_MS * 2 ** (attempt - 1);
|
|
365
|
+
logger.debug("Initialization race for table {tableName}; retrying in {backoffMs}ms (attempt {attempt}/{maxAttempts}).", {
|
|
366
|
+
tableName: this.#tableName,
|
|
367
|
+
backoffMs,
|
|
368
|
+
attempt,
|
|
369
|
+
maxAttempts: INITIALIZE_MAX_ATTEMPTS,
|
|
370
|
+
error
|
|
371
|
+
});
|
|
372
|
+
await sleep(backoffMs);
|
|
373
|
+
}
|
|
374
|
+
this.#initialized = true;
|
|
375
|
+
logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Drops the message queue table if it exists. Resets the initialized flag
|
|
379
|
+
* so that {@link MysqlMessageQueue.initialize} can recreate the table on
|
|
380
|
+
* the next call.
|
|
381
|
+
*
|
|
382
|
+
* @since 2.1.0
|
|
383
|
+
*/
|
|
384
|
+
async drop() {
|
|
385
|
+
if (this.#initPromise != null) try {
|
|
386
|
+
await this.#initPromise;
|
|
387
|
+
} catch {}
|
|
388
|
+
await this.#pool.query(`DROP TABLE IF EXISTS \`${this.#tableName}\``);
|
|
389
|
+
this.#initialized = false;
|
|
390
|
+
this.#initPromise = void 0;
|
|
391
|
+
}
|
|
392
|
+
};
|
|
393
|
+
|
|
394
|
+
//#endregion
|
|
395
|
+
exports.MysqlMessageQueue = MysqlMessageQueue;
|