@fedify/mysql 2.1.0-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +48 -0
- package/dist/_virtual/rolldown_runtime.cjs +33 -0
- package/dist/kv.cjs +235 -0
- package/dist/kv.d.cts +103 -0
- package/dist/kv.d.ts +104 -0
- package/dist/kv.js +234 -0
- package/dist/mod.cjs +8 -0
- package/dist/mod.d.cts +3 -0
- package/dist/mod.d.ts +4 -0
- package/dist/mod.js +7 -0
- package/dist/mq.cjs +395 -0
- package/dist/mq.d.cts +126 -0
- package/dist/mq.d.ts +127 -0
- package/dist/mq.js +394 -0
- package/package.json +95 -0
package/dist/mq.d.cts
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { MessageQueue, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
|
|
2
|
+
import { Pool } from "mysql2/promise";
|
|
3
|
+
|
|
4
|
+
//#region src/mq.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* Options for the MySQL message queue.
|
|
7
|
+
*
|
|
8
|
+
* @since 2.1.0
|
|
9
|
+
*/
|
|
10
|
+
interface MysqlMessageQueueOptions {
|
|
11
|
+
/**
|
|
12
|
+
* The table name to use for the message queue.
|
|
13
|
+
* `"fedify_mq"` by default.
|
|
14
|
+
* @default `"fedify_mq"`
|
|
15
|
+
* @since 2.1.0
|
|
16
|
+
*/
|
|
17
|
+
readonly tableName?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Whether the table has been initialized. `false` by default.
|
|
20
|
+
* @default `false`
|
|
21
|
+
* @since 2.1.0
|
|
22
|
+
*/
|
|
23
|
+
readonly initialized?: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* The poll interval for the message queue. 1 second by default.
|
|
26
|
+
*
|
|
27
|
+
* Since MySQL/MariaDB has no `LISTEN`/`NOTIFY` equivalent, messages are
|
|
28
|
+
* discovered using periodic polling. A shorter interval reduces message
|
|
29
|
+
* delivery latency at the cost of additional database load.
|
|
30
|
+
*
|
|
31
|
+
* @default `{ seconds: 1 }`
|
|
32
|
+
* @since 2.1.0
|
|
33
|
+
*/
|
|
34
|
+
readonly pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
35
|
+
/**
|
|
36
|
+
* The maximum time to wait for a message handler to complete before
|
|
37
|
+
* the queue moves on. This is a *soft* timeout: when a handler exceeds
|
|
38
|
+
* the limit, the queue stops waiting and logs a timeout error, but it
|
|
39
|
+
* cannot cancel the handler itself — the handler's promise continues
|
|
40
|
+
* running in the background. In the ordered-key path the advisory lock
|
|
41
|
+
* is always released in a `finally` block, so a timed-out handler will
|
|
42
|
+
* not permanently block the same ordering key. However, concurrent
|
|
43
|
+
* side-effects from the still-running handler may interleave with the
|
|
44
|
+
* next handler for the same key; callers should be aware of this.
|
|
45
|
+
*
|
|
46
|
+
* Set to zero to disable the timeout (not recommended in production).
|
|
47
|
+
*
|
|
48
|
+
* 60 seconds by default.
|
|
49
|
+
* @default `{ seconds: 60 }`
|
|
50
|
+
* @since 2.1.0
|
|
51
|
+
*/
|
|
52
|
+
readonly handlerTimeout?: Temporal.Duration | Temporal.DurationLike;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* A message queue that uses MySQL or MariaDB as the underlying storage.
|
|
56
|
+
* Messages are delivered via periodic polling, since MySQL and MariaDB do not
|
|
57
|
+
* provide a `LISTEN`/`NOTIFY` equivalent.
|
|
58
|
+
*
|
|
59
|
+
* Concurrent workers are supported via `SELECT … FOR UPDATE SKIP LOCKED`
|
|
60
|
+
* (requires MySQL 8.0+ or MariaDB 10.6+) and MySQL advisory locks
|
|
61
|
+
* (`GET_LOCK`/`RELEASE_LOCK`) for ordering-key serialization.
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```ts
|
|
65
|
+
* import { createFederation } from "@fedify/fedify";
|
|
66
|
+
* import { MysqlKvStore, MysqlMessageQueue } from "@fedify/mysql";
|
|
67
|
+
* import mysql from "mysql2/promise";
|
|
68
|
+
*
|
|
69
|
+
* const pool = mysql.createPool("mysql://user:pass@localhost/db");
|
|
70
|
+
*
|
|
71
|
+
* const federation = createFederation({
|
|
72
|
+
* kv: new MysqlKvStore(pool),
|
|
73
|
+
* queue: new MysqlMessageQueue(pool),
|
|
74
|
+
* });
|
|
75
|
+
* ```
|
|
76
|
+
*
|
|
77
|
+
* @since 2.1.0
|
|
78
|
+
*/
|
|
79
|
+
declare class MysqlMessageQueue implements MessageQueue {
|
|
80
|
+
#private;
|
|
81
|
+
/**
|
|
82
|
+
* MySQL/MariaDB does not provide native retry mechanisms; Fedify handles
|
|
83
|
+
* retries itself.
|
|
84
|
+
* @since 2.1.0
|
|
85
|
+
*/
|
|
86
|
+
readonly nativeRetrial: false;
|
|
87
|
+
/**
|
|
88
|
+
* Creates a new MySQL message queue.
|
|
89
|
+
* @param pool The MySQL connection pool to use.
|
|
90
|
+
* @param options Options for the message queue.
|
|
91
|
+
* @since 2.1.0
|
|
92
|
+
*/
|
|
93
|
+
constructor(pool: Pool, options?: MysqlMessageQueueOptions);
|
|
94
|
+
/**
|
|
95
|
+
* {@inheritDoc MessageQueue.enqueue}
|
|
96
|
+
* @since 2.1.0
|
|
97
|
+
*/
|
|
98
|
+
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
99
|
+
/**
|
|
100
|
+
* {@inheritDoc MessageQueue.enqueueMany}
|
|
101
|
+
* @since 2.1.0
|
|
102
|
+
*/
|
|
103
|
+
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* {@inheritDoc MessageQueue.listen}
|
|
106
|
+
* @since 2.1.0
|
|
107
|
+
*/
|
|
108
|
+
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
109
|
+
/**
|
|
110
|
+
* Initializes the message queue table if it does not already exist.
|
|
111
|
+
* Concurrent calls are coalesced — only one initialization runs at a time.
|
|
112
|
+
*
|
|
113
|
+
* @since 2.1.0
|
|
114
|
+
*/
|
|
115
|
+
initialize(): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Drops the message queue table if it exists. Resets the initialized flag
|
|
118
|
+
* so that {@link MysqlMessageQueue.initialize} can recreate the table on
|
|
119
|
+
* the next call.
|
|
120
|
+
*
|
|
121
|
+
* @since 2.1.0
|
|
122
|
+
*/
|
|
123
|
+
drop(): Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
//#endregion
|
|
126
|
+
export { MysqlMessageQueue, MysqlMessageQueueOptions };
|
package/dist/mq.d.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { Temporal } from "@js-temporal/polyfill";
|
|
2
|
+
import { MessageQueue, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
|
|
3
|
+
import { Pool } from "mysql2/promise";
|
|
4
|
+
|
|
5
|
+
//#region src/mq.d.ts
|
|
6
|
+
/**
|
|
7
|
+
* Options for the MySQL message queue.
|
|
8
|
+
*
|
|
9
|
+
* @since 2.1.0
|
|
10
|
+
*/
|
|
11
|
+
interface MysqlMessageQueueOptions {
|
|
12
|
+
/**
|
|
13
|
+
* The table name to use for the message queue.
|
|
14
|
+
* `"fedify_mq"` by default.
|
|
15
|
+
* @default `"fedify_mq"`
|
|
16
|
+
* @since 2.1.0
|
|
17
|
+
*/
|
|
18
|
+
readonly tableName?: string;
|
|
19
|
+
/**
|
|
20
|
+
* Whether the table has been initialized. `false` by default.
|
|
21
|
+
* @default `false`
|
|
22
|
+
* @since 2.1.0
|
|
23
|
+
*/
|
|
24
|
+
readonly initialized?: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* The poll interval for the message queue. 1 second by default.
|
|
27
|
+
*
|
|
28
|
+
* Since MySQL/MariaDB has no `LISTEN`/`NOTIFY` equivalent, messages are
|
|
29
|
+
* discovered using periodic polling. A shorter interval reduces message
|
|
30
|
+
* delivery latency at the cost of additional database load.
|
|
31
|
+
*
|
|
32
|
+
* @default `{ seconds: 1 }`
|
|
33
|
+
* @since 2.1.0
|
|
34
|
+
*/
|
|
35
|
+
readonly pollInterval?: Temporal.Duration | Temporal.DurationLike;
|
|
36
|
+
/**
|
|
37
|
+
* The maximum time to wait for a message handler to complete before
|
|
38
|
+
* the queue moves on. This is a *soft* timeout: when a handler exceeds
|
|
39
|
+
* the limit, the queue stops waiting and logs a timeout error, but it
|
|
40
|
+
* cannot cancel the handler itself — the handler's promise continues
|
|
41
|
+
* running in the background. In the ordered-key path the advisory lock
|
|
42
|
+
* is always released in a `finally` block, so a timed-out handler will
|
|
43
|
+
* not permanently block the same ordering key. However, concurrent
|
|
44
|
+
* side-effects from the still-running handler may interleave with the
|
|
45
|
+
* next handler for the same key; callers should be aware of this.
|
|
46
|
+
*
|
|
47
|
+
* Set to zero to disable the timeout (not recommended in production).
|
|
48
|
+
*
|
|
49
|
+
* 60 seconds by default.
|
|
50
|
+
* @default `{ seconds: 60 }`
|
|
51
|
+
* @since 2.1.0
|
|
52
|
+
*/
|
|
53
|
+
readonly handlerTimeout?: Temporal.Duration | Temporal.DurationLike;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A message queue that uses MySQL or MariaDB as the underlying storage.
|
|
57
|
+
* Messages are delivered via periodic polling, since MySQL and MariaDB do not
|
|
58
|
+
* provide a `LISTEN`/`NOTIFY` equivalent.
|
|
59
|
+
*
|
|
60
|
+
* Concurrent workers are supported via `SELECT … FOR UPDATE SKIP LOCKED`
|
|
61
|
+
* (requires MySQL 8.0+ or MariaDB 10.6+) and MySQL advisory locks
|
|
62
|
+
* (`GET_LOCK`/`RELEASE_LOCK`) for ordering-key serialization.
|
|
63
|
+
*
|
|
64
|
+
* @example
|
|
65
|
+
* ```ts
|
|
66
|
+
* import { createFederation } from "@fedify/fedify";
|
|
67
|
+
* import { MysqlKvStore, MysqlMessageQueue } from "@fedify/mysql";
|
|
68
|
+
* import mysql from "mysql2/promise";
|
|
69
|
+
*
|
|
70
|
+
* const pool = mysql.createPool("mysql://user:pass@localhost/db");
|
|
71
|
+
*
|
|
72
|
+
* const federation = createFederation({
|
|
73
|
+
* kv: new MysqlKvStore(pool),
|
|
74
|
+
* queue: new MysqlMessageQueue(pool),
|
|
75
|
+
* });
|
|
76
|
+
* ```
|
|
77
|
+
*
|
|
78
|
+
* @since 2.1.0
|
|
79
|
+
*/
|
|
80
|
+
declare class MysqlMessageQueue implements MessageQueue {
|
|
81
|
+
#private;
|
|
82
|
+
/**
|
|
83
|
+
* MySQL/MariaDB does not provide native retry mechanisms; Fedify handles
|
|
84
|
+
* retries itself.
|
|
85
|
+
* @since 2.1.0
|
|
86
|
+
*/
|
|
87
|
+
readonly nativeRetrial: false;
|
|
88
|
+
/**
|
|
89
|
+
* Creates a new MySQL message queue.
|
|
90
|
+
* @param pool The MySQL connection pool to use.
|
|
91
|
+
* @param options Options for the message queue.
|
|
92
|
+
* @since 2.1.0
|
|
93
|
+
*/
|
|
94
|
+
constructor(pool: Pool, options?: MysqlMessageQueueOptions);
|
|
95
|
+
/**
|
|
96
|
+
* {@inheritDoc MessageQueue.enqueue}
|
|
97
|
+
* @since 2.1.0
|
|
98
|
+
*/
|
|
99
|
+
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* {@inheritDoc MessageQueue.enqueueMany}
|
|
102
|
+
* @since 2.1.0
|
|
103
|
+
*/
|
|
104
|
+
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
105
|
+
/**
|
|
106
|
+
* {@inheritDoc MessageQueue.listen}
|
|
107
|
+
* @since 2.1.0
|
|
108
|
+
*/
|
|
109
|
+
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Initializes the message queue table if it does not already exist.
|
|
112
|
+
* Concurrent calls are coalesced — only one initialization runs at a time.
|
|
113
|
+
*
|
|
114
|
+
* @since 2.1.0
|
|
115
|
+
*/
|
|
116
|
+
initialize(): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Drops the message queue table if it exists. Resets the initialized flag
|
|
119
|
+
* so that {@link MysqlMessageQueue.initialize} can recreate the table on
|
|
120
|
+
* the next call.
|
|
121
|
+
*
|
|
122
|
+
* @since 2.1.0
|
|
123
|
+
*/
|
|
124
|
+
drop(): Promise<void>;
|
|
125
|
+
}
|
|
126
|
+
//#endregion
|
|
127
|
+
export { MysqlMessageQueue, MysqlMessageQueueOptions };
|
package/dist/mq.js
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
|
|
2
|
+
import { Temporal } from "@js-temporal/polyfill";
|
|
3
|
+
|
|
4
|
+
import { getLogger } from "@logtape/logtape";
|
|
5
|
+
|
|
6
|
+
//#region src/mq.ts
|
|
7
|
+
const logger = getLogger([
|
|
8
|
+
"fedify",
|
|
9
|
+
"mysql",
|
|
10
|
+
"mq"
|
|
11
|
+
]);
|
|
12
|
+
const INITIALIZE_MAX_ATTEMPTS = 5;
|
|
13
|
+
const INITIALIZE_BACKOFF_MS = 10;
|
|
14
|
+
const ORDERING_KEY_CANDIDATE_LIMIT = 10;
|
|
15
|
+
function sleep(ms) {
|
|
16
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
17
|
+
}
|
|
18
|
+
function withTimeout(result, timeoutMs) {
|
|
19
|
+
const resolved = Promise.resolve(result);
|
|
20
|
+
if (timeoutMs <= 0) return resolved;
|
|
21
|
+
let timer;
|
|
22
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
23
|
+
timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`Message handler timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
24
|
+
});
|
|
25
|
+
return Promise.race([resolved, timeoutPromise]).finally(() => clearTimeout(timer));
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Computes a MySQL advisory lock name for the given table name and ordering
|
|
29
|
+
* key. The result is always at most 64 characters, which is well within
|
|
30
|
+
* MySQL's advisory lock name length limit.
|
|
31
|
+
*/
|
|
32
|
+
function getMysqlLockName(tableName, orderingKey) {
|
|
33
|
+
const raw = `${tableName}:${orderingKey}`;
|
|
34
|
+
if (raw.length <= 64) return raw;
|
|
35
|
+
let h1 = 0;
|
|
36
|
+
let h2 = 5381;
|
|
37
|
+
for (let i = 0; i < raw.length; i++) {
|
|
38
|
+
const c = raw.charCodeAt(i);
|
|
39
|
+
h1 = (h1 << 5) - h1 + c | 0;
|
|
40
|
+
h2 = (h2 << 5) + h2 + c | 0;
|
|
41
|
+
}
|
|
42
|
+
return `fdy:${(h1 >>> 0).toString(16).padStart(8, "0")}${(h2 >>> 0).toString(16).padStart(8, "0")}`;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* A message queue that uses MySQL or MariaDB as the underlying storage.
|
|
46
|
+
* Messages are delivered via periodic polling, since MySQL and MariaDB do not
|
|
47
|
+
* provide a `LISTEN`/`NOTIFY` equivalent.
|
|
48
|
+
*
|
|
49
|
+
* Concurrent workers are supported via `SELECT … FOR UPDATE SKIP LOCKED`
|
|
50
|
+
* (requires MySQL 8.0+ or MariaDB 10.6+) and MySQL advisory locks
|
|
51
|
+
* (`GET_LOCK`/`RELEASE_LOCK`) for ordering-key serialization.
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```ts
|
|
55
|
+
* import { createFederation } from "@fedify/fedify";
|
|
56
|
+
* import { MysqlKvStore, MysqlMessageQueue } from "@fedify/mysql";
|
|
57
|
+
* import mysql from "mysql2/promise";
|
|
58
|
+
*
|
|
59
|
+
* const pool = mysql.createPool("mysql://user:pass@localhost/db");
|
|
60
|
+
*
|
|
61
|
+
* const federation = createFederation({
|
|
62
|
+
* kv: new MysqlKvStore(pool),
|
|
63
|
+
* queue: new MysqlMessageQueue(pool),
|
|
64
|
+
* });
|
|
65
|
+
* ```
|
|
66
|
+
*
|
|
67
|
+
* @since 2.1.0
|
|
68
|
+
*/
|
|
69
|
+
var MysqlMessageQueue = class {
|
|
70
|
+
/**
|
|
71
|
+
* MySQL/MariaDB does not provide native retry mechanisms; Fedify handles
|
|
72
|
+
* retries itself.
|
|
73
|
+
* @since 2.1.0
|
|
74
|
+
*/
|
|
75
|
+
nativeRetrial = false;
|
|
76
|
+
#pool;
|
|
77
|
+
#tableName;
|
|
78
|
+
#pollIntervalMs;
|
|
79
|
+
#handlerTimeoutMs;
|
|
80
|
+
#initialized;
|
|
81
|
+
#initPromise;
|
|
82
|
+
/**
|
|
83
|
+
* Creates a new MySQL message queue.
|
|
84
|
+
* @param pool The MySQL connection pool to use.
|
|
85
|
+
* @param options Options for the message queue.
|
|
86
|
+
* @since 2.1.0
|
|
87
|
+
*/
|
|
88
|
+
constructor(pool, options = {}) {
|
|
89
|
+
this.#pool = pool;
|
|
90
|
+
const tableName = options.tableName ?? "fedify_mq";
|
|
91
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must start with a letter or underscore and contain only letters, digits, and underscores.`);
|
|
92
|
+
if (tableName.length > 46) throw new RangeError(`Invalid table name: ${JSON.stringify(tableName)}. Table names must be at most 46 characters long (MySQL identifier limit is 64 chars; the derived index "idx_<name>_deliver_after" uses 18 more).`);
|
|
93
|
+
this.#tableName = tableName;
|
|
94
|
+
this.#pollIntervalMs = Temporal.Duration.from(options.pollInterval ?? { seconds: 1 }).total("millisecond");
|
|
95
|
+
this.#handlerTimeoutMs = Temporal.Duration.from(options.handlerTimeout ?? { seconds: 60 }).total("millisecond");
|
|
96
|
+
this.#initialized = options.initialized ?? false;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* {@inheritDoc MessageQueue.enqueue}
|
|
100
|
+
* @since 2.1.0
|
|
101
|
+
*/
|
|
102
|
+
async enqueue(message, options) {
|
|
103
|
+
await this.initialize();
|
|
104
|
+
const delayMs = options?.delay == null ? 0 : Math.max(Math.round(options.delay.total("millisecond")), 0);
|
|
105
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
106
|
+
if (options?.delay) logger.debug("Enqueuing a message with a delay of {delayMs}ms...", {
|
|
107
|
+
delayMs,
|
|
108
|
+
message,
|
|
109
|
+
orderingKey
|
|
110
|
+
});
|
|
111
|
+
else logger.debug("Enqueuing a message...", {
|
|
112
|
+
message,
|
|
113
|
+
orderingKey
|
|
114
|
+
});
|
|
115
|
+
await this.#pool.query(`INSERT INTO \`${this.#tableName}\`
|
|
116
|
+
(\`id\`, \`message\`, \`deliver_after\`, \`ordering_key\`)
|
|
117
|
+
VALUES (
|
|
118
|
+
UUID(),
|
|
119
|
+
?,
|
|
120
|
+
DATE_ADD(NOW(6), INTERVAL ? MICROSECOND),
|
|
121
|
+
?
|
|
122
|
+
)`, [
|
|
123
|
+
JSON.stringify(message),
|
|
124
|
+
delayMs * 1e3,
|
|
125
|
+
orderingKey
|
|
126
|
+
]);
|
|
127
|
+
logger.debug("Enqueued a message.", {
|
|
128
|
+
message,
|
|
129
|
+
orderingKey
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* {@inheritDoc MessageQueue.enqueueMany}
|
|
134
|
+
* @since 2.1.0
|
|
135
|
+
*/
|
|
136
|
+
async enqueueMany(messages, options) {
|
|
137
|
+
if (messages.length === 0) return;
|
|
138
|
+
await this.initialize();
|
|
139
|
+
const delayMs = options?.delay == null ? 0 : Math.max(Math.round(options.delay.total("millisecond")), 0);
|
|
140
|
+
const orderingKey = options?.orderingKey ?? null;
|
|
141
|
+
if (options?.delay) logger.debug("Enqueuing {count} messages with a delay of {delayMs}ms...", {
|
|
142
|
+
count: messages.length,
|
|
143
|
+
delayMs,
|
|
144
|
+
orderingKey
|
|
145
|
+
});
|
|
146
|
+
else logger.debug("Enqueuing {count} messages...", {
|
|
147
|
+
count: messages.length,
|
|
148
|
+
orderingKey
|
|
149
|
+
});
|
|
150
|
+
const placeholders = messages.map(() => "(UUID(), ?, DATE_ADD(NOW(6), INTERVAL ? MICROSECOND), ?)").join(", ");
|
|
151
|
+
const values = messages.flatMap((message, index) => [
|
|
152
|
+
JSON.stringify(message),
|
|
153
|
+
delayMs * 1e3 + index,
|
|
154
|
+
orderingKey
|
|
155
|
+
]);
|
|
156
|
+
let conn;
|
|
157
|
+
try {
|
|
158
|
+
conn = await this.#pool.getConnection();
|
|
159
|
+
await conn.beginTransaction();
|
|
160
|
+
await conn.query(`INSERT INTO \`${this.#tableName}\`
|
|
161
|
+
(\`id\`, \`message\`, \`deliver_after\`, \`ordering_key\`)
|
|
162
|
+
VALUES ${placeholders}`, values);
|
|
163
|
+
await conn.commit();
|
|
164
|
+
} catch (e) {
|
|
165
|
+
if (conn != null) await conn.rollback();
|
|
166
|
+
throw e;
|
|
167
|
+
} finally {
|
|
168
|
+
conn?.release();
|
|
169
|
+
}
|
|
170
|
+
logger.debug("Enqueued {count} messages.", {
|
|
171
|
+
count: messages.length,
|
|
172
|
+
orderingKey
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* {@inheritDoc MessageQueue.listen}
|
|
177
|
+
* @since 2.1.0
|
|
178
|
+
*/
|
|
179
|
+
async listen(handler, options = {}) {
|
|
180
|
+
await this.initialize();
|
|
181
|
+
const { signal } = options;
|
|
182
|
+
const poll = async () => {
|
|
183
|
+
while (!signal?.aborted) {
|
|
184
|
+
let processed = false;
|
|
185
|
+
const noKeyMsg = await this.#dequeueWithoutOrderingKey();
|
|
186
|
+
if (noKeyMsg !== void 0) {
|
|
187
|
+
if (signal?.aborted) return;
|
|
188
|
+
await withTimeout(handler(noKeyMsg), this.#handlerTimeoutMs);
|
|
189
|
+
processed = true;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const candidates = await this.#findOrderingKeyCandidates();
|
|
193
|
+
for (const orderingKey of candidates) {
|
|
194
|
+
if (signal?.aborted) break;
|
|
195
|
+
const lockName = getMysqlLockName(this.#tableName, orderingKey);
|
|
196
|
+
let conn;
|
|
197
|
+
try {
|
|
198
|
+
conn = await this.#pool.getConnection();
|
|
199
|
+
const [lockResult] = await conn.query(`SELECT GET_LOCK(?, 0) AS acquired`, [lockName]);
|
|
200
|
+
if (lockResult[0].acquired === null) logger.warn("GET_LOCK({lockName}) returned NULL (server error); skipping ordering key {orderingKey}.", {
|
|
201
|
+
lockName,
|
|
202
|
+
orderingKey
|
|
203
|
+
});
|
|
204
|
+
else if (lockResult[0].acquired === 1) {
|
|
205
|
+
try {
|
|
206
|
+
const msg = await this.#dequeueOrderedMessage(conn, orderingKey);
|
|
207
|
+
if (msg !== void 0) {
|
|
208
|
+
if (signal?.aborted) return;
|
|
209
|
+
await withTimeout(handler(msg), this.#handlerTimeoutMs);
|
|
210
|
+
processed = true;
|
|
211
|
+
}
|
|
212
|
+
} finally {
|
|
213
|
+
await conn.query(`SELECT RELEASE_LOCK(?)`, [lockName]);
|
|
214
|
+
}
|
|
215
|
+
if (processed) break;
|
|
216
|
+
}
|
|
217
|
+
} finally {
|
|
218
|
+
conn?.release();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (!processed) break;
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
const safePoll = async (trigger) => {
|
|
225
|
+
try {
|
|
226
|
+
await poll();
|
|
227
|
+
} catch (error) {
|
|
228
|
+
logger.error("Error while polling for messages ({trigger}); will retry on next poll: {error}", {
|
|
229
|
+
trigger,
|
|
230
|
+
error
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
};
|
|
234
|
+
await safePoll("initial");
|
|
235
|
+
while (!signal?.aborted) {
|
|
236
|
+
await new Promise((resolve) => {
|
|
237
|
+
const timeoutId = setTimeout(() => {
|
|
238
|
+
signal?.removeEventListener("abort", onAbort);
|
|
239
|
+
resolve(0);
|
|
240
|
+
}, this.#pollIntervalMs);
|
|
241
|
+
function onAbort() {
|
|
242
|
+
clearTimeout(timeoutId);
|
|
243
|
+
resolve(void 0);
|
|
244
|
+
}
|
|
245
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
246
|
+
});
|
|
247
|
+
if (signal?.aborted) break;
|
|
248
|
+
await safePoll("interval");
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Atomically dequeues the oldest ready message that has no ordering key,
|
|
253
|
+
* using `FOR UPDATE SKIP LOCKED` within a transaction.
|
|
254
|
+
* Returns `undefined` when no such message is available.
|
|
255
|
+
*/
|
|
256
|
+
async #dequeueWithoutOrderingKey() {
|
|
257
|
+
let conn;
|
|
258
|
+
try {
|
|
259
|
+
conn = await this.#pool.getConnection();
|
|
260
|
+
await conn.beginTransaction();
|
|
261
|
+
const [rows] = await conn.query(`SELECT \`id\`, \`message\`
|
|
262
|
+
FROM \`${this.#tableName}\`
|
|
263
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` IS NULL
|
|
264
|
+
ORDER BY \`deliver_after\`
|
|
265
|
+
LIMIT 1
|
|
266
|
+
FOR UPDATE SKIP LOCKED`);
|
|
267
|
+
if (rows.length === 0) {
|
|
268
|
+
await conn.rollback();
|
|
269
|
+
return void 0;
|
|
270
|
+
}
|
|
271
|
+
const { id, message } = rows[0];
|
|
272
|
+
await conn.query(`DELETE FROM \`${this.#tableName}\` WHERE \`id\` = ?`, [id]);
|
|
273
|
+
await conn.commit();
|
|
274
|
+
return message;
|
|
275
|
+
} catch (e) {
|
|
276
|
+
if (conn != null) await conn.rollback();
|
|
277
|
+
throw e;
|
|
278
|
+
} finally {
|
|
279
|
+
conn?.release();
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Returns up to {@link ORDERING_KEY_CANDIDATE_LIMIT} distinct ordering keys
|
|
284
|
+
* that have at least one ready message, ordered by their earliest
|
|
285
|
+
* `deliver_after`. Fetching a batch in one query avoids the growing
|
|
286
|
+
* `NOT IN (…)` list that results from repeatedly calling a single-result
|
|
287
|
+
* version after each failed lock attempt.
|
|
288
|
+
*/
|
|
289
|
+
async #findOrderingKeyCandidates() {
|
|
290
|
+
const [rows] = await this.#pool.query(`SELECT \`ordering_key\`
|
|
291
|
+
FROM \`${this.#tableName}\`
|
|
292
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` IS NOT NULL
|
|
293
|
+
GROUP BY \`ordering_key\`
|
|
294
|
+
ORDER BY MIN(\`deliver_after\`)
|
|
295
|
+
LIMIT ?`, [ORDERING_KEY_CANDIDATE_LIMIT]);
|
|
296
|
+
return rows.map((r) => r.ordering_key);
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Dequeues the oldest ready message for the given ordering key using
|
|
300
|
+
* the supplied (dedicated) connection. The caller MUST hold the advisory
|
|
301
|
+
* lock for `orderingKey` before calling this method.
|
|
302
|
+
* Returns `undefined` when no ready message exists for the ordering key.
|
|
303
|
+
*/
|
|
304
|
+
async #dequeueOrderedMessage(conn, orderingKey) {
|
|
305
|
+
await conn.beginTransaction();
|
|
306
|
+
try {
|
|
307
|
+
const [rows] = await conn.query(`SELECT \`id\`, \`message\`
|
|
308
|
+
FROM \`${this.#tableName}\`
|
|
309
|
+
WHERE \`deliver_after\` <= NOW(6) AND \`ordering_key\` = ?
|
|
310
|
+
ORDER BY \`deliver_after\`
|
|
311
|
+
LIMIT 1`, [orderingKey]);
|
|
312
|
+
if (rows.length === 0) {
|
|
313
|
+
await conn.rollback();
|
|
314
|
+
return void 0;
|
|
315
|
+
}
|
|
316
|
+
const { id, message } = rows[0];
|
|
317
|
+
await conn.query(`DELETE FROM \`${this.#tableName}\` WHERE \`id\` = ?`, [id]);
|
|
318
|
+
await conn.commit();
|
|
319
|
+
return message;
|
|
320
|
+
} catch (e) {
|
|
321
|
+
await conn.rollback();
|
|
322
|
+
throw e;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Initializes the message queue table if it does not already exist.
|
|
327
|
+
* Concurrent calls are coalesced — only one initialization runs at a time.
|
|
328
|
+
*
|
|
329
|
+
* @since 2.1.0
|
|
330
|
+
*/
|
|
331
|
+
initialize() {
|
|
332
|
+
if (this.#initialized) return Promise.resolve();
|
|
333
|
+
return this.#initPromise ??= this.#doInitialize();
|
|
334
|
+
}
|
|
335
|
+
async #doInitialize() {
|
|
336
|
+
logger.debug("Initializing the message queue table {tableName}...", { tableName: this.#tableName });
|
|
337
|
+
for (let attempt = 1; attempt <= INITIALIZE_MAX_ATTEMPTS; attempt++) try {
|
|
338
|
+
await this.#pool.query(`CREATE TABLE IF NOT EXISTS \`${this.#tableName}\` (
|
|
339
|
+
\`id\` CHAR(36) NOT NULL,
|
|
340
|
+
\`message\` JSON NOT NULL,
|
|
341
|
+
\`deliver_after\` DATETIME(6) NOT NULL DEFAULT NOW(6),
|
|
342
|
+
\`ordering_key\` TEXT NULL DEFAULT NULL,
|
|
343
|
+
PRIMARY KEY (\`id\`)
|
|
344
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`);
|
|
345
|
+
try {
|
|
346
|
+
await this.#pool.query(`CREATE INDEX \`idx_${this.#tableName}_deliver_after\`
|
|
347
|
+
ON \`${this.#tableName}\` (\`deliver_after\`)`);
|
|
348
|
+
} catch (e) {
|
|
349
|
+
if (e.code !== "ER_DUP_KEYNAME") throw e;
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
await this.#pool.query(`CREATE INDEX \`idx_${this.#tableName}_ok_da\`
|
|
353
|
+
ON \`${this.#tableName}\` (\`ordering_key\`(766), \`deliver_after\`)`);
|
|
354
|
+
} catch (e) {
|
|
355
|
+
if (e.code !== "ER_DUP_KEYNAME") throw e;
|
|
356
|
+
}
|
|
357
|
+
break;
|
|
358
|
+
} catch (error) {
|
|
359
|
+
if (attempt >= INITIALIZE_MAX_ATTEMPTS) {
|
|
360
|
+
logger.error("Failed to initialize the message queue table: {error}", { error });
|
|
361
|
+
throw error;
|
|
362
|
+
}
|
|
363
|
+
const backoffMs = INITIALIZE_BACKOFF_MS * 2 ** (attempt - 1);
|
|
364
|
+
logger.debug("Initialization race for table {tableName}; retrying in {backoffMs}ms (attempt {attempt}/{maxAttempts}).", {
|
|
365
|
+
tableName: this.#tableName,
|
|
366
|
+
backoffMs,
|
|
367
|
+
attempt,
|
|
368
|
+
maxAttempts: INITIALIZE_MAX_ATTEMPTS,
|
|
369
|
+
error
|
|
370
|
+
});
|
|
371
|
+
await sleep(backoffMs);
|
|
372
|
+
}
|
|
373
|
+
this.#initialized = true;
|
|
374
|
+
logger.debug("Initialized the message queue table {tableName}.", { tableName: this.#tableName });
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Drops the message queue table if it exists. Resets the initialized flag
|
|
378
|
+
* so that {@link MysqlMessageQueue.initialize} can recreate the table on
|
|
379
|
+
* the next call.
|
|
380
|
+
*
|
|
381
|
+
* @since 2.1.0
|
|
382
|
+
*/
|
|
383
|
+
async drop() {
|
|
384
|
+
if (this.#initPromise != null) try {
|
|
385
|
+
await this.#initPromise;
|
|
386
|
+
} catch {}
|
|
387
|
+
await this.#pool.query(`DROP TABLE IF EXISTS \`${this.#tableName}\``);
|
|
388
|
+
this.#initialized = false;
|
|
389
|
+
this.#initPromise = void 0;
|
|
390
|
+
}
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
//#endregion
|
|
394
|
+
export { MysqlMessageQueue };
|