@fedify/amqp 2.0.0-dev.237 → 2.0.0-dev.279
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno.json +3 -2
- package/dist/mod.d.cts +2 -2
- package/dist/mod.d.ts +2 -2
- package/dist/mq.cjs +112 -13
- package/dist/mq.d.cts +80 -1
- package/dist/mq.d.ts +80 -1
- package/dist/mq.js +112 -13
- package/package.json +10 -7
- package/src/mq.test.ts +41 -0
- package/src/mq.ts +230 -14
package/deno.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fedify/amqp",
|
|
3
|
-
"version": "2.0.0-dev.
|
|
3
|
+
"version": "2.0.0-dev.279+ce1bdc22",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./src/mod.ts",
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
],
|
|
19
19
|
"publish": {
|
|
20
20
|
"exclude": [
|
|
21
|
-
"**/*.test.ts"
|
|
21
|
+
"**/*.test.ts",
|
|
22
|
+
"tsdown.config.ts"
|
|
22
23
|
]
|
|
23
24
|
},
|
|
24
25
|
"tasks": {
|
package/dist/mod.d.cts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { AmqpMessageQueue, AmqpMessageQueueOptions } from "./mq.cjs";
|
|
2
|
-
export { AmqpMessageQueue, AmqpMessageQueueOptions };
|
|
1
|
+
import { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions } from "./mq.cjs";
|
|
2
|
+
export { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions };
|
package/dist/mod.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { AmqpMessageQueue, AmqpMessageQueueOptions } from "./mq.js";
|
|
2
|
-
export { AmqpMessageQueue, AmqpMessageQueueOptions };
|
|
1
|
+
import { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions } from "./mq.js";
|
|
2
|
+
export { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions };
|
package/dist/mq.cjs
CHANGED
|
@@ -23,6 +23,8 @@ var AmqpMessageQueue = class {
|
|
|
23
23
|
#delayedQueuePrefix;
|
|
24
24
|
#durable;
|
|
25
25
|
#senderChannel;
|
|
26
|
+
#ordering;
|
|
27
|
+
#orderingPrepared = false;
|
|
26
28
|
nativeRetrial;
|
|
27
29
|
/**
|
|
28
30
|
* Creates a new `AmqpMessageQueue`.
|
|
@@ -35,30 +37,85 @@ var AmqpMessageQueue = class {
|
|
|
35
37
|
this.#delayedQueuePrefix = options.delayedQueuePrefix ?? "fedify_delayed_";
|
|
36
38
|
this.#durable = options.durable ?? true;
|
|
37
39
|
this.nativeRetrial = options.nativeRetrial ?? false;
|
|
40
|
+
if (options.ordering != null) this.#ordering = {
|
|
41
|
+
exchange: options.ordering.exchange ?? "fedify_ordering",
|
|
42
|
+
queuePrefix: options.ordering.queuePrefix ?? "fedify_ordering_",
|
|
43
|
+
partitions: options.ordering.partitions ?? 4
|
|
44
|
+
};
|
|
38
45
|
}
|
|
39
46
|
async #prepareQueue(channel) {
|
|
40
47
|
await channel.assertQueue(this.#queue, { durable: this.#durable });
|
|
41
48
|
}
|
|
49
|
+
#getOrderingQueueName(partition) {
|
|
50
|
+
return `${this.#ordering.queuePrefix}${partition}`;
|
|
51
|
+
}
|
|
52
|
+
async #prepareOrdering(channel) {
|
|
53
|
+
if (this.#ordering == null || this.#orderingPrepared) return;
|
|
54
|
+
await channel.assertExchange(this.#ordering.exchange, "x-consistent-hash", { durable: this.#durable });
|
|
55
|
+
for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
56
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
57
|
+
await channel.assertQueue(queueName, {
|
|
58
|
+
durable: this.#durable,
|
|
59
|
+
arguments: { "x-single-active-consumer": true }
|
|
60
|
+
});
|
|
61
|
+
await channel.bindQueue(queueName, this.#ordering.exchange, "1");
|
|
62
|
+
}
|
|
63
|
+
this.#orderingPrepared = true;
|
|
64
|
+
}
|
|
42
65
|
async #getSenderChannel() {
|
|
43
66
|
if (this.#senderChannel != null) return this.#senderChannel;
|
|
44
67
|
const channel = await this.#connection.createChannel();
|
|
45
68
|
this.#senderChannel = channel;
|
|
46
|
-
this.#prepareQueue(channel);
|
|
69
|
+
await this.#prepareQueue(channel);
|
|
70
|
+
await this.#prepareOrdering(channel);
|
|
47
71
|
return channel;
|
|
48
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Enqueues a message to be processed.
|
|
75
|
+
*
|
|
76
|
+
* When an `orderingKey` is provided without a `delay`, the message is routed
|
|
77
|
+
* through the consistent hash exchange, ensuring messages with the same
|
|
78
|
+
* ordering key are processed by the same consumer in FIFO order.
|
|
79
|
+
*
|
|
80
|
+
* When both `orderingKey` and `delay` are provided, the message is first
|
|
81
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
82
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
83
|
+
* delayed messages.
|
|
84
|
+
*
|
|
85
|
+
* @param message The message to enqueue.
|
|
86
|
+
* @param options The options for enqueueing the message.
|
|
87
|
+
*/
|
|
49
88
|
async enqueue(message, options) {
|
|
50
89
|
const channel = await this.#getSenderChannel();
|
|
51
90
|
const delay = options?.delay?.total("millisecond");
|
|
91
|
+
const orderingKey = options?.orderingKey;
|
|
92
|
+
if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
|
|
93
|
+
channel.publish(this.#ordering.exchange, orderingKey, node_buffer.Buffer.from(JSON.stringify(message), "utf-8"), {
|
|
94
|
+
persistent: this.#durable,
|
|
95
|
+
contentType: "application/json"
|
|
96
|
+
});
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
52
99
|
let queue;
|
|
100
|
+
let deadLetterExchange;
|
|
101
|
+
let deadLetterRoutingKey;
|
|
53
102
|
if (delay == null || delay <= 0) queue = this.#queue;
|
|
54
103
|
else {
|
|
55
104
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
56
|
-
|
|
105
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
106
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
107
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
108
|
+
deadLetterRoutingKey = orderingKey;
|
|
109
|
+
} else {
|
|
110
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
111
|
+
deadLetterExchange = "";
|
|
112
|
+
deadLetterRoutingKey = this.#queue;
|
|
113
|
+
}
|
|
57
114
|
await channel.assertQueue(queue, {
|
|
58
115
|
autoDelete: true,
|
|
59
116
|
durable: this.#durable,
|
|
60
|
-
deadLetterExchange
|
|
61
|
-
deadLetterRoutingKey
|
|
117
|
+
deadLetterExchange,
|
|
118
|
+
deadLetterRoutingKey,
|
|
62
119
|
messageTtl: delay
|
|
63
120
|
});
|
|
64
121
|
}
|
|
@@ -67,19 +124,52 @@ var AmqpMessageQueue = class {
|
|
|
67
124
|
contentType: "application/json"
|
|
68
125
|
});
|
|
69
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Enqueues multiple messages to be processed.
|
|
129
|
+
*
|
|
130
|
+
* When an `orderingKey` is provided without a `delay`, the messages are
|
|
131
|
+
* routed through the consistent hash exchange, ensuring messages with the
|
|
132
|
+
* same ordering key are processed by the same consumer in FIFO order.
|
|
133
|
+
*
|
|
134
|
+
* When both `orderingKey` and `delay` are provided, the messages are first
|
|
135
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
136
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
137
|
+
* delayed messages.
|
|
138
|
+
*
|
|
139
|
+
* @param messages The messages to enqueue.
|
|
140
|
+
* @param options The options for enqueueing the messages.
|
|
141
|
+
*/
|
|
70
142
|
async enqueueMany(messages, options) {
|
|
71
143
|
const channel = await this.#getSenderChannel();
|
|
72
144
|
const delay = options?.delay?.total("millisecond");
|
|
145
|
+
const orderingKey = options?.orderingKey;
|
|
146
|
+
if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
|
|
147
|
+
for (const message of messages) channel.publish(this.#ordering.exchange, orderingKey, node_buffer.Buffer.from(JSON.stringify(message), "utf-8"), {
|
|
148
|
+
persistent: this.#durable,
|
|
149
|
+
contentType: "application/json"
|
|
150
|
+
});
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
73
153
|
let queue;
|
|
154
|
+
let deadLetterExchange;
|
|
155
|
+
let deadLetterRoutingKey;
|
|
74
156
|
if (delay == null || delay <= 0) queue = this.#queue;
|
|
75
157
|
else {
|
|
76
158
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
77
|
-
|
|
159
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
160
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
161
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
162
|
+
deadLetterRoutingKey = orderingKey;
|
|
163
|
+
} else {
|
|
164
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
165
|
+
deadLetterExchange = "";
|
|
166
|
+
deadLetterRoutingKey = this.#queue;
|
|
167
|
+
}
|
|
78
168
|
await channel.assertQueue(queue, {
|
|
79
169
|
autoDelete: true,
|
|
80
170
|
durable: this.#durable,
|
|
81
|
-
deadLetterExchange
|
|
82
|
-
deadLetterRoutingKey
|
|
171
|
+
deadLetterExchange,
|
|
172
|
+
deadLetterRoutingKey,
|
|
83
173
|
messageTtl: delay
|
|
84
174
|
});
|
|
85
175
|
}
|
|
@@ -91,8 +181,9 @@ var AmqpMessageQueue = class {
|
|
|
91
181
|
async listen(handler, options = {}) {
|
|
92
182
|
const channel = await this.#connection.createChannel();
|
|
93
183
|
await this.#prepareQueue(channel);
|
|
184
|
+
await this.#prepareOrdering(channel);
|
|
94
185
|
await channel.prefetch(1);
|
|
95
|
-
const
|
|
186
|
+
const messageHandler = (msg) => {
|
|
96
187
|
if (msg == null) return;
|
|
97
188
|
const message = JSON.parse(msg.content.toString("utf-8"));
|
|
98
189
|
try {
|
|
@@ -105,13 +196,21 @@ var AmqpMessageQueue = class {
|
|
|
105
196
|
} finally {
|
|
106
197
|
if (!this.nativeRetrial) channel.ack(msg);
|
|
107
198
|
}
|
|
108
|
-
}
|
|
199
|
+
};
|
|
200
|
+
const consumerTags = [];
|
|
201
|
+
const reply = await channel.consume(this.#queue, messageHandler, { noAck: false });
|
|
202
|
+
consumerTags.push(reply.consumerTag);
|
|
203
|
+
if (this.#ordering != null) for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
204
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
205
|
+
const orderingReply = await channel.consume(queueName, messageHandler, { noAck: false });
|
|
206
|
+
consumerTags.push(orderingReply.consumerTag);
|
|
207
|
+
}
|
|
109
208
|
return await new Promise((resolve) => {
|
|
110
209
|
if (options.signal?.aborted) resolve();
|
|
111
|
-
options.signal?.addEventListener("abort", () => {
|
|
112
|
-
channel.cancel(
|
|
113
|
-
|
|
114
|
-
|
|
210
|
+
options.signal?.addEventListener("abort", async () => {
|
|
211
|
+
for (const tag of consumerTags) await channel.cancel(tag);
|
|
212
|
+
await channel.close();
|
|
213
|
+
resolve();
|
|
115
214
|
});
|
|
116
215
|
});
|
|
117
216
|
}
|
package/dist/mq.d.cts
CHANGED
|
@@ -3,6 +3,39 @@ import { ChannelModel } from "amqplib";
|
|
|
3
3
|
|
|
4
4
|
//#region src/mq.d.ts
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Options for ordering key support in {@link AmqpMessageQueue}.
|
|
8
|
+
*
|
|
9
|
+
* Ordering key support requires the `rabbitmq_consistent_hash_exchange`
|
|
10
|
+
* plugin to be enabled on the RabbitMQ server. You can enable it by running:
|
|
11
|
+
*
|
|
12
|
+
* ```sh
|
|
13
|
+
* rabbitmq-plugins enable rabbitmq_consistent_hash_exchange
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* @since 2.0.0
|
|
17
|
+
*/
|
|
18
|
+
interface AmqpOrderingOptions {
|
|
19
|
+
/**
|
|
20
|
+
* The name of the consistent hash exchange to use for ordering.
|
|
21
|
+
* Defaults to `"fedify_ordering"`.
|
|
22
|
+
* @default `"fedify_ordering"`
|
|
23
|
+
*/
|
|
24
|
+
readonly exchange?: string;
|
|
25
|
+
/**
|
|
26
|
+
* The prefix to use for ordering queues.
|
|
27
|
+
* Defaults to `"fedify_ordering_"`.
|
|
28
|
+
* @default `"fedify_ordering_"`
|
|
29
|
+
*/
|
|
30
|
+
readonly queuePrefix?: string;
|
|
31
|
+
/**
|
|
32
|
+
* The number of partitions (queues) to use for ordering.
|
|
33
|
+
* More partitions allow better parallelism for different ordering keys.
|
|
34
|
+
* Defaults to `4`.
|
|
35
|
+
* @default `4`
|
|
36
|
+
*/
|
|
37
|
+
readonly partitions?: number;
|
|
38
|
+
}
|
|
6
39
|
/**
|
|
7
40
|
* Options for {@link AmqpMessageQueue}.
|
|
8
41
|
*/
|
|
@@ -39,6 +72,22 @@ interface AmqpMessageQueueOptions {
|
|
|
39
72
|
* @since 0.3.0
|
|
40
73
|
*/
|
|
41
74
|
readonly nativeRetrial?: boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Options for ordering key support. If provided, the message queue will
|
|
77
|
+
* support the `orderingKey` option in {@link MessageQueueEnqueueOptions}.
|
|
78
|
+
* Messages with the same ordering key will be processed in order,
|
|
79
|
+
* while messages with different ordering keys can be processed in parallel.
|
|
80
|
+
*
|
|
81
|
+
* This feature requires the `rabbitmq_consistent_hash_exchange` plugin
|
|
82
|
+
* to be enabled on the RabbitMQ server. See {@link AmqpOrderingOptions}
|
|
83
|
+
* for more details.
|
|
84
|
+
*
|
|
85
|
+
* If not provided, ordering key support is disabled and any `orderingKey`
|
|
86
|
+
* option passed to `enqueue()` will be ignored.
|
|
87
|
+
*
|
|
88
|
+
* @since 0.4.0
|
|
89
|
+
*/
|
|
90
|
+
readonly ordering?: AmqpOrderingOptions;
|
|
42
91
|
}
|
|
43
92
|
/**
|
|
44
93
|
* A message queue that uses AMQP.
|
|
@@ -64,9 +113,39 @@ declare class AmqpMessageQueue implements MessageQueue {
|
|
|
64
113
|
* @param options Options for the message queue.
|
|
65
114
|
*/
|
|
66
115
|
constructor(connection: ChannelModel, options?: AmqpMessageQueueOptions);
|
|
116
|
+
/**
|
|
117
|
+
* Enqueues a message to be processed.
|
|
118
|
+
*
|
|
119
|
+
* When an `orderingKey` is provided without a `delay`, the message is routed
|
|
120
|
+
* through the consistent hash exchange, ensuring messages with the same
|
|
121
|
+
* ordering key are processed by the same consumer in FIFO order.
|
|
122
|
+
*
|
|
123
|
+
* When both `orderingKey` and `delay` are provided, the message is first
|
|
124
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
125
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
126
|
+
* delayed messages.
|
|
127
|
+
*
|
|
128
|
+
* @param message The message to enqueue.
|
|
129
|
+
* @param options The options for enqueueing the message.
|
|
130
|
+
*/
|
|
67
131
|
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Enqueues multiple messages to be processed.
|
|
134
|
+
*
|
|
135
|
+
* When an `orderingKey` is provided without a `delay`, the messages are
|
|
136
|
+
* routed through the consistent hash exchange, ensuring messages with the
|
|
137
|
+
* same ordering key are processed by the same consumer in FIFO order.
|
|
138
|
+
*
|
|
139
|
+
* When both `orderingKey` and `delay` are provided, the messages are first
|
|
140
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
141
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
142
|
+
* delayed messages.
|
|
143
|
+
*
|
|
144
|
+
* @param messages The messages to enqueue.
|
|
145
|
+
* @param options The options for enqueueing the messages.
|
|
146
|
+
*/
|
|
68
147
|
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
69
148
|
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
70
149
|
}
|
|
71
150
|
//#endregion
|
|
72
|
-
export { AmqpMessageQueue, AmqpMessageQueueOptions };
|
|
151
|
+
export { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions };
|
package/dist/mq.d.ts
CHANGED
|
@@ -3,6 +3,39 @@ import { ChannelModel } from "amqplib";
|
|
|
3
3
|
|
|
4
4
|
//#region src/mq.d.ts
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Options for ordering key support in {@link AmqpMessageQueue}.
|
|
8
|
+
*
|
|
9
|
+
* Ordering key support requires the `rabbitmq_consistent_hash_exchange`
|
|
10
|
+
* plugin to be enabled on the RabbitMQ server. You can enable it by running:
|
|
11
|
+
*
|
|
12
|
+
* ```sh
|
|
13
|
+
* rabbitmq-plugins enable rabbitmq_consistent_hash_exchange
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* @since 2.0.0
|
|
17
|
+
*/
|
|
18
|
+
interface AmqpOrderingOptions {
|
|
19
|
+
/**
|
|
20
|
+
* The name of the consistent hash exchange to use for ordering.
|
|
21
|
+
* Defaults to `"fedify_ordering"`.
|
|
22
|
+
* @default `"fedify_ordering"`
|
|
23
|
+
*/
|
|
24
|
+
readonly exchange?: string;
|
|
25
|
+
/**
|
|
26
|
+
* The prefix to use for ordering queues.
|
|
27
|
+
* Defaults to `"fedify_ordering_"`.
|
|
28
|
+
* @default `"fedify_ordering_"`
|
|
29
|
+
*/
|
|
30
|
+
readonly queuePrefix?: string;
|
|
31
|
+
/**
|
|
32
|
+
* The number of partitions (queues) to use for ordering.
|
|
33
|
+
* More partitions allow better parallelism for different ordering keys.
|
|
34
|
+
* Defaults to `4`.
|
|
35
|
+
* @default `4`
|
|
36
|
+
*/
|
|
37
|
+
readonly partitions?: number;
|
|
38
|
+
}
|
|
6
39
|
/**
|
|
7
40
|
* Options for {@link AmqpMessageQueue}.
|
|
8
41
|
*/
|
|
@@ -39,6 +72,22 @@ interface AmqpMessageQueueOptions {
|
|
|
39
72
|
* @since 0.3.0
|
|
40
73
|
*/
|
|
41
74
|
readonly nativeRetrial?: boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Options for ordering key support. If provided, the message queue will
|
|
77
|
+
* support the `orderingKey` option in {@link MessageQueueEnqueueOptions}.
|
|
78
|
+
* Messages with the same ordering key will be processed in order,
|
|
79
|
+
* while messages with different ordering keys can be processed in parallel.
|
|
80
|
+
*
|
|
81
|
+
* This feature requires the `rabbitmq_consistent_hash_exchange` plugin
|
|
82
|
+
* to be enabled on the RabbitMQ server. See {@link AmqpOrderingOptions}
|
|
83
|
+
* for more details.
|
|
84
|
+
*
|
|
85
|
+
* If not provided, ordering key support is disabled and any `orderingKey`
|
|
86
|
+
* option passed to `enqueue()` will be ignored.
|
|
87
|
+
*
|
|
88
|
+
* @since 0.4.0
|
|
89
|
+
*/
|
|
90
|
+
readonly ordering?: AmqpOrderingOptions;
|
|
42
91
|
}
|
|
43
92
|
/**
|
|
44
93
|
* A message queue that uses AMQP.
|
|
@@ -64,9 +113,39 @@ declare class AmqpMessageQueue implements MessageQueue {
|
|
|
64
113
|
* @param options Options for the message queue.
|
|
65
114
|
*/
|
|
66
115
|
constructor(connection: ChannelModel, options?: AmqpMessageQueueOptions);
|
|
116
|
+
/**
|
|
117
|
+
* Enqueues a message to be processed.
|
|
118
|
+
*
|
|
119
|
+
* When an `orderingKey` is provided without a `delay`, the message is routed
|
|
120
|
+
* through the consistent hash exchange, ensuring messages with the same
|
|
121
|
+
* ordering key are processed by the same consumer in FIFO order.
|
|
122
|
+
*
|
|
123
|
+
* When both `orderingKey` and `delay` are provided, the message is first
|
|
124
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
125
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
126
|
+
* delayed messages.
|
|
127
|
+
*
|
|
128
|
+
* @param message The message to enqueue.
|
|
129
|
+
* @param options The options for enqueueing the message.
|
|
130
|
+
*/
|
|
67
131
|
enqueue(message: any, options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Enqueues multiple messages to be processed.
|
|
134
|
+
*
|
|
135
|
+
* When an `orderingKey` is provided without a `delay`, the messages are
|
|
136
|
+
* routed through the consistent hash exchange, ensuring messages with the
|
|
137
|
+
* same ordering key are processed by the same consumer in FIFO order.
|
|
138
|
+
*
|
|
139
|
+
* When both `orderingKey` and `delay` are provided, the messages are first
|
|
140
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
141
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
142
|
+
* delayed messages.
|
|
143
|
+
*
|
|
144
|
+
* @param messages The messages to enqueue.
|
|
145
|
+
* @param options The options for enqueueing the messages.
|
|
146
|
+
*/
|
|
68
147
|
enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
|
|
69
148
|
listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
|
|
70
149
|
}
|
|
71
150
|
//#endregion
|
|
72
|
-
export { AmqpMessageQueue, AmqpMessageQueueOptions };
|
|
151
|
+
export { AmqpMessageQueue, AmqpMessageQueueOptions, AmqpOrderingOptions };
|
package/dist/mq.js
CHANGED
|
@@ -22,6 +22,8 @@ var AmqpMessageQueue = class {
|
|
|
22
22
|
#delayedQueuePrefix;
|
|
23
23
|
#durable;
|
|
24
24
|
#senderChannel;
|
|
25
|
+
#ordering;
|
|
26
|
+
#orderingPrepared = false;
|
|
25
27
|
nativeRetrial;
|
|
26
28
|
/**
|
|
27
29
|
* Creates a new `AmqpMessageQueue`.
|
|
@@ -34,30 +36,85 @@ var AmqpMessageQueue = class {
|
|
|
34
36
|
this.#delayedQueuePrefix = options.delayedQueuePrefix ?? "fedify_delayed_";
|
|
35
37
|
this.#durable = options.durable ?? true;
|
|
36
38
|
this.nativeRetrial = options.nativeRetrial ?? false;
|
|
39
|
+
if (options.ordering != null) this.#ordering = {
|
|
40
|
+
exchange: options.ordering.exchange ?? "fedify_ordering",
|
|
41
|
+
queuePrefix: options.ordering.queuePrefix ?? "fedify_ordering_",
|
|
42
|
+
partitions: options.ordering.partitions ?? 4
|
|
43
|
+
};
|
|
37
44
|
}
|
|
38
45
|
async #prepareQueue(channel) {
|
|
39
46
|
await channel.assertQueue(this.#queue, { durable: this.#durable });
|
|
40
47
|
}
|
|
48
|
+
#getOrderingQueueName(partition) {
|
|
49
|
+
return `${this.#ordering.queuePrefix}${partition}`;
|
|
50
|
+
}
|
|
51
|
+
async #prepareOrdering(channel) {
|
|
52
|
+
if (this.#ordering == null || this.#orderingPrepared) return;
|
|
53
|
+
await channel.assertExchange(this.#ordering.exchange, "x-consistent-hash", { durable: this.#durable });
|
|
54
|
+
for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
55
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
56
|
+
await channel.assertQueue(queueName, {
|
|
57
|
+
durable: this.#durable,
|
|
58
|
+
arguments: { "x-single-active-consumer": true }
|
|
59
|
+
});
|
|
60
|
+
await channel.bindQueue(queueName, this.#ordering.exchange, "1");
|
|
61
|
+
}
|
|
62
|
+
this.#orderingPrepared = true;
|
|
63
|
+
}
|
|
41
64
|
async #getSenderChannel() {
|
|
42
65
|
if (this.#senderChannel != null) return this.#senderChannel;
|
|
43
66
|
const channel = await this.#connection.createChannel();
|
|
44
67
|
this.#senderChannel = channel;
|
|
45
|
-
this.#prepareQueue(channel);
|
|
68
|
+
await this.#prepareQueue(channel);
|
|
69
|
+
await this.#prepareOrdering(channel);
|
|
46
70
|
return channel;
|
|
47
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Enqueues a message to be processed.
|
|
74
|
+
*
|
|
75
|
+
* When an `orderingKey` is provided without a `delay`, the message is routed
|
|
76
|
+
* through the consistent hash exchange, ensuring messages with the same
|
|
77
|
+
* ordering key are processed by the same consumer in FIFO order.
|
|
78
|
+
*
|
|
79
|
+
* When both `orderingKey` and `delay` are provided, the message is first
|
|
80
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
81
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
82
|
+
* delayed messages.
|
|
83
|
+
*
|
|
84
|
+
* @param message The message to enqueue.
|
|
85
|
+
* @param options The options for enqueueing the message.
|
|
86
|
+
*/
|
|
48
87
|
async enqueue(message, options) {
|
|
49
88
|
const channel = await this.#getSenderChannel();
|
|
50
89
|
const delay = options?.delay?.total("millisecond");
|
|
90
|
+
const orderingKey = options?.orderingKey;
|
|
91
|
+
if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
|
|
92
|
+
channel.publish(this.#ordering.exchange, orderingKey, Buffer.from(JSON.stringify(message), "utf-8"), {
|
|
93
|
+
persistent: this.#durable,
|
|
94
|
+
contentType: "application/json"
|
|
95
|
+
});
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
51
98
|
let queue;
|
|
99
|
+
let deadLetterExchange;
|
|
100
|
+
let deadLetterRoutingKey;
|
|
52
101
|
if (delay == null || delay <= 0) queue = this.#queue;
|
|
53
102
|
else {
|
|
54
103
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
55
|
-
|
|
104
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
105
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
106
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
107
|
+
deadLetterRoutingKey = orderingKey;
|
|
108
|
+
} else {
|
|
109
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
110
|
+
deadLetterExchange = "";
|
|
111
|
+
deadLetterRoutingKey = this.#queue;
|
|
112
|
+
}
|
|
56
113
|
await channel.assertQueue(queue, {
|
|
57
114
|
autoDelete: true,
|
|
58
115
|
durable: this.#durable,
|
|
59
|
-
deadLetterExchange
|
|
60
|
-
deadLetterRoutingKey
|
|
116
|
+
deadLetterExchange,
|
|
117
|
+
deadLetterRoutingKey,
|
|
61
118
|
messageTtl: delay
|
|
62
119
|
});
|
|
63
120
|
}
|
|
@@ -66,19 +123,52 @@ var AmqpMessageQueue = class {
|
|
|
66
123
|
contentType: "application/json"
|
|
67
124
|
});
|
|
68
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Enqueues multiple messages to be processed.
|
|
128
|
+
*
|
|
129
|
+
* When an `orderingKey` is provided without a `delay`, the messages are
|
|
130
|
+
* routed through the consistent hash exchange, ensuring messages with the
|
|
131
|
+
* same ordering key are processed by the same consumer in FIFO order.
|
|
132
|
+
*
|
|
133
|
+
* When both `orderingKey` and `delay` are provided, the messages are first
|
|
134
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
135
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
136
|
+
* delayed messages.
|
|
137
|
+
*
|
|
138
|
+
* @param messages The messages to enqueue.
|
|
139
|
+
* @param options The options for enqueueing the messages.
|
|
140
|
+
*/
|
|
69
141
|
async enqueueMany(messages, options) {
|
|
70
142
|
const channel = await this.#getSenderChannel();
|
|
71
143
|
const delay = options?.delay?.total("millisecond");
|
|
144
|
+
const orderingKey = options?.orderingKey;
|
|
145
|
+
if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
|
|
146
|
+
for (const message of messages) channel.publish(this.#ordering.exchange, orderingKey, Buffer.from(JSON.stringify(message), "utf-8"), {
|
|
147
|
+
persistent: this.#durable,
|
|
148
|
+
contentType: "application/json"
|
|
149
|
+
});
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
72
152
|
let queue;
|
|
153
|
+
let deadLetterExchange;
|
|
154
|
+
let deadLetterRoutingKey;
|
|
73
155
|
if (delay == null || delay <= 0) queue = this.#queue;
|
|
74
156
|
else {
|
|
75
157
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
76
|
-
|
|
158
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
159
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
160
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
161
|
+
deadLetterRoutingKey = orderingKey;
|
|
162
|
+
} else {
|
|
163
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
164
|
+
deadLetterExchange = "";
|
|
165
|
+
deadLetterRoutingKey = this.#queue;
|
|
166
|
+
}
|
|
77
167
|
await channel.assertQueue(queue, {
|
|
78
168
|
autoDelete: true,
|
|
79
169
|
durable: this.#durable,
|
|
80
|
-
deadLetterExchange
|
|
81
|
-
deadLetterRoutingKey
|
|
170
|
+
deadLetterExchange,
|
|
171
|
+
deadLetterRoutingKey,
|
|
82
172
|
messageTtl: delay
|
|
83
173
|
});
|
|
84
174
|
}
|
|
@@ -90,8 +180,9 @@ var AmqpMessageQueue = class {
|
|
|
90
180
|
async listen(handler, options = {}) {
|
|
91
181
|
const channel = await this.#connection.createChannel();
|
|
92
182
|
await this.#prepareQueue(channel);
|
|
183
|
+
await this.#prepareOrdering(channel);
|
|
93
184
|
await channel.prefetch(1);
|
|
94
|
-
const
|
|
185
|
+
const messageHandler = (msg) => {
|
|
95
186
|
if (msg == null) return;
|
|
96
187
|
const message = JSON.parse(msg.content.toString("utf-8"));
|
|
97
188
|
try {
|
|
@@ -104,13 +195,21 @@ var AmqpMessageQueue = class {
|
|
|
104
195
|
} finally {
|
|
105
196
|
if (!this.nativeRetrial) channel.ack(msg);
|
|
106
197
|
}
|
|
107
|
-
}
|
|
198
|
+
};
|
|
199
|
+
const consumerTags = [];
|
|
200
|
+
const reply = await channel.consume(this.#queue, messageHandler, { noAck: false });
|
|
201
|
+
consumerTags.push(reply.consumerTag);
|
|
202
|
+
if (this.#ordering != null) for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
203
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
204
|
+
const orderingReply = await channel.consume(queueName, messageHandler, { noAck: false });
|
|
205
|
+
consumerTags.push(orderingReply.consumerTag);
|
|
206
|
+
}
|
|
108
207
|
return await new Promise((resolve) => {
|
|
109
208
|
if (options.signal?.aborted) resolve();
|
|
110
|
-
options.signal?.addEventListener("abort", () => {
|
|
111
|
-
channel.cancel(
|
|
112
|
-
|
|
113
|
-
|
|
209
|
+
options.signal?.addEventListener("abort", async () => {
|
|
210
|
+
for (const tag of consumerTags) await channel.cancel(tag);
|
|
211
|
+
await channel.close();
|
|
212
|
+
resolve();
|
|
114
213
|
});
|
|
115
214
|
});
|
|
116
215
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fedify/amqp",
|
|
3
|
-
"version": "2.0.0-dev.
|
|
3
|
+
"version": "2.0.0-dev.279+ce1bdc22",
|
|
4
4
|
"description": "AMQP/RabbitMQ driver for Fedify",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"fedify",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
},
|
|
56
56
|
"peerDependencies": {
|
|
57
57
|
"amqplib": "^0.10.9",
|
|
58
|
-
"@fedify/fedify": "^2.0.0-dev.
|
|
58
|
+
"@fedify/fedify": "^2.0.0-dev.279+ce1bdc22"
|
|
59
59
|
},
|
|
60
60
|
"devDependencies": {
|
|
61
61
|
"@alinea/suite": "^0.6.3",
|
|
@@ -65,13 +65,16 @@
|
|
|
65
65
|
"@types/amqplib": "^0.10.7",
|
|
66
66
|
"tsdown": "^0.12.9",
|
|
67
67
|
"typescript": "^5.9.3",
|
|
68
|
-
"@fedify/testing": "^2.0.0-dev.
|
|
68
|
+
"@fedify/testing": "^2.0.0-dev.279+ce1bdc22"
|
|
69
69
|
},
|
|
70
70
|
"scripts": {
|
|
71
|
-
"build": "tsdown",
|
|
72
|
-
"
|
|
73
|
-
"
|
|
74
|
-
"
|
|
71
|
+
"build:self": "tsdown",
|
|
72
|
+
"build": "pnpm --filter @fedify/amqp... run build:self",
|
|
73
|
+
"prepublish": "pnpm build",
|
|
74
|
+
"pretest": "pnpm build",
|
|
75
|
+
"test": "node --experimental-transform-types --test",
|
|
76
|
+
"pretest:bun": "pnpm build",
|
|
77
|
+
"test:bun": "bun test --timeout 15000",
|
|
75
78
|
"test:deno": "deno task test",
|
|
76
79
|
"test-all": "tsdown && node --experimental-transform-types --test && bun test --timeout 15000 && deno task test"
|
|
77
80
|
}
|
package/src/mq.test.ts
CHANGED
|
@@ -37,6 +37,47 @@ test(
|
|
|
37
37
|
),
|
|
38
38
|
);
|
|
39
39
|
|
|
40
|
+
// Test with ordering key support (requires rabbitmq_consistent_hash_exchange plugin)
|
|
41
|
+
const orderingConnections: ChannelModel[] = [];
|
|
42
|
+
const orderingQueue = getRandomKey("ordering_queue");
|
|
43
|
+
const orderingDelayedQueuePrefix = getRandomKey("ordering_delayed") + "_";
|
|
44
|
+
const orderingExchange = getRandomKey("ordering_exchange");
|
|
45
|
+
const orderingQueuePrefix = getRandomKey("ordering_partition") + "_";
|
|
46
|
+
|
|
47
|
+
// Only run ordering key tests if AMQP_ORDERING_TEST env var is set
|
|
48
|
+
// (requires rabbitmq_consistent_hash_exchange plugin to be enabled)
|
|
49
|
+
const orderingTest = process.env.AMQP_ORDERING_TEST
|
|
50
|
+
? test
|
|
51
|
+
: suite(import.meta).skip;
|
|
52
|
+
|
|
53
|
+
orderingTest(
|
|
54
|
+
"AmqpMessageQueue [ordering]",
|
|
55
|
+
{ sanitizeOps: false, sanitizeExit: false, sanitizeResources: false },
|
|
56
|
+
() =>
|
|
57
|
+
testMessageQueue(
|
|
58
|
+
async () => {
|
|
59
|
+
const conn = await getConnection();
|
|
60
|
+
orderingConnections.push(conn);
|
|
61
|
+
return new AmqpMessageQueue(conn, {
|
|
62
|
+
queue: orderingQueue,
|
|
63
|
+
delayedQueuePrefix: orderingDelayedQueuePrefix,
|
|
64
|
+
ordering: {
|
|
65
|
+
exchange: orderingExchange,
|
|
66
|
+
queuePrefix: orderingQueuePrefix,
|
|
67
|
+
partitions: 4,
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
},
|
|
71
|
+
async ({ controller }) => {
|
|
72
|
+
controller.abort();
|
|
73
|
+
for (const conn of orderingConnections) {
|
|
74
|
+
await conn.close();
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
{ testOrderingKey: true },
|
|
78
|
+
),
|
|
79
|
+
);
|
|
80
|
+
|
|
40
81
|
test(
|
|
41
82
|
"AmqpMessageQueue [nativeRetrial: false]",
|
|
42
83
|
{ sanitizeOps: false, sanitizeExit: false, sanitizeResources: false },
|
package/src/mq.ts
CHANGED
|
@@ -4,9 +4,45 @@ import type {
|
|
|
4
4
|
MessageQueueListenOptions,
|
|
5
5
|
} from "@fedify/fedify";
|
|
6
6
|
// @deno-types="npm:@types/amqplib@^0.10.7"
|
|
7
|
-
import type { Channel, ChannelModel } from "amqplib";
|
|
7
|
+
import type { Channel, ChannelModel, ConsumeMessage } from "amqplib";
|
|
8
8
|
import { Buffer } from "node:buffer";
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Options for ordering key support in {@link AmqpMessageQueue}.
|
|
12
|
+
*
|
|
13
|
+
* Ordering key support requires the `rabbitmq_consistent_hash_exchange`
|
|
14
|
+
* plugin to be enabled on the RabbitMQ server. You can enable it by running:
|
|
15
|
+
*
|
|
16
|
+
* ```sh
|
|
17
|
+
* rabbitmq-plugins enable rabbitmq_consistent_hash_exchange
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* @since 2.0.0
|
|
21
|
+
*/
|
|
22
|
+
export interface AmqpOrderingOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The name of the consistent hash exchange to use for ordering.
|
|
25
|
+
* Defaults to `"fedify_ordering"`.
|
|
26
|
+
* @default `"fedify_ordering"`
|
|
27
|
+
*/
|
|
28
|
+
readonly exchange?: string;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The prefix to use for ordering queues.
|
|
32
|
+
* Defaults to `"fedify_ordering_"`.
|
|
33
|
+
* @default `"fedify_ordering_"`
|
|
34
|
+
*/
|
|
35
|
+
readonly queuePrefix?: string;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The number of partitions (queues) to use for ordering.
|
|
39
|
+
* More partitions allow better parallelism for different ordering keys.
|
|
40
|
+
* Defaults to `4`.
|
|
41
|
+
* @default `4`
|
|
42
|
+
*/
|
|
43
|
+
readonly partitions?: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
10
46
|
/**
|
|
11
47
|
* Options for {@link AmqpMessageQueue}.
|
|
12
48
|
*/
|
|
@@ -46,6 +82,23 @@ export interface AmqpMessageQueueOptions {
|
|
|
46
82
|
* @since 0.3.0
|
|
47
83
|
*/
|
|
48
84
|
readonly nativeRetrial?: boolean;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Options for ordering key support. If provided, the message queue will
|
|
88
|
+
* support the `orderingKey` option in {@link MessageQueueEnqueueOptions}.
|
|
89
|
+
* Messages with the same ordering key will be processed in order,
|
|
90
|
+
* while messages with different ordering keys can be processed in parallel.
|
|
91
|
+
*
|
|
92
|
+
* This feature requires the `rabbitmq_consistent_hash_exchange` plugin
|
|
93
|
+
* to be enabled on the RabbitMQ server. See {@link AmqpOrderingOptions}
|
|
94
|
+
* for more details.
|
|
95
|
+
*
|
|
96
|
+
* If not provided, ordering key support is disabled and any `orderingKey`
|
|
97
|
+
* option passed to `enqueue()` will be ignored.
|
|
98
|
+
*
|
|
99
|
+
* @since 0.4.0
|
|
100
|
+
*/
|
|
101
|
+
readonly ordering?: AmqpOrderingOptions;
|
|
49
102
|
}
|
|
50
103
|
|
|
51
104
|
/**
|
|
@@ -69,6 +122,12 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
69
122
|
#delayedQueuePrefix: string;
|
|
70
123
|
#durable: boolean;
|
|
71
124
|
#senderChannel?: Channel;
|
|
125
|
+
#ordering?: {
|
|
126
|
+
exchange: string;
|
|
127
|
+
queuePrefix: string;
|
|
128
|
+
partitions: number;
|
|
129
|
+
};
|
|
130
|
+
#orderingPrepared: boolean = false;
|
|
72
131
|
|
|
73
132
|
readonly nativeRetrial: boolean;
|
|
74
133
|
|
|
@@ -86,6 +145,13 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
86
145
|
this.#delayedQueuePrefix = options.delayedQueuePrefix ?? "fedify_delayed_";
|
|
87
146
|
this.#durable = options.durable ?? true;
|
|
88
147
|
this.nativeRetrial = options.nativeRetrial ?? false;
|
|
148
|
+
if (options.ordering != null) {
|
|
149
|
+
this.#ordering = {
|
|
150
|
+
exchange: options.ordering.exchange ?? "fedify_ordering",
|
|
151
|
+
queuePrefix: options.ordering.queuePrefix ?? "fedify_ordering_",
|
|
152
|
+
partitions: options.ordering.partitions ?? 4,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
89
155
|
}
|
|
90
156
|
|
|
91
157
|
async #prepareQueue(channel: Channel): Promise<void> {
|
|
@@ -94,14 +160,55 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
94
160
|
});
|
|
95
161
|
}
|
|
96
162
|
|
|
163
|
+
#getOrderingQueueName(partition: number): string {
|
|
164
|
+
return `${this.#ordering!.queuePrefix}${partition}`;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async #prepareOrdering(channel: Channel): Promise<void> {
|
|
168
|
+
if (this.#ordering == null || this.#orderingPrepared) return;
|
|
169
|
+
// Declare the consistent hash exchange
|
|
170
|
+
await channel.assertExchange(this.#ordering.exchange, "x-consistent-hash", {
|
|
171
|
+
durable: this.#durable,
|
|
172
|
+
});
|
|
173
|
+
// Declare and bind ordering queues with Single Active Consumer
|
|
174
|
+
for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
175
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
176
|
+
await channel.assertQueue(queueName, {
|
|
177
|
+
durable: this.#durable,
|
|
178
|
+
arguments: {
|
|
179
|
+
"x-single-active-consumer": true,
|
|
180
|
+
},
|
|
181
|
+
});
|
|
182
|
+
// Bind with weight "1" (equal distribution)
|
|
183
|
+
await channel.bindQueue(queueName, this.#ordering.exchange, "1");
|
|
184
|
+
}
|
|
185
|
+
this.#orderingPrepared = true;
|
|
186
|
+
}
|
|
187
|
+
|
|
97
188
|
async #getSenderChannel(): Promise<Channel> {
|
|
98
189
|
if (this.#senderChannel != null) return this.#senderChannel;
|
|
99
190
|
const channel = await this.#connection.createChannel();
|
|
100
191
|
this.#senderChannel = channel;
|
|
101
|
-
this.#prepareQueue(channel);
|
|
192
|
+
await this.#prepareQueue(channel);
|
|
193
|
+
await this.#prepareOrdering(channel);
|
|
102
194
|
return channel;
|
|
103
195
|
}
|
|
104
196
|
|
|
197
|
+
/**
|
|
198
|
+
* Enqueues a message to be processed.
|
|
199
|
+
*
|
|
200
|
+
* When an `orderingKey` is provided without a `delay`, the message is routed
|
|
201
|
+
* through the consistent hash exchange, ensuring messages with the same
|
|
202
|
+
* ordering key are processed by the same consumer in FIFO order.
|
|
203
|
+
*
|
|
204
|
+
* When both `orderingKey` and `delay` are provided, the message is first
|
|
205
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
206
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
207
|
+
* delayed messages.
|
|
208
|
+
*
|
|
209
|
+
* @param message The message to enqueue.
|
|
210
|
+
* @param options The options for enqueueing the message.
|
|
211
|
+
*/
|
|
105
212
|
async enqueue(
|
|
106
213
|
// deno-lint-ignore no-explicit-any
|
|
107
214
|
message: any,
|
|
@@ -109,17 +216,51 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
109
216
|
): Promise<void> {
|
|
110
217
|
const channel = await this.#getSenderChannel();
|
|
111
218
|
const delay = options?.delay?.total("millisecond");
|
|
219
|
+
const orderingKey = options?.orderingKey;
|
|
220
|
+
|
|
221
|
+
// If ordering key is provided and ordering is enabled, use consistent hash
|
|
222
|
+
// Treat delay <= 0 the same as no delay for routing purposes
|
|
223
|
+
if (
|
|
224
|
+
orderingKey != null &&
|
|
225
|
+
this.#ordering != null &&
|
|
226
|
+
(delay == null || delay <= 0)
|
|
227
|
+
) {
|
|
228
|
+
channel.publish(
|
|
229
|
+
this.#ordering.exchange,
|
|
230
|
+
orderingKey, // routing key = ordering key
|
|
231
|
+
Buffer.from(JSON.stringify(message), "utf-8"),
|
|
232
|
+
{
|
|
233
|
+
persistent: this.#durable,
|
|
234
|
+
contentType: "application/json",
|
|
235
|
+
},
|
|
236
|
+
);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// For delayed messages or messages without ordering key, use direct queue
|
|
112
241
|
let queue: string;
|
|
242
|
+
let deadLetterExchange: string | undefined;
|
|
243
|
+
let deadLetterRoutingKey: string | undefined;
|
|
244
|
+
|
|
113
245
|
if (delay == null || delay <= 0) {
|
|
114
246
|
queue = this.#queue;
|
|
115
247
|
} else {
|
|
116
248
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
117
|
-
|
|
249
|
+
// For delayed messages with ordering key, route to ordering exchange
|
|
250
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
251
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
252
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
253
|
+
deadLetterRoutingKey = orderingKey;
|
|
254
|
+
} else {
|
|
255
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
256
|
+
deadLetterExchange = "";
|
|
257
|
+
deadLetterRoutingKey = this.#queue;
|
|
258
|
+
}
|
|
118
259
|
await channel.assertQueue(queue, {
|
|
119
260
|
autoDelete: true,
|
|
120
261
|
durable: this.#durable,
|
|
121
|
-
deadLetterExchange
|
|
122
|
-
deadLetterRoutingKey
|
|
262
|
+
deadLetterExchange,
|
|
263
|
+
deadLetterRoutingKey,
|
|
123
264
|
messageTtl: delay,
|
|
124
265
|
});
|
|
125
266
|
}
|
|
@@ -133,6 +274,21 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
133
274
|
);
|
|
134
275
|
}
|
|
135
276
|
|
|
277
|
+
/**
|
|
278
|
+
* Enqueues multiple messages to be processed.
|
|
279
|
+
*
|
|
280
|
+
* When an `orderingKey` is provided without a `delay`, the messages are
|
|
281
|
+
* routed through the consistent hash exchange, ensuring messages with the
|
|
282
|
+
* same ordering key are processed by the same consumer in FIFO order.
|
|
283
|
+
*
|
|
284
|
+
* When both `orderingKey` and `delay` are provided, the messages are first
|
|
285
|
+
* placed in a delay queue, then routed to the consistent hash exchange
|
|
286
|
+
* after the delay expires. This ensures ordering is preserved even for
|
|
287
|
+
* delayed messages.
|
|
288
|
+
*
|
|
289
|
+
* @param messages The messages to enqueue.
|
|
290
|
+
* @param options The options for enqueueing the messages.
|
|
291
|
+
*/
|
|
136
292
|
async enqueueMany(
|
|
137
293
|
// deno-lint-ignore no-explicit-any
|
|
138
294
|
messages: readonly any[],
|
|
@@ -140,17 +296,53 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
140
296
|
): Promise<void> {
|
|
141
297
|
const channel = await this.#getSenderChannel();
|
|
142
298
|
const delay = options?.delay?.total("millisecond");
|
|
299
|
+
const orderingKey = options?.orderingKey;
|
|
300
|
+
|
|
301
|
+
// If ordering key is provided and ordering is enabled, use consistent hash
|
|
302
|
+
// Treat delay <= 0 the same as no delay for routing purposes
|
|
303
|
+
if (
|
|
304
|
+
orderingKey != null &&
|
|
305
|
+
this.#ordering != null &&
|
|
306
|
+
(delay == null || delay <= 0)
|
|
307
|
+
) {
|
|
308
|
+
for (const message of messages) {
|
|
309
|
+
channel.publish(
|
|
310
|
+
this.#ordering.exchange,
|
|
311
|
+
orderingKey, // routing key = ordering key
|
|
312
|
+
Buffer.from(JSON.stringify(message), "utf-8"),
|
|
313
|
+
{
|
|
314
|
+
persistent: this.#durable,
|
|
315
|
+
contentType: "application/json",
|
|
316
|
+
},
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// For delayed messages or messages without ordering key, use direct queue
|
|
143
323
|
let queue: string;
|
|
324
|
+
let deadLetterExchange: string | undefined;
|
|
325
|
+
let deadLetterRoutingKey: string | undefined;
|
|
326
|
+
|
|
144
327
|
if (delay == null || delay <= 0) {
|
|
145
328
|
queue = this.#queue;
|
|
146
329
|
} else {
|
|
147
330
|
const delayStr = delay.toLocaleString("en", { useGrouping: false });
|
|
148
|
-
|
|
331
|
+
// For delayed messages with ordering key, route to ordering exchange
|
|
332
|
+
if (orderingKey != null && this.#ordering != null) {
|
|
333
|
+
queue = `${this.#delayedQueuePrefix}ordering_${delayStr}`;
|
|
334
|
+
deadLetterExchange = this.#ordering.exchange;
|
|
335
|
+
deadLetterRoutingKey = orderingKey;
|
|
336
|
+
} else {
|
|
337
|
+
queue = this.#delayedQueuePrefix + delayStr;
|
|
338
|
+
deadLetterExchange = "";
|
|
339
|
+
deadLetterRoutingKey = this.#queue;
|
|
340
|
+
}
|
|
149
341
|
await channel.assertQueue(queue, {
|
|
150
342
|
autoDelete: true,
|
|
151
343
|
durable: this.#durable,
|
|
152
|
-
deadLetterExchange
|
|
153
|
-
deadLetterRoutingKey
|
|
344
|
+
deadLetterExchange,
|
|
345
|
+
deadLetterRoutingKey,
|
|
154
346
|
messageTtl: delay,
|
|
155
347
|
});
|
|
156
348
|
}
|
|
@@ -174,8 +366,10 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
174
366
|
): Promise<void> {
|
|
175
367
|
const channel = await this.#connection.createChannel();
|
|
176
368
|
await this.#prepareQueue(channel);
|
|
369
|
+
await this.#prepareOrdering(channel);
|
|
177
370
|
await channel.prefetch(1);
|
|
178
|
-
|
|
371
|
+
|
|
372
|
+
const messageHandler = (msg: ConsumeMessage | null) => {
|
|
179
373
|
if (msg == null) return;
|
|
180
374
|
const message = JSON.parse(msg.content.toString("utf-8"));
|
|
181
375
|
try {
|
|
@@ -200,15 +394,37 @@ export class AmqpMessageQueue implements MessageQueue {
|
|
|
200
394
|
channel.ack(msg);
|
|
201
395
|
}
|
|
202
396
|
}
|
|
203
|
-
}
|
|
397
|
+
};
|
|
398
|
+
|
|
399
|
+
// Consume from main queue
|
|
400
|
+
const consumerTags: string[] = [];
|
|
401
|
+
const reply = await channel.consume(this.#queue, messageHandler, {
|
|
204
402
|
noAck: false,
|
|
205
403
|
});
|
|
404
|
+
consumerTags.push(reply.consumerTag);
|
|
405
|
+
|
|
406
|
+
// Also consume from ordering queues if ordering is enabled
|
|
407
|
+
if (this.#ordering != null) {
|
|
408
|
+
for (let i = 0; i < this.#ordering.partitions; i++) {
|
|
409
|
+
const queueName = this.#getOrderingQueueName(i);
|
|
410
|
+
const orderingReply = await channel.consume(
|
|
411
|
+
queueName,
|
|
412
|
+
messageHandler,
|
|
413
|
+
{ noAck: false },
|
|
414
|
+
);
|
|
415
|
+
consumerTags.push(orderingReply.consumerTag);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
206
419
|
return await new Promise((resolve) => {
|
|
207
420
|
if (options.signal?.aborted) resolve();
|
|
208
|
-
options.signal?.addEventListener("abort", () => {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
421
|
+
options.signal?.addEventListener("abort", async () => {
|
|
422
|
+
// Cancel all consumers
|
|
423
|
+
for (const tag of consumerTags) {
|
|
424
|
+
await channel.cancel(tag);
|
|
425
|
+
}
|
|
426
|
+
await channel.close();
|
|
427
|
+
resolve();
|
|
212
428
|
});
|
|
213
429
|
});
|
|
214
430
|
}
|