@fedify/amqp 2.3.0-dev.1004 → 2.3.0-dev.1013

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/deno.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fedify/amqp",
3
- "version": "2.3.0-dev.1004+c312f0ba",
3
+ "version": "2.3.0-dev.1013+4aa6a88c",
4
4
  "license": "MIT",
5
5
  "exports": {
6
6
  ".": "./src/mod.ts",
package/dist/mq.cjs CHANGED
@@ -1,6 +1,15 @@
1
1
  Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
2
2
  let node_buffer = require("node:buffer");
3
3
  //#region src/mq.ts
4
+ function isQueueNotFoundError(error) {
5
+ return typeof error === "object" && error != null && "code" in error && error.code === 404;
6
+ }
7
+ function isPreconditionFailedError(error) {
8
+ return typeof error === "object" && error != null && "code" in error && error.code === 406;
9
+ }
10
+ const depthProbeConcurrency = 8;
11
+ const delayedQueueExpiryMargin = 6e4;
12
+ const delayedQueueCleanupThreshold = 4096;
4
13
  /**
5
14
  * A message queue that uses AMQP.
6
15
  *
@@ -23,6 +32,8 @@ var AmqpMessageQueue = class {
23
32
  #durable;
24
33
  #senderChannel;
25
34
  #ordering;
35
+ #delayedQueues = /* @__PURE__ */ new Set();
36
+ #delayedQueueCleanup;
26
37
  #orderingPrepared = false;
27
38
  nativeRetrial;
28
39
  /**
@@ -69,6 +80,12 @@ var AmqpMessageQueue = class {
69
80
  await this.#prepareOrdering(channel);
70
81
  return channel;
71
82
  }
83
+ async #dropSenderChannel(channel) {
84
+ if (this.#senderChannel === channel) this.#senderChannel = void 0;
85
+ try {
86
+ await channel.close();
87
+ } catch {}
88
+ }
72
89
  /**
73
90
  * Enqueues a message to be processed.
74
91
  *
@@ -85,7 +102,7 @@ var AmqpMessageQueue = class {
85
102
  * @param options The options for enqueueing the message.
86
103
  */
87
104
  async enqueue(message, options) {
88
- const channel = await this.#getSenderChannel();
105
+ let channel = await this.#getSenderChannel();
89
106
  const delay = options?.delay?.total("millisecond");
90
107
  const orderingKey = options?.orderingKey;
91
108
  if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
@@ -110,13 +127,12 @@ var AmqpMessageQueue = class {
110
127
  deadLetterExchange = "";
111
128
  deadLetterRoutingKey = this.#queue;
112
129
  }
113
- await channel.assertQueue(queue, {
114
- autoDelete: true,
115
- durable: this.#durable,
130
+ channel = await this.#assertDelayedQueue(channel, queue, {
116
131
  deadLetterExchange,
117
132
  deadLetterRoutingKey,
118
- messageTtl: delay
133
+ delay
119
134
  });
135
+ this.#trackDelayedQueue(queue);
120
136
  }
121
137
  channel.sendToQueue(queue, node_buffer.Buffer.from(JSON.stringify(message), "utf-8"), {
122
138
  persistent: this.#durable,
@@ -139,7 +155,7 @@ var AmqpMessageQueue = class {
139
155
  * @param options The options for enqueueing the messages.
140
156
  */
141
157
  async enqueueMany(messages, options) {
142
- const channel = await this.#getSenderChannel();
158
+ let channel = await this.#getSenderChannel();
143
159
  const delay = options?.delay?.total("millisecond");
144
160
  const orderingKey = options?.orderingKey;
145
161
  if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
@@ -164,19 +180,107 @@ var AmqpMessageQueue = class {
164
180
  deadLetterExchange = "";
165
181
  deadLetterRoutingKey = this.#queue;
166
182
  }
167
- await channel.assertQueue(queue, {
168
- autoDelete: true,
169
- durable: this.#durable,
183
+ channel = await this.#assertDelayedQueue(channel, queue, {
170
184
  deadLetterExchange,
171
185
  deadLetterRoutingKey,
172
- messageTtl: delay
186
+ delay
173
187
  });
188
+ this.#trackDelayedQueue(queue);
174
189
  }
175
190
  for (const message of messages) channel.sendToQueue(queue, node_buffer.Buffer.from(JSON.stringify(message), "utf-8"), {
176
191
  persistent: this.#durable,
177
192
  contentType: "application/json"
178
193
  });
179
194
  }
195
+ #trackDelayedQueue(queue) {
196
+ this.#delayedQueues.add(queue);
197
+ if (this.#delayedQueues.size > delayedQueueCleanupThreshold && this.#delayedQueueCleanup == null) this.#delayedQueueCleanup = this.#pruneMissingDelayedQueues().catch(() => void 0).finally(() => {
198
+ this.#delayedQueueCleanup = void 0;
199
+ });
200
+ }
201
+ async #assertDelayedQueue(channel, queue, options) {
202
+ const assertOptions = {
203
+ autoDelete: true,
204
+ durable: this.#durable,
205
+ deadLetterExchange: options.deadLetterExchange,
206
+ deadLetterRoutingKey: options.deadLetterRoutingKey,
207
+ messageTtl: options.delay
208
+ };
209
+ try {
210
+ await channel.assertQueue(queue, {
211
+ ...assertOptions,
212
+ expires: options.delay + delayedQueueExpiryMargin
213
+ });
214
+ return channel;
215
+ } catch (error) {
216
+ if (!isPreconditionFailedError(error)) throw error;
217
+ await this.#dropSenderChannel(channel);
218
+ }
219
+ const fallbackChannel = await this.#getSenderChannel();
220
+ await fallbackChannel.assertQueue(queue, assertOptions);
221
+ return fallbackChannel;
222
+ }
223
+ async #createDepthChannel() {
224
+ const channel = await this.#connection.createChannel();
225
+ channel.on("error", () => void 0);
226
+ return channel;
227
+ }
228
+ async #checkQueueDepths(queueNames) {
229
+ const results = new Array(queueNames.length);
230
+ let nextIndex = 0;
231
+ const worker = async () => {
232
+ let channel;
233
+ const closeChannel = async () => {
234
+ if (channel == null) return;
235
+ const currentChannel = channel;
236
+ channel = void 0;
237
+ try {
238
+ await currentChannel.close();
239
+ } catch {}
240
+ };
241
+ const checkQueue = async (queue) => {
242
+ channel ??= await this.#createDepthChannel();
243
+ try {
244
+ return (await channel.checkQueue(queue)).messageCount;
245
+ } catch (error) {
246
+ await closeChannel();
247
+ if (!isQueueNotFoundError(error)) throw error;
248
+ return;
249
+ }
250
+ };
251
+ try {
252
+ while (nextIndex < queueNames.length) {
253
+ const index = nextIndex++;
254
+ const queue = queueNames[index];
255
+ results[index] = [queue, await checkQueue(queue)];
256
+ }
257
+ } finally {
258
+ await closeChannel();
259
+ }
260
+ };
261
+ const workers = Array.from({ length: Math.min(depthProbeConcurrency, queueNames.length) }, () => worker());
262
+ await Promise.all(workers);
263
+ return results;
264
+ }
265
+ async #pruneMissingDelayedQueues() {
266
+ const delayedQueues = [...this.#delayedQueues];
267
+ for (const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)) if (messageCount == null) this.#delayedQueues.delete(queue);
268
+ }
269
+ async getDepth() {
270
+ const readyQueues = [this.#queue];
271
+ if (this.#ordering != null) for (let i = 0; i < this.#ordering.partitions; i++) readyQueues.push(this.#getOrderingQueueName(i));
272
+ let ready = 0;
273
+ for (const [, messageCount] of await this.#checkQueueDepths(readyQueues)) ready += messageCount ?? 0;
274
+ let delayed = 0;
275
+ const delayedQueues = [...this.#delayedQueues];
276
+ for (const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)) if (messageCount == null) this.#delayedQueues.delete(queue);
277
+ else delayed += messageCount;
278
+ return {
279
+ queued: ready + delayed,
280
+ ready,
281
+ delayed
282
+ };
283
+ }
180
284
  async listen(handler, options = {}) {
181
285
  const channel = await this.#connection.createChannel();
182
286
  await this.#prepareQueue(channel);
package/dist/mq.d.cts CHANGED
@@ -1,4 +1,4 @@
1
- import { MessageQueue, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
1
+ import { MessageQueue, MessageQueueDepth, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
2
2
  import { ChannelModel } from "amqplib";
3
3
 
4
4
  //#region src/mq.d.ts
@@ -144,6 +144,7 @@ declare class AmqpMessageQueue implements MessageQueue {
144
144
  * @param options The options for enqueueing the messages.
145
145
  */
146
146
  enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
147
+ getDepth(): Promise<MessageQueueDepth>;
147
148
  listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
148
149
  }
149
150
  //#endregion
package/dist/mq.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { MessageQueue, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
1
+ import { MessageQueue, MessageQueueDepth, MessageQueueEnqueueOptions, MessageQueueListenOptions } from "@fedify/fedify";
2
2
  import { ChannelModel } from "amqplib";
3
3
 
4
4
  //#region src/mq.d.ts
@@ -144,6 +144,7 @@ declare class AmqpMessageQueue implements MessageQueue {
144
144
  * @param options The options for enqueueing the messages.
145
145
  */
146
146
  enqueueMany(messages: readonly any[], options?: MessageQueueEnqueueOptions): Promise<void>;
147
+ getDepth(): Promise<MessageQueueDepth>;
147
148
  listen(handler: (message: any) => void | Promise<void>, options?: MessageQueueListenOptions): Promise<void>;
148
149
  }
149
150
  //#endregion
package/dist/mq.js CHANGED
@@ -1,5 +1,14 @@
1
1
  import { Buffer } from "node:buffer";
2
2
  //#region src/mq.ts
3
+ function isQueueNotFoundError(error) {
4
+ return typeof error === "object" && error != null && "code" in error && error.code === 404;
5
+ }
6
+ function isPreconditionFailedError(error) {
7
+ return typeof error === "object" && error != null && "code" in error && error.code === 406;
8
+ }
9
+ const depthProbeConcurrency = 8;
10
+ const delayedQueueExpiryMargin = 6e4;
11
+ const delayedQueueCleanupThreshold = 4096;
3
12
  /**
4
13
  * A message queue that uses AMQP.
5
14
  *
@@ -22,6 +31,8 @@ var AmqpMessageQueue = class {
22
31
  #durable;
23
32
  #senderChannel;
24
33
  #ordering;
34
+ #delayedQueues = /* @__PURE__ */ new Set();
35
+ #delayedQueueCleanup;
25
36
  #orderingPrepared = false;
26
37
  nativeRetrial;
27
38
  /**
@@ -68,6 +79,12 @@ var AmqpMessageQueue = class {
68
79
  await this.#prepareOrdering(channel);
69
80
  return channel;
70
81
  }
82
+ async #dropSenderChannel(channel) {
83
+ if (this.#senderChannel === channel) this.#senderChannel = void 0;
84
+ try {
85
+ await channel.close();
86
+ } catch {}
87
+ }
71
88
  /**
72
89
  * Enqueues a message to be processed.
73
90
  *
@@ -84,7 +101,7 @@ var AmqpMessageQueue = class {
84
101
  * @param options The options for enqueueing the message.
85
102
  */
86
103
  async enqueue(message, options) {
87
- const channel = await this.#getSenderChannel();
104
+ let channel = await this.#getSenderChannel();
88
105
  const delay = options?.delay?.total("millisecond");
89
106
  const orderingKey = options?.orderingKey;
90
107
  if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
@@ -109,13 +126,12 @@ var AmqpMessageQueue = class {
109
126
  deadLetterExchange = "";
110
127
  deadLetterRoutingKey = this.#queue;
111
128
  }
112
- await channel.assertQueue(queue, {
113
- autoDelete: true,
114
- durable: this.#durable,
129
+ channel = await this.#assertDelayedQueue(channel, queue, {
115
130
  deadLetterExchange,
116
131
  deadLetterRoutingKey,
117
- messageTtl: delay
132
+ delay
118
133
  });
134
+ this.#trackDelayedQueue(queue);
119
135
  }
120
136
  channel.sendToQueue(queue, Buffer.from(JSON.stringify(message), "utf-8"), {
121
137
  persistent: this.#durable,
@@ -138,7 +154,7 @@ var AmqpMessageQueue = class {
138
154
  * @param options The options for enqueueing the messages.
139
155
  */
140
156
  async enqueueMany(messages, options) {
141
- const channel = await this.#getSenderChannel();
157
+ let channel = await this.#getSenderChannel();
142
158
  const delay = options?.delay?.total("millisecond");
143
159
  const orderingKey = options?.orderingKey;
144
160
  if (orderingKey != null && this.#ordering != null && (delay == null || delay <= 0)) {
@@ -163,19 +179,107 @@ var AmqpMessageQueue = class {
163
179
  deadLetterExchange = "";
164
180
  deadLetterRoutingKey = this.#queue;
165
181
  }
166
- await channel.assertQueue(queue, {
167
- autoDelete: true,
168
- durable: this.#durable,
182
+ channel = await this.#assertDelayedQueue(channel, queue, {
169
183
  deadLetterExchange,
170
184
  deadLetterRoutingKey,
171
- messageTtl: delay
185
+ delay
172
186
  });
187
+ this.#trackDelayedQueue(queue);
173
188
  }
174
189
  for (const message of messages) channel.sendToQueue(queue, Buffer.from(JSON.stringify(message), "utf-8"), {
175
190
  persistent: this.#durable,
176
191
  contentType: "application/json"
177
192
  });
178
193
  }
194
+ #trackDelayedQueue(queue) {
195
+ this.#delayedQueues.add(queue);
196
+ if (this.#delayedQueues.size > delayedQueueCleanupThreshold && this.#delayedQueueCleanup == null) this.#delayedQueueCleanup = this.#pruneMissingDelayedQueues().catch(() => void 0).finally(() => {
197
+ this.#delayedQueueCleanup = void 0;
198
+ });
199
+ }
200
+ async #assertDelayedQueue(channel, queue, options) {
201
+ const assertOptions = {
202
+ autoDelete: true,
203
+ durable: this.#durable,
204
+ deadLetterExchange: options.deadLetterExchange,
205
+ deadLetterRoutingKey: options.deadLetterRoutingKey,
206
+ messageTtl: options.delay
207
+ };
208
+ try {
209
+ await channel.assertQueue(queue, {
210
+ ...assertOptions,
211
+ expires: options.delay + delayedQueueExpiryMargin
212
+ });
213
+ return channel;
214
+ } catch (error) {
215
+ if (!isPreconditionFailedError(error)) throw error;
216
+ await this.#dropSenderChannel(channel);
217
+ }
218
+ const fallbackChannel = await this.#getSenderChannel();
219
+ await fallbackChannel.assertQueue(queue, assertOptions);
220
+ return fallbackChannel;
221
+ }
222
+ async #createDepthChannel() {
223
+ const channel = await this.#connection.createChannel();
224
+ channel.on("error", () => void 0);
225
+ return channel;
226
+ }
227
+ async #checkQueueDepths(queueNames) {
228
+ const results = new Array(queueNames.length);
229
+ let nextIndex = 0;
230
+ const worker = async () => {
231
+ let channel;
232
+ const closeChannel = async () => {
233
+ if (channel == null) return;
234
+ const currentChannel = channel;
235
+ channel = void 0;
236
+ try {
237
+ await currentChannel.close();
238
+ } catch {}
239
+ };
240
+ const checkQueue = async (queue) => {
241
+ channel ??= await this.#createDepthChannel();
242
+ try {
243
+ return (await channel.checkQueue(queue)).messageCount;
244
+ } catch (error) {
245
+ await closeChannel();
246
+ if (!isQueueNotFoundError(error)) throw error;
247
+ return;
248
+ }
249
+ };
250
+ try {
251
+ while (nextIndex < queueNames.length) {
252
+ const index = nextIndex++;
253
+ const queue = queueNames[index];
254
+ results[index] = [queue, await checkQueue(queue)];
255
+ }
256
+ } finally {
257
+ await closeChannel();
258
+ }
259
+ };
260
+ const workers = Array.from({ length: Math.min(depthProbeConcurrency, queueNames.length) }, () => worker());
261
+ await Promise.all(workers);
262
+ return results;
263
+ }
264
+ async #pruneMissingDelayedQueues() {
265
+ const delayedQueues = [...this.#delayedQueues];
266
+ for (const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)) if (messageCount == null) this.#delayedQueues.delete(queue);
267
+ }
268
+ async getDepth() {
269
+ const readyQueues = [this.#queue];
270
+ if (this.#ordering != null) for (let i = 0; i < this.#ordering.partitions; i++) readyQueues.push(this.#getOrderingQueueName(i));
271
+ let ready = 0;
272
+ for (const [, messageCount] of await this.#checkQueueDepths(readyQueues)) ready += messageCount ?? 0;
273
+ let delayed = 0;
274
+ const delayedQueues = [...this.#delayedQueues];
275
+ for (const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)) if (messageCount == null) this.#delayedQueues.delete(queue);
276
+ else delayed += messageCount;
277
+ return {
278
+ queued: ready + delayed,
279
+ ready,
280
+ delayed
281
+ };
282
+ }
179
283
  async listen(handler, options = {}) {
180
284
  const channel = await this.#connection.createChannel();
181
285
  await this.#prepareQueue(channel);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fedify/amqp",
3
- "version": "2.3.0-dev.1004+c312f0ba",
3
+ "version": "2.3.0-dev.1013+4aa6a88c",
4
4
  "description": "AMQP/RabbitMQ driver for Fedify",
5
5
  "keywords": [
6
6
  "fedify",
@@ -55,7 +55,7 @@
55
55
  },
56
56
  "peerDependencies": {
57
57
  "amqplib": "^0.10.9",
58
- "@fedify/fedify": "^2.3.0-dev.1004+c312f0ba"
58
+ "@fedify/fedify": "^2.3.0-dev.1013+4aa6a88c"
59
59
  },
60
60
  "devDependencies": {
61
61
  "@alinea/suite": "^0.6.3",
@@ -65,7 +65,7 @@
65
65
  "@types/amqplib": "^0.10.7",
66
66
  "tsdown": "^0.21.6",
67
67
  "typescript": "^5.9.2",
68
- "@fedify/testing": "^2.3.0-dev.1004+c312f0ba"
68
+ "@fedify/testing": "^2.3.0-dev.1013+4aa6a88c"
69
69
  },
70
70
  "scripts": {
71
71
  "build:self": "tsdown",
package/src/mq.test.ts CHANGED
@@ -1,14 +1,240 @@
1
1
  import { suite } from "@alinea/suite";
2
2
  import { AmqpMessageQueue } from "@fedify/amqp/mq";
3
3
  import { getRandomKey, testMessageQueue, waitFor } from "@fedify/testing";
4
+ import * as temporal from "@js-temporal/polyfill";
4
5
  import { assert, assertEquals, assertFalse, assertGreater } from "@std/assert";
5
6
  import { delay } from "@std/async/delay";
6
7
  // @deno-types="npm:@types/amqplib"
7
- import { type ChannelModel, connect } from "amqplib";
8
+ import { type Channel, type ChannelModel, connect } from "amqplib";
8
9
  import process from "node:process";
9
10
 
11
+ const Temporal = globalThis.Temporal ?? temporal.Temporal;
12
+
10
13
  const AMQP_URL = process.env.AMQP_URL;
11
- const test = AMQP_URL ? suite(import.meta) : suite(import.meta).skip;
14
+ const unitTest = suite(import.meta);
15
+ const test = AMQP_URL ? unitTest : unitTest.skip;
16
+
17
+ class FakeDepthChannel {
18
+ #closed = false;
19
+
20
+ constructor(private readonly connection: FakeDepthConnection) {
21
+ }
22
+
23
+ on(): void {
24
+ }
25
+
26
+ assertQueue(
27
+ queue: string,
28
+ options?: { expires?: number },
29
+ ): Promise<void> {
30
+ if (this.#closed) {
31
+ return Promise.reject(new Error("Channel is closed"));
32
+ }
33
+ if (
34
+ options?.expires != null &&
35
+ this.connection.preconditionOnExpires.has(queue)
36
+ ) {
37
+ this.#closed = true;
38
+ this.connection.preconditionOnExpires.delete(queue);
39
+ return Promise.reject(
40
+ Object.assign(new Error("PRECONDITION_FAILED"), { code: 406 }),
41
+ );
42
+ }
43
+ this.connection.queues.add(queue);
44
+ if (options?.expires != null) {
45
+ this.connection.queueExpires.set(queue, options.expires);
46
+ }
47
+ return Promise.resolve();
48
+ }
49
+
50
+ sendToQueue(queue: string): boolean {
51
+ if (this.#closed) throw new Error("Channel is closed");
52
+ this.connection.messageCounts.set(
53
+ queue,
54
+ (this.connection.messageCounts.get(queue) ?? 0) + 1,
55
+ );
56
+ return true;
57
+ }
58
+
59
+ async checkQueue(queue: string): Promise<{ messageCount: number }> {
60
+ if (this.#closed) throw new Error("Channel is closed");
61
+ this.connection.activeChecks++;
62
+ this.connection.maxActiveChecks = Math.max(
63
+ this.connection.maxActiveChecks,
64
+ this.connection.activeChecks,
65
+ );
66
+ try {
67
+ if (this.connection.checkDelayMs > 0) {
68
+ await delay(this.connection.checkDelayMs);
69
+ }
70
+ return { messageCount: this.connection.messageCounts.get(queue) ?? 0 };
71
+ } finally {
72
+ this.connection.activeChecks--;
73
+ }
74
+ }
75
+
76
+ close(): Promise<void> {
77
+ this.#closed = true;
78
+ return Promise.resolve();
79
+ }
80
+ }
81
+
82
+ class FakeDepthConnection {
83
+ readonly queues = new Set<string>();
84
+ readonly queueExpires = new Map<string, number>();
85
+ readonly messageCounts = new Map<string, number>();
86
+ readonly preconditionOnExpires = new Set<string>();
87
+ activeChecks = 0;
88
+ channelCount = 0;
89
+ maxActiveChecks = 0;
90
+
91
+ constructor(readonly checkDelayMs: number = 25) {
92
+ }
93
+
94
+ createChannel(): Promise<Channel> {
95
+ this.channelCount++;
96
+ return Promise.resolve(new FakeDepthChannel(this) as unknown as Channel);
97
+ }
98
+ }
99
+
100
+ unitTest(
101
+ "AmqpMessageQueue.getDepth() probes delayed queues concurrently",
102
+ async () => {
103
+ const conn = new FakeDepthConnection();
104
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
105
+ queue: "ready",
106
+ delayedQueuePrefix: "delayed_",
107
+ });
108
+
109
+ await mq.enqueue("first", {
110
+ delay: Temporal.Duration.from({ milliseconds: 1_000 }),
111
+ });
112
+ await mq.enqueue("second", {
113
+ delay: Temporal.Duration.from({ milliseconds: 2_000 }),
114
+ });
115
+ await mq.enqueue("third", {
116
+ delay: Temporal.Duration.from({ milliseconds: 3_000 }),
117
+ });
118
+
119
+ assertEquals(await mq.getDepth(), {
120
+ queued: 3,
121
+ ready: 0,
122
+ delayed: 3,
123
+ });
124
+ assertGreater(conn.maxActiveChecks, 1);
125
+ },
126
+ );
127
+
128
+ unitTest("AmqpMessageQueue sets delayed queue expiry", async () => {
129
+ const conn = new FakeDepthConnection();
130
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
131
+ queue: "ready",
132
+ delayedQueuePrefix: "delayed_",
133
+ });
134
+
135
+ await mq.enqueue("delayed", {
136
+ delay: Temporal.Duration.from({ milliseconds: 1_000 }),
137
+ });
138
+
139
+ assertEquals(conn.queueExpires.get("delayed_1000"), 61_000);
140
+ });
141
+
142
+ unitTest(
143
+ "AmqpMessageQueue falls back for existing delayed queues without expiry",
144
+ async () => {
145
+ const conn = new FakeDepthConnection();
146
+ conn.preconditionOnExpires.add("delayed_1000");
147
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
148
+ queue: "ready",
149
+ delayedQueuePrefix: "delayed_",
150
+ });
151
+
152
+ await mq.enqueue("delayed", {
153
+ delay: Temporal.Duration.from({ milliseconds: 1_000 }),
154
+ });
155
+
156
+ assertEquals(conn.messageCounts.get("delayed_1000"), 1);
157
+ assertEquals(conn.queueExpires.get("delayed_1000"), undefined);
158
+ assertGreater(conn.channelCount, 1);
159
+ },
160
+ );
161
+
162
+ unitTest("AmqpMessageQueue reuses depth probe channels", async () => {
163
+ const conn = new FakeDepthConnection(0);
164
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
165
+ queue: "ready",
166
+ delayedQueuePrefix: "delayed_",
167
+ });
168
+
169
+ for (let milliseconds = 1; milliseconds <= 12; milliseconds++) {
170
+ await mq.enqueue("delayed", {
171
+ delay: Temporal.Duration.from({ milliseconds }),
172
+ });
173
+ }
174
+ conn.channelCount = 0;
175
+
176
+ assertEquals(await mq.getDepth(), {
177
+ queued: 12,
178
+ ready: 0,
179
+ delayed: 12,
180
+ });
181
+ assert(
182
+ conn.channelCount <= 9,
183
+ `expected at most 9 depth probe channels, got ${conn.channelCount}`,
184
+ );
185
+ });
186
+
187
+ unitTest(
188
+ "AmqpMessageQueue keeps delayed queues past cleanup threshold",
189
+ async () => {
190
+ const conn = new FakeDepthConnection(0);
191
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
192
+ queue: "ready",
193
+ delayedQueuePrefix: "delayed_",
194
+ });
195
+
196
+ for (let milliseconds = 1; milliseconds <= 4097; milliseconds++) {
197
+ await mq.enqueue("delayed", {
198
+ delay: Temporal.Duration.from({ milliseconds }),
199
+ });
200
+ }
201
+
202
+ assertEquals(await mq.getDepth(), {
203
+ queued: 4097,
204
+ ready: 0,
205
+ delayed: 4097,
206
+ });
207
+ },
208
+ );
209
+
210
+ unitTest(
211
+ "AmqpMessageQueue.getDepth() keeps delayed queues past local expiry",
212
+ async () => {
213
+ const now = Date.now;
214
+ const started = now();
215
+ Date.now = () => started;
216
+ try {
217
+ const conn = new FakeDepthConnection();
218
+ const mq = new AmqpMessageQueue(conn as unknown as ChannelModel, {
219
+ queue: "ready",
220
+ delayedQueuePrefix: "delayed_",
221
+ });
222
+
223
+ await mq.enqueue("delayed", {
224
+ delay: Temporal.Duration.from({ milliseconds: 1_000 }),
225
+ });
226
+ Date.now = () => started + 62_000;
227
+
228
+ assertEquals(await mq.getDepth(), {
229
+ queued: 1,
230
+ ready: 0,
231
+ delayed: 1,
232
+ });
233
+ } finally {
234
+ Date.now = now;
235
+ }
236
+ },
237
+ );
12
238
 
13
239
  function getConnection(): Promise<ChannelModel> {
14
240
  return connect(AMQP_URL!);
@@ -37,6 +263,51 @@ test(
37
263
  ),
38
264
  );
39
265
 
266
+ test(
267
+ "AmqpMessageQueue.getDepth()",
268
+ { sanitizeOps: false, sanitizeExit: false, sanitizeResources: false },
269
+ async () => {
270
+ const conn = await getConnection();
271
+ const queue = getRandomKey("depth_queue");
272
+ const delayedQueuePrefix = getRandomKey("depth_delayed") + "_";
273
+ const mq = new AmqpMessageQueue(conn, { queue, delayedQueuePrefix });
274
+ try {
275
+ assertEquals(await mq.getDepth(), {
276
+ queued: 0,
277
+ ready: 0,
278
+ delayed: 0,
279
+ });
280
+ await mq.enqueue("ready");
281
+ await mq.enqueue("delayed", {
282
+ delay: Temporal.Duration.from({ seconds: 60 }),
283
+ });
284
+ const started = Date.now();
285
+ while (Date.now() - started < 15_000) {
286
+ const depth = await mq.getDepth();
287
+ if (depth.queued === 2 && depth.ready === 1 && depth.delayed === 1) {
288
+ break;
289
+ }
290
+ await delay(100);
291
+ }
292
+ assertEquals(await mq.getDepth(), {
293
+ queued: 2,
294
+ ready: 1,
295
+ delayed: 1,
296
+ });
297
+ } finally {
298
+ const channel = await conn.createChannel().catch(() => undefined);
299
+ try {
300
+ await channel?.deleteQueue(queue).catch(() => {});
301
+ await channel?.deleteQueue(`${delayedQueuePrefix}60000`).catch(() => {
302
+ });
303
+ } finally {
304
+ await channel?.close().catch(() => {});
305
+ await conn.close().catch(() => {});
306
+ }
307
+ }
308
+ },
309
+ );
310
+
40
311
  // Test with ordering key support (requires rabbitmq_consistent_hash_exchange plugin)
41
312
  const orderingConnections: ChannelModel[] = [];
42
313
  const orderingQueue = getRandomKey("ordering_queue");
package/src/mq.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type {
2
2
  MessageQueue,
3
+ MessageQueueDepth,
3
4
  MessageQueueEnqueueOptions,
4
5
  MessageQueueListenOptions,
5
6
  } from "@fedify/fedify";
@@ -7,6 +8,20 @@ import type {
7
8
  import type { Channel, ChannelModel, ConsumeMessage } from "amqplib";
8
9
  import { Buffer } from "node:buffer";
9
10
 
11
+ function isQueueNotFoundError(error: unknown): boolean {
12
+ return typeof error === "object" && error != null &&
13
+ "code" in error && error.code === 404;
14
+ }
15
+
16
+ function isPreconditionFailedError(error: unknown): boolean {
17
+ return typeof error === "object" && error != null &&
18
+ "code" in error && error.code === 406;
19
+ }
20
+
21
+ const depthProbeConcurrency = 8;
22
+ const delayedQueueExpiryMargin = 60_000;
23
+ const delayedQueueCleanupThreshold = 4096;
24
+
10
25
  /**
11
26
  * Options for ordering key support in {@link AmqpMessageQueue}.
12
27
  *
@@ -127,6 +142,8 @@ export class AmqpMessageQueue implements MessageQueue {
127
142
  queuePrefix: string;
128
143
  partitions: number;
129
144
  };
145
+ #delayedQueues: Set<string> = new Set();
146
+ #delayedQueueCleanup?: Promise<void>;
130
147
  #orderingPrepared: boolean = false;
131
148
 
132
149
  readonly nativeRetrial: boolean;
@@ -194,6 +211,15 @@ export class AmqpMessageQueue implements MessageQueue {
194
211
  return channel;
195
212
  }
196
213
 
214
+ async #dropSenderChannel(channel: Channel): Promise<void> {
215
+ if (this.#senderChannel === channel) this.#senderChannel = undefined;
216
+ try {
217
+ await channel.close();
218
+ } catch {
219
+ // The channel may already have been closed by an AMQP exception.
220
+ }
221
+ }
222
+
197
223
  /**
198
224
  * Enqueues a message to be processed.
199
225
  *
@@ -214,7 +240,7 @@ export class AmqpMessageQueue implements MessageQueue {
214
240
  message: any,
215
241
  options?: MessageQueueEnqueueOptions,
216
242
  ): Promise<void> {
217
- const channel = await this.#getSenderChannel();
243
+ let channel = await this.#getSenderChannel();
218
244
  const delay = options?.delay?.total("millisecond");
219
245
  const orderingKey = options?.orderingKey;
220
246
 
@@ -256,13 +282,12 @@ export class AmqpMessageQueue implements MessageQueue {
256
282
  deadLetterExchange = "";
257
283
  deadLetterRoutingKey = this.#queue;
258
284
  }
259
- await channel.assertQueue(queue, {
260
- autoDelete: true,
261
- durable: this.#durable,
285
+ channel = await this.#assertDelayedQueue(channel, queue, {
262
286
  deadLetterExchange,
263
287
  deadLetterRoutingKey,
264
- messageTtl: delay,
288
+ delay,
265
289
  });
290
+ this.#trackDelayedQueue(queue);
266
291
  }
267
292
  channel.sendToQueue(
268
293
  queue,
@@ -294,7 +319,7 @@ export class AmqpMessageQueue implements MessageQueue {
294
319
  messages: readonly any[],
295
320
  options?: MessageQueueEnqueueOptions,
296
321
  ): Promise<void> {
297
- const channel = await this.#getSenderChannel();
322
+ let channel = await this.#getSenderChannel();
298
323
  const delay = options?.delay?.total("millisecond");
299
324
  const orderingKey = options?.orderingKey;
300
325
 
@@ -338,13 +363,12 @@ export class AmqpMessageQueue implements MessageQueue {
338
363
  deadLetterExchange = "";
339
364
  deadLetterRoutingKey = this.#queue;
340
365
  }
341
- await channel.assertQueue(queue, {
342
- autoDelete: true,
343
- durable: this.#durable,
366
+ channel = await this.#assertDelayedQueue(channel, queue, {
344
367
  deadLetterExchange,
345
368
  deadLetterRoutingKey,
346
- messageTtl: delay,
369
+ delay,
347
370
  });
371
+ this.#trackDelayedQueue(queue);
348
372
  }
349
373
 
350
374
  for (const message of messages) {
@@ -359,6 +383,148 @@ export class AmqpMessageQueue implements MessageQueue {
359
383
  }
360
384
  }
361
385
 
386
+ #trackDelayedQueue(queue: string): void {
387
+ this.#delayedQueues.add(queue);
388
+ if (
389
+ this.#delayedQueues.size > delayedQueueCleanupThreshold &&
390
+ this.#delayedQueueCleanup == null
391
+ ) {
392
+ this.#delayedQueueCleanup = this.#pruneMissingDelayedQueues()
393
+ .catch(() => undefined)
394
+ .finally(() => {
395
+ this.#delayedQueueCleanup = undefined;
396
+ });
397
+ }
398
+ }
399
+
400
+ async #assertDelayedQueue(
401
+ channel: Channel,
402
+ queue: string,
403
+ options: {
404
+ deadLetterExchange?: string;
405
+ deadLetterRoutingKey?: string;
406
+ delay: number;
407
+ },
408
+ ): Promise<Channel> {
409
+ const assertOptions = {
410
+ autoDelete: true,
411
+ durable: this.#durable,
412
+ deadLetterExchange: options.deadLetterExchange,
413
+ deadLetterRoutingKey: options.deadLetterRoutingKey,
414
+ messageTtl: options.delay,
415
+ };
416
+ try {
417
+ await channel.assertQueue(queue, {
418
+ ...assertOptions,
419
+ expires: options.delay + delayedQueueExpiryMargin,
420
+ });
421
+ return channel;
422
+ } catch (error) {
423
+ if (!isPreconditionFailedError(error)) throw error;
424
+ await this.#dropSenderChannel(channel);
425
+ }
426
+
427
+ const fallbackChannel = await this.#getSenderChannel();
428
+ await fallbackChannel.assertQueue(queue, assertOptions);
429
+ return fallbackChannel;
430
+ }
431
+
432
+ async #createDepthChannel(): Promise<Channel> {
433
+ const channel = await this.#connection.createChannel();
434
+ channel.on("error", () => undefined);
435
+ return channel;
436
+ }
437
+
438
+ async #checkQueueDepths(
439
+ queueNames: readonly string[],
440
+ ): Promise<readonly (readonly [string, number | undefined])[]> {
441
+ const results = new Array<readonly [string, number | undefined]>(
442
+ queueNames.length,
443
+ );
444
+ let nextIndex = 0;
445
+ const worker = async () => {
446
+ let channel: Channel | undefined;
447
+ const closeChannel = async () => {
448
+ if (channel == null) return;
449
+ const currentChannel = channel;
450
+ channel = undefined;
451
+ try {
452
+ await currentChannel.close();
453
+ } catch {
454
+ // The channel can already be closed by a failed passive queue check.
455
+ }
456
+ };
457
+ const checkQueue = async (
458
+ queue: string,
459
+ ): Promise<number | undefined> => {
460
+ channel ??= await this.#createDepthChannel();
461
+ try {
462
+ return (await channel.checkQueue(queue)).messageCount;
463
+ } catch (error) {
464
+ await closeChannel();
465
+ if (!isQueueNotFoundError(error)) throw error;
466
+ return undefined;
467
+ }
468
+ };
469
+ try {
470
+ while (nextIndex < queueNames.length) {
471
+ const index = nextIndex++;
472
+ const queue = queueNames[index];
473
+ results[index] = [queue, await checkQueue(queue)];
474
+ }
475
+ } finally {
476
+ await closeChannel();
477
+ }
478
+ };
479
+ const workers = Array.from(
480
+ { length: Math.min(depthProbeConcurrency, queueNames.length) },
481
+ () => worker(),
482
+ );
483
+ await Promise.all(workers);
484
+ return results;
485
+ }
486
+
487
+ async #pruneMissingDelayedQueues(): Promise<void> {
488
+ const delayedQueues = [...this.#delayedQueues];
489
+ for (
490
+ const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)
491
+ ) {
492
+ if (messageCount == null) this.#delayedQueues.delete(queue);
493
+ }
494
+ }
495
+
496
+ async getDepth(): Promise<MessageQueueDepth> {
497
+ const readyQueues = [this.#queue];
498
+ if (this.#ordering != null) {
499
+ for (let i = 0; i < this.#ordering.partitions; i++) {
500
+ readyQueues.push(this.#getOrderingQueueName(i));
501
+ }
502
+ }
503
+
504
+ let ready = 0;
505
+ for (const [, messageCount] of await this.#checkQueueDepths(readyQueues)) {
506
+ ready += messageCount ?? 0;
507
+ }
508
+
509
+ let delayed = 0;
510
+ const delayedQueues = [...this.#delayedQueues];
511
+ for (
512
+ const [queue, messageCount] of await this.#checkQueueDepths(delayedQueues)
513
+ ) {
514
+ if (messageCount == null) {
515
+ this.#delayedQueues.delete(queue);
516
+ } else {
517
+ delayed += messageCount;
518
+ }
519
+ }
520
+
521
+ return {
522
+ queued: ready + delayed,
523
+ ready,
524
+ delayed,
525
+ };
526
+ }
527
+
362
528
  async listen(
363
529
  // deno-lint-ignore no-explicit-any
364
530
  handler: (message: any) => void | Promise<void>,