@boopkit/cap-background-jobs 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/capability.json +2 -4
- package/package.json +1 -1
- package/scaffold/reference/background-jobs.ref.md +14 -2
- package/scaffold/src/scheduler.ts +30 -0
- package/skills/design.background-jobs.skill.md +1 -1
- package/scaffold/src/services/.gitkeep +0 -0
- package/scaffold/src/services/message-queue.service.ts +0 -316
- package/scaffold/tests/services/.gitkeep +0 -0
- package/scaffold/tests/services/message-queue.service.test.ts +0 -419
package/capability.json
CHANGED
|
@@ -12,15 +12,13 @@
|
|
|
12
12
|
{ "name": "MAX_MESSAGE_RETRIES", "description": "Maximum retry attempts before dead-lettering (default: 5)" }
|
|
13
13
|
],
|
|
14
14
|
"scripts": {
|
|
15
|
-
"worker": { "cmd": "
|
|
16
|
-
"scheduler": { "cmd": "
|
|
15
|
+
"worker": { "cmd": "tsx src/worker.ts", "description": "Start the background job worker process" },
|
|
16
|
+
"scheduler": { "cmd": "tsx src/scheduler.ts", "description": "Publish trigger messages to job queues — supports --interval and --job filtering" }
|
|
17
17
|
},
|
|
18
18
|
"files": [
|
|
19
19
|
"src/scheduler.ts",
|
|
20
20
|
"src/worker.ts",
|
|
21
|
-
"src/services/message-queue.service.ts",
|
|
22
21
|
"src/controllers/example-job.controller.ts",
|
|
23
|
-
"tests/services/message-queue.service.test.ts",
|
|
24
22
|
"tests/controllers/example-job.controller.test.ts",
|
|
25
23
|
"reference/background-jobs.ref.md"
|
|
26
24
|
],
|
package/package.json
CHANGED
|
@@ -87,6 +87,12 @@ Each queue has a corresponding direct exchange for routing. Retry messages dead-
|
|
|
87
87
|
|
|
88
88
|
The web process only opens an AMQP connection if it actually publishes a message.
|
|
89
89
|
|
|
90
|
+
> **Package vs scaffold:** The `MessageQueueService` implementation lives in the
|
|
91
|
+
> `@boopkit/cap-background-jobs` package (`src/message-queue.service.ts`). The
|
|
92
|
+
> scaffold's `src/services/message-queue.service.ts` is a thin re-export so
|
|
93
|
+
> existing local imports keep working. Package updates flow to consuming
|
|
94
|
+
> projects via `npm update` without re-scaffolding.
|
|
95
|
+
|
|
90
96
|
---
|
|
91
97
|
|
|
92
98
|
## Consumer Registration
|
|
@@ -182,7 +188,13 @@ If drain exceeds 30s, the worker force-exits with code 1.
|
|
|
182
188
|
|------|---------|
|
|
183
189
|
| `src/worker.ts` | Worker entry point — boots services, registers consumers, handles shutdown |
|
|
184
190
|
| `src/scheduler.ts` | Scheduler entry point — CLI with interval/job filtering, publishes trigger messages, exits |
|
|
185
|
-
| `src/services/message-queue.service.ts` |
|
|
191
|
+
| `src/services/message-queue.service.ts` | Re-exports `MessageQueueService` from `@boopkit/cap-background-jobs` |
|
|
186
192
|
| `src/controllers/example-job.controller.ts` | Example consumer — copyable template for new job handlers |
|
|
187
|
-
| `tests/services/message-queue.service.test.ts` | MQ service unit tests |
|
|
188
193
|
| `tests/controllers/example-job.controller.test.ts` | Example controller unit tests |
|
|
194
|
+
|
|
195
|
+
### Package-level files
|
|
196
|
+
|
|
197
|
+
| File | Purpose |
|
|
198
|
+
|------|---------||
|
|
199
|
+
| `@boopkit/cap-background-jobs` `src/message-queue.service.ts` | Core MQ service — publish, listen (auto/manual ACK), topology, retry/DLQ |
|
|
200
|
+
| `@boopkit/cap-background-jobs` `tests/message-queue.service.test.ts` | MQ service unit tests |
|
|
@@ -69,11 +69,15 @@ function printUsage(): void {
|
|
|
69
69
|
|
|
70
70
|
Commands:
|
|
71
71
|
run-jobs Publish trigger messages to matching job queues
|
|
72
|
+
publish Publish a single message to a queue and exit
|
|
72
73
|
list List all registered jobs
|
|
73
74
|
|
|
74
75
|
Options:
|
|
75
76
|
--interval <tag> Filter jobs by interval tag (e.g. 10-minutes, 60-minutes)
|
|
76
77
|
--job <name> Filter jobs by name
|
|
78
|
+
--queue <name> Target queue for publish command
|
|
79
|
+
--task <name> Task name for publish payload (default: "cli")
|
|
80
|
+
--data <json> JSON string of extra data for publish payload
|
|
77
81
|
`);
|
|
78
82
|
}
|
|
79
83
|
|
|
@@ -104,6 +108,32 @@ async function main(args: string[], opts: Record<string, unknown>): Promise<void
|
|
|
104
108
|
break;
|
|
105
109
|
}
|
|
106
110
|
|
|
111
|
+
case 'publish': {
|
|
112
|
+
const queue = opts.queue as string | undefined;
|
|
113
|
+
if (!queue) {
|
|
114
|
+
console.error('[scheduler] --queue is required for publish command');
|
|
115
|
+
process.exit(1);
|
|
116
|
+
}
|
|
117
|
+
const task = (opts.task as string) ?? 'cli';
|
|
118
|
+
let data: Record<string, unknown> = {};
|
|
119
|
+
if (opts.data) {
|
|
120
|
+
try {
|
|
121
|
+
data = JSON.parse(opts.data as string);
|
|
122
|
+
} catch {
|
|
123
|
+
console.error('[scheduler] --data must be valid JSON');
|
|
124
|
+
process.exit(1);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const ok = await MessageQueueService.publish(queue, { task, data });
|
|
128
|
+
if (ok) {
|
|
129
|
+
console.log(`[scheduler] published to "${queue}": ${task}`);
|
|
130
|
+
} else {
|
|
131
|
+
console.error(`[scheduler] failed to publish to "${queue}"`);
|
|
132
|
+
process.exit(1);
|
|
133
|
+
}
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
|
|
107
137
|
default:
|
|
108
138
|
console.error(`[scheduler] unknown command: ${command}`);
|
|
109
139
|
printUsage();
|
|
@@ -50,7 +50,7 @@ That's it. The MQ service creates the queue topology (main + retry + DLQ) automa
|
|
|
50
50
|
Use `MessageQueueService.publish(queue, payload)` from any process:
|
|
51
51
|
|
|
52
52
|
```typescript
|
|
53
|
-
import MessageQueueService from '
|
|
53
|
+
import { MessageQueueService } from '@boopkit/cap-background-jobs';
|
|
54
54
|
|
|
55
55
|
await MessageQueueService.publish('my-job', {
|
|
56
56
|
task: 'process-upload',
|
|
File without changes
|
|
@@ -1,316 +0,0 @@
|
|
|
1
|
-
/** @boopkit/cap-background-jobs — RabbitMQ message queue service. */
|
|
2
|
-
|
|
3
|
-
import amqplib from 'amqplib';
|
|
4
|
-
import type { ChannelModel, Channel, ConsumeMessage } from 'amqplib';
|
|
5
|
-
|
|
6
|
-
const QUEUE_PREFIX = process.env.QUEUE_PREFIX ?? '';
|
|
7
|
-
const MAX_RETRIES = parseInt(process.env.MAX_MESSAGE_RETRIES ?? '5', 10);
|
|
8
|
-
const BACKOFF_BASE_MS = 4000;
|
|
9
|
-
const BACKOFF_MAX_MS = 60000;
|
|
10
|
-
|
|
11
|
-
/** All open connections tracked for shutdown cleanup. */
|
|
12
|
-
const connections: ChannelModel[] = [];
|
|
13
|
-
|
|
14
|
-
/** Lazy-initialized publish connection and channel. */
|
|
15
|
-
let publishConnection: ChannelModel | null = null;
|
|
16
|
-
let publishChannel: Channel | null = null;
|
|
17
|
-
|
|
18
|
-
const MessageQueueService = {
|
|
19
|
-
name: 'message-queue',
|
|
20
|
-
init,
|
|
21
|
-
publish,
|
|
22
|
-
listen,
|
|
23
|
-
close,
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
export default MessageQueueService;
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Validate that CLOUDAMQP_URL is set. Does NOT open a connection —
|
|
30
|
-
* publish connection is lazy (first publish call), listener connections
|
|
31
|
-
* are created at listen() time.
|
|
32
|
-
*/
|
|
33
|
-
function init(): void {
|
|
34
|
-
if (!process.env.CLOUDAMQP_URL) {
|
|
35
|
-
throw new Error(
|
|
36
|
-
'E: CLOUDAMQP_URL is not set. ' +
|
|
37
|
-
'R: The message queue service requires a CloudAMQP connection URL. ' +
|
|
38
|
-
'R: Set CLOUDAMQP_URL in your environment (e.g. .env file).',
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Publish a message to the given queue.
|
|
45
|
-
* Lazy-initializes a persistent connection on first call, reuses it
|
|
46
|
-
* across subsequent calls. Returns true on success, false on failure.
|
|
47
|
-
*/
|
|
48
|
-
async function publish(queue: string, payload: unknown): Promise<boolean> {
|
|
49
|
-
const queueName = prefixedName(queue);
|
|
50
|
-
|
|
51
|
-
try {
|
|
52
|
-
const channel = await getPublishChannel();
|
|
53
|
-
await channel.assertQueue(queueName, { durable: true });
|
|
54
|
-
|
|
55
|
-
const body = Buffer.from(JSON.stringify(payload));
|
|
56
|
-
const ok = channel.sendToQueue(queueName, body, {
|
|
57
|
-
persistent: true,
|
|
58
|
-
messageId: crypto.randomUUID(),
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
if (!ok) {
|
|
62
|
-
// Channel buffer full — wait for drain
|
|
63
|
-
await new Promise<void>((resolve) => channel.once('drain', resolve));
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
return true;
|
|
67
|
-
} catch (err) {
|
|
68
|
-
console.error(`[message-queue] publish error on "${queue}":`, err);
|
|
69
|
-
resetPublishConnection();
|
|
70
|
-
return false;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** Options for listen(). */
|
|
75
|
-
interface ListenOpts {
|
|
76
|
-
/** When true, handler receives guarded ack/nack functions instead of returning boolean. */
|
|
77
|
-
manualAck?: boolean;
|
|
78
|
-
/** Channel prefetch count — limits concurrent message delivery for backpressure. */
|
|
79
|
-
prefetch?: number;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Listen on a queue with auto-ACK or manual-ACK semantics.
|
|
84
|
-
*
|
|
85
|
-
* Asserts full queue topology on registration:
|
|
86
|
-
* - Main queue: `{prefix}/{name}`
|
|
87
|
-
* - Retry queue: `{prefix}/{name}.retry` (TTL redelivery to main)
|
|
88
|
-
* - DLQ: `{prefix}/{name}.failed` (dead-letter exchange)
|
|
89
|
-
*
|
|
90
|
-
* **Auto-ACK (default):** Handler returns true → ACK, false → retry with backoff,
|
|
91
|
-
* throws → NACK to DLQ after max retries.
|
|
92
|
-
*
|
|
93
|
-
* **Manual-ACK (opts.manualAck = true):** Handler receives guarded ack()/nack()
|
|
94
|
-
* functions. First call wins — subsequent calls are no-ops. If handler resolves
|
|
95
|
-
* or throws without settling → retry/DLQ after max retries.
|
|
96
|
-
*/
|
|
97
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
98
|
-
async function listen(queue: string, handler: (...args: any[]) => Promise<any>, opts?: ListenOpts): Promise<void> {
|
|
99
|
-
const connection = await amqplib.connect(process.env.CLOUDAMQP_URL!);
|
|
100
|
-
connections.push(connection);
|
|
101
|
-
|
|
102
|
-
const channel = await connection.createChannel();
|
|
103
|
-
const { mainQueue, retryQueue, failedQueue } = await assertTopology(channel, queue);
|
|
104
|
-
|
|
105
|
-
if (opts?.prefetch != null) {
|
|
106
|
-
await channel.prefetch(opts.prefetch);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
await channel.consume(mainQueue, async (raw) => {
|
|
110
|
-
if (!raw) return;
|
|
111
|
-
|
|
112
|
-
const retryCount = getRetryCount(raw);
|
|
113
|
-
let parsed: unknown;
|
|
114
|
-
|
|
115
|
-
try {
|
|
116
|
-
parsed = JSON.parse(raw.content.toString());
|
|
117
|
-
} catch {
|
|
118
|
-
// Poison message — ACK to prevent infinite redelivery
|
|
119
|
-
console.error(`[message-queue] unparseable message on "${queue}", sending to DLQ`);
|
|
120
|
-
channel.nack(raw, false, false);
|
|
121
|
-
return;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
if (opts?.manualAck) {
|
|
125
|
-
// Manual-ACK: handler receives guarded ack/nack
|
|
126
|
-
let settled = false;
|
|
127
|
-
|
|
128
|
-
const guardedAck = (): void => {
|
|
129
|
-
if (settled) {
|
|
130
|
-
console.warn(`[message-queue] duplicate ack on "${queue}" — ignored`);
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
settled = true;
|
|
134
|
-
channel.ack(raw);
|
|
135
|
-
};
|
|
136
|
-
|
|
137
|
-
const guardedNack = async (): Promise<void> => {
|
|
138
|
-
if (settled) {
|
|
139
|
-
console.warn(`[message-queue] duplicate nack on "${queue}" — ignored`);
|
|
140
|
-
return;
|
|
141
|
-
}
|
|
142
|
-
settled = true;
|
|
143
|
-
await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
try {
|
|
147
|
-
await (handler as (msg: unknown, ack: () => void, nack: () => void) => Promise<void>)(parsed, guardedAck, guardedNack);
|
|
148
|
-
|
|
149
|
-
if (!settled) {
|
|
150
|
-
console.warn(`[message-queue] handler resolved without settling on "${queue}" — treating as failure`);
|
|
151
|
-
await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
|
|
152
|
-
}
|
|
153
|
-
} catch (err) {
|
|
154
|
-
console.error(`[message-queue] handler threw on "${queue}":`, err);
|
|
155
|
-
if (!settled) {
|
|
156
|
-
await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
} else {
|
|
160
|
-
// Auto-ACK: handler returns boolean
|
|
161
|
-
try {
|
|
162
|
-
const ok = await (handler as (msg: unknown) => Promise<boolean>)(parsed);
|
|
163
|
-
|
|
164
|
-
if (ok) {
|
|
165
|
-
channel.ack(raw);
|
|
166
|
-
} else {
|
|
167
|
-
await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
|
|
168
|
-
}
|
|
169
|
-
} catch (err) {
|
|
170
|
-
console.error(`[message-queue] handler threw on "${queue}":`, err);
|
|
171
|
-
await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
}, { noAck: false });
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/** Close all tracked connections (publish + listeners). */
|
|
178
|
-
async function close(): Promise<void> {
|
|
179
|
-
resetPublishConnection();
|
|
180
|
-
|
|
181
|
-
const closing = connections.map(async (conn) => {
|
|
182
|
-
try {
|
|
183
|
-
await conn.close();
|
|
184
|
-
} catch {
|
|
185
|
-
// Already closed — ignore
|
|
186
|
-
}
|
|
187
|
-
});
|
|
188
|
-
await Promise.all(closing);
|
|
189
|
-
connections.length = 0;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
// ---------------------------------------------------------------------------
|
|
193
|
-
// Internal helpers
|
|
194
|
-
// ---------------------------------------------------------------------------
|
|
195
|
-
|
|
196
|
-
function prefixedName(name: string): string {
|
|
197
|
-
return QUEUE_PREFIX ? `${QUEUE_PREFIX}/${name}` : name;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/** Lazy-create or return the existing publish channel. */
|
|
201
|
-
async function getPublishChannel(): Promise<Channel> {
|
|
202
|
-
if (publishChannel) return publishChannel;
|
|
203
|
-
|
|
204
|
-
const conn = await amqplib.connect(process.env.CLOUDAMQP_URL!);
|
|
205
|
-
connections.push(conn);
|
|
206
|
-
publishConnection = conn;
|
|
207
|
-
|
|
208
|
-
conn.on('error', () => resetPublishConnection());
|
|
209
|
-
conn.on('close', () => resetPublishConnection());
|
|
210
|
-
|
|
211
|
-
publishChannel = await conn.createChannel();
|
|
212
|
-
return publishChannel;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
/** Clear cached publish connection so the next call reconnects. */
|
|
216
|
-
function resetPublishConnection(): void {
|
|
217
|
-
if (publishConnection) {
|
|
218
|
-
publishConnection.close().catch(() => { /* already closing */ });
|
|
219
|
-
const idx = connections.indexOf(publishConnection);
|
|
220
|
-
if (idx !== -1) connections.splice(idx, 1);
|
|
221
|
-
}
|
|
222
|
-
publishConnection = null;
|
|
223
|
-
publishChannel = null;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
/**
|
|
227
|
-
* Assert main + retry + DLQ queues for a given queue name.
|
|
228
|
-
*
|
|
229
|
-
* Topology:
|
|
230
|
-
* main ──(dead-letter)──▸ {name}.failed (via exchange)
|
|
231
|
-
* retry ──(TTL expiry)──▸ main (via dead-letter back to main exchange)
|
|
232
|
-
*/
|
|
233
|
-
async function assertTopology(
|
|
234
|
-
channel: Channel,
|
|
235
|
-
queue: string,
|
|
236
|
-
): Promise<{ mainQueue: string; retryQueue: string; failedQueue: string }> {
|
|
237
|
-
const mainQueue = prefixedName(queue);
|
|
238
|
-
const retryQueue = `${mainQueue}.retry`;
|
|
239
|
-
const failedQueue = `${mainQueue}.failed`;
|
|
240
|
-
|
|
241
|
-
const mainExchange = `${mainQueue}.exchange`;
|
|
242
|
-
const retryExchange = `${retryQueue}.exchange`;
|
|
243
|
-
const failedExchange = `${failedQueue}.exchange`;
|
|
244
|
-
|
|
245
|
-
// Exchanges
|
|
246
|
-
await channel.assertExchange(mainExchange, 'direct', { durable: true });
|
|
247
|
-
await channel.assertExchange(retryExchange, 'direct', { durable: true });
|
|
248
|
-
await channel.assertExchange(failedExchange, 'direct', { durable: true });
|
|
249
|
-
|
|
250
|
-
// Main queue — dead-letters to failed exchange.
|
|
251
|
-
// Note: the DLX here is primarily exercised for poison messages (unparseable
|
|
252
|
-
// JSON) which are nacked without requeue. Normal failures go through
|
|
253
|
-
// retryOrDLQ() which manually publishes to the retry/failed queue and acks.
|
|
254
|
-
await channel.assertQueue(mainQueue, {
|
|
255
|
-
durable: true,
|
|
256
|
-
deadLetterExchange: failedExchange,
|
|
257
|
-
});
|
|
258
|
-
await channel.bindQueue(mainQueue, mainExchange, '');
|
|
259
|
-
|
|
260
|
-
// Retry queue — messages TTL then dead-letter back to main exchange
|
|
261
|
-
await channel.assertQueue(retryQueue, {
|
|
262
|
-
durable: true,
|
|
263
|
-
deadLetterExchange: mainExchange,
|
|
264
|
-
});
|
|
265
|
-
await channel.bindQueue(retryQueue, retryExchange, '');
|
|
266
|
-
|
|
267
|
-
// Failed (DLQ) — terminal resting place
|
|
268
|
-
await channel.assertQueue(failedQueue, { durable: true });
|
|
269
|
-
await channel.bindQueue(failedQueue, failedExchange, '');
|
|
270
|
-
|
|
271
|
-
return { mainQueue, retryQueue, failedQueue };
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
function getRetryCount(msg: ConsumeMessage): number {
|
|
275
|
-
const headers = msg.properties.headers ?? {};
|
|
276
|
-
return typeof headers['x-retry-count'] === 'number' ? headers['x-retry-count'] : 0;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
function computeBackoff(retryCount: number): number {
|
|
280
|
-
return Math.min(BACKOFF_BASE_MS * Math.pow(2, retryCount), BACKOFF_MAX_MS);
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
/**
|
|
284
|
-
* Retry a message (publish to retry queue with incremented count and TTL)
|
|
285
|
-
* or send to DLQ if max retries exceeded.
|
|
286
|
-
*/
|
|
287
|
-
async function retryOrDLQ(
|
|
288
|
-
channel: Channel,
|
|
289
|
-
raw: ConsumeMessage,
|
|
290
|
-
retryCount: number,
|
|
291
|
-
retryQueue: string,
|
|
292
|
-
failedQueue: string,
|
|
293
|
-
): Promise<void> {
|
|
294
|
-
if (retryCount >= MAX_RETRIES) {
|
|
295
|
-
console.error(
|
|
296
|
-
`[message-queue] max retries (${MAX_RETRIES}) exceeded, moving to DLQ: ${failedQueue}`,
|
|
297
|
-
);
|
|
298
|
-
channel.publish('', failedQueue, raw.content, {
|
|
299
|
-
persistent: true,
|
|
300
|
-
headers: { ...raw.properties.headers, 'x-retry-count': retryCount },
|
|
301
|
-
});
|
|
302
|
-
channel.ack(raw);
|
|
303
|
-
return;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
const ttl = computeBackoff(retryCount);
|
|
307
|
-
channel.publish('', retryQueue, raw.content, {
|
|
308
|
-
persistent: true,
|
|
309
|
-
expiration: String(ttl),
|
|
310
|
-
headers: {
|
|
311
|
-
...raw.properties.headers,
|
|
312
|
-
'x-retry-count': retryCount + 1,
|
|
313
|
-
},
|
|
314
|
-
});
|
|
315
|
-
channel.ack(raw);
|
|
316
|
-
}
|
|
File without changes
|
|
@@ -1,419 +0,0 @@
|
|
|
1
|
-
import type { ConsumeMessage } from 'amqplib';
|
|
2
|
-
|
|
3
|
-
// Mock amqplib before importing the service
|
|
4
|
-
const mockChannel: Record<string, jest.Mock> = {
|
|
5
|
-
assertExchange: jest.fn().mockResolvedValue(undefined),
|
|
6
|
-
assertQueue: jest.fn().mockResolvedValue({ queue: 'test' }),
|
|
7
|
-
bindQueue: jest.fn().mockResolvedValue(undefined),
|
|
8
|
-
sendToQueue: jest.fn().mockReturnValue(true),
|
|
9
|
-
publish: jest.fn().mockReturnValue(true),
|
|
10
|
-
consume: jest.fn().mockResolvedValue({ consumerTag: 'test-tag' }),
|
|
11
|
-
prefetch: jest.fn().mockResolvedValue(undefined),
|
|
12
|
-
ack: jest.fn(),
|
|
13
|
-
nack: jest.fn(),
|
|
14
|
-
once: jest.fn(),
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
const mockConnection: Record<string, jest.Mock> = {
|
|
18
|
-
createChannel: jest.fn().mockResolvedValue(mockChannel),
|
|
19
|
-
close: jest.fn().mockResolvedValue(undefined),
|
|
20
|
-
on: jest.fn(),
|
|
21
|
-
};
|
|
22
|
-
|
|
23
|
-
jest.mock('amqplib', () => ({
|
|
24
|
-
connect: jest.fn().mockResolvedValue(mockConnection),
|
|
25
|
-
}));
|
|
26
|
-
|
|
27
|
-
import amqplib from 'amqplib';
|
|
28
|
-
import MessageQueueService from '../../src/services/message-queue.service';
|
|
29
|
-
|
|
30
|
-
describe('MessageQueueService', () => {
|
|
31
|
-
const ORIGINAL_ENV = process.env;
|
|
32
|
-
|
|
33
|
-
beforeEach(() => {
|
|
34
|
-
jest.clearAllMocks();
|
|
35
|
-
process.env = { ...ORIGINAL_ENV, CLOUDAMQP_URL: 'amqp://test:test@localhost/test' };
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
afterEach(() => {
|
|
39
|
-
process.env = ORIGINAL_ENV;
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
describe('init()', () => {
|
|
43
|
-
it('succeeds when CLOUDAMQP_URL is set', () => {
|
|
44
|
-
expect(() => MessageQueueService.init()).not.toThrow();
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it('throws when CLOUDAMQP_URL is missing', () => {
|
|
48
|
-
delete process.env.CLOUDAMQP_URL;
|
|
49
|
-
expect(() => MessageQueueService.init()).toThrow('CLOUDAMQP_URL is not set');
|
|
50
|
-
});
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
describe('publish()', () => {
|
|
54
|
-
it('connects lazily on first publish', async () => {
|
|
55
|
-
const result = await MessageQueueService.publish('test-queue', { foo: 'bar' });
|
|
56
|
-
|
|
57
|
-
expect(result).toBe(true);
|
|
58
|
-
expect(amqplib.connect).toHaveBeenCalledWith(process.env.CLOUDAMQP_URL);
|
|
59
|
-
expect(mockConnection.createChannel).toHaveBeenCalled();
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
it('publishes with persistent flag and messageId', async () => {
|
|
63
|
-
await MessageQueueService.publish('test-queue', { data: 1 });
|
|
64
|
-
|
|
65
|
-
expect(mockChannel.sendToQueue).toHaveBeenCalledWith(
|
|
66
|
-
'test-queue',
|
|
67
|
-
expect.any(Buffer),
|
|
68
|
-
expect.objectContaining({
|
|
69
|
-
persistent: true,
|
|
70
|
-
messageId: expect.any(String),
|
|
71
|
-
}),
|
|
72
|
-
);
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
it('serializes payload as JSON', async () => {
|
|
76
|
-
const payload = { key: 'value', count: 42 };
|
|
77
|
-
await MessageQueueService.publish('test-queue', payload);
|
|
78
|
-
|
|
79
|
-
const sentBuffer = (mockChannel.sendToQueue as jest.Mock).mock.calls[0][1];
|
|
80
|
-
expect(JSON.parse(sentBuffer.toString())).toEqual(payload);
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
it('returns false on error', async () => {
|
|
84
|
-
(mockChannel.sendToQueue as jest.Mock).mockImplementationOnce(() => {
|
|
85
|
-
throw new Error('channel closed');
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
const result = await MessageQueueService.publish('test-queue', {});
|
|
89
|
-
expect(result).toBe(false);
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
it('waits for drain when channel buffer is full', async () => {
|
|
93
|
-
(mockChannel.sendToQueue as jest.Mock).mockReturnValueOnce(false);
|
|
94
|
-
(mockChannel.once as jest.Mock).mockImplementationOnce(
|
|
95
|
-
(_event: string, cb: () => void) => { cb(); },
|
|
96
|
-
);
|
|
97
|
-
|
|
98
|
-
const result = await MessageQueueService.publish('test-queue', {});
|
|
99
|
-
|
|
100
|
-
expect(result).toBe(true);
|
|
101
|
-
expect(mockChannel.once).toHaveBeenCalledWith('drain', expect.any(Function));
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
// Note: QUEUE_PREFIX is captured once at module load time via const.
|
|
105
|
-
// It cannot be tested by setting process.env after import. The prefix
|
|
106
|
-
// behavior is verified through listen() topology assertions instead.
|
|
107
|
-
});
|
|
108
|
-
|
|
109
|
-
describe('listen()', () => {
|
|
110
|
-
it('creates a new connection per listener', async () => {
|
|
111
|
-
await MessageQueueService.listen('jobs', async () => true);
|
|
112
|
-
|
|
113
|
-
expect(amqplib.connect).toHaveBeenCalledWith(process.env.CLOUDAMQP_URL);
|
|
114
|
-
expect(mockConnection.createChannel).toHaveBeenCalled();
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
it('asserts full queue topology (main + retry + DLQ)', async () => {
|
|
118
|
-
await MessageQueueService.listen('jobs', async () => true);
|
|
119
|
-
|
|
120
|
-
// Exchanges
|
|
121
|
-
expect(mockChannel.assertExchange).toHaveBeenCalledWith(
|
|
122
|
-
'jobs.exchange', 'direct', { durable: true },
|
|
123
|
-
);
|
|
124
|
-
expect(mockChannel.assertExchange).toHaveBeenCalledWith(
|
|
125
|
-
'jobs.retry.exchange', 'direct', { durable: true },
|
|
126
|
-
);
|
|
127
|
-
expect(mockChannel.assertExchange).toHaveBeenCalledWith(
|
|
128
|
-
'jobs.failed.exchange', 'direct', { durable: true },
|
|
129
|
-
);
|
|
130
|
-
|
|
131
|
-
// Queues
|
|
132
|
-
expect(mockChannel.assertQueue).toHaveBeenCalledWith(
|
|
133
|
-
'jobs',
|
|
134
|
-
expect.objectContaining({ durable: true, deadLetterExchange: 'jobs.failed.exchange' }),
|
|
135
|
-
);
|
|
136
|
-
expect(mockChannel.assertQueue).toHaveBeenCalledWith(
|
|
137
|
-
'jobs.retry',
|
|
138
|
-
expect.objectContaining({ durable: true, deadLetterExchange: 'jobs.exchange' }),
|
|
139
|
-
);
|
|
140
|
-
expect(mockChannel.assertQueue).toHaveBeenCalledWith(
|
|
141
|
-
'jobs.failed',
|
|
142
|
-
expect.objectContaining({ durable: true }),
|
|
143
|
-
);
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
it('sets up a consumer on the main queue', async () => {
|
|
147
|
-
await MessageQueueService.listen('jobs', async () => true);
|
|
148
|
-
expect(mockChannel.consume).toHaveBeenCalledWith('jobs', expect.any(Function), { noAck: false });
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
describe('message dispatch', () => {
|
|
152
|
-
let consumeCallback: (msg: ConsumeMessage | null) => Promise<void>;
|
|
153
|
-
|
|
154
|
-
beforeEach(async () => {
|
|
155
|
-
(mockChannel.consume as jest.Mock).mockImplementationOnce(
|
|
156
|
-
(_queue: string, cb: (msg: ConsumeMessage | null) => Promise<void>) => {
|
|
157
|
-
consumeCallback = cb;
|
|
158
|
-
return Promise.resolve({ consumerTag: 'tag' });
|
|
159
|
-
},
|
|
160
|
-
);
|
|
161
|
-
await MessageQueueService.listen('jobs', async (msg) => {
|
|
162
|
-
if ((msg as any).fail) return false;
|
|
163
|
-
if ((msg as any).throw) throw new Error('boom');
|
|
164
|
-
return true;
|
|
165
|
-
});
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
function makeMessage(payload: unknown, retryCount = 0): ConsumeMessage {
|
|
169
|
-
return {
|
|
170
|
-
content: Buffer.from(JSON.stringify(payload)),
|
|
171
|
-
properties: {
|
|
172
|
-
headers: { 'x-retry-count': retryCount },
|
|
173
|
-
},
|
|
174
|
-
fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
|
|
175
|
-
} as unknown as ConsumeMessage;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
it('ACKs when handler returns true', async () => {
|
|
179
|
-
await consumeCallback(makeMessage({ ok: true }));
|
|
180
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
it('retries when handler returns false', async () => {
|
|
184
|
-
await consumeCallback(makeMessage({ fail: true }));
|
|
185
|
-
|
|
186
|
-
// Should publish to retry queue with incremented retry count
|
|
187
|
-
expect(mockChannel.publish).toHaveBeenCalledWith(
|
|
188
|
-
'',
|
|
189
|
-
'jobs.retry',
|
|
190
|
-
expect.any(Buffer),
|
|
191
|
-
expect.objectContaining({
|
|
192
|
-
persistent: true,
|
|
193
|
-
expiration: '4000', // 4000 * 2^0
|
|
194
|
-
headers: expect.objectContaining({ 'x-retry-count': 1 }),
|
|
195
|
-
}),
|
|
196
|
-
);
|
|
197
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
198
|
-
});
|
|
199
|
-
|
|
200
|
-
it('sends to DLQ after max retries', async () => {
|
|
201
|
-
await consumeCallback(makeMessage({ fail: true }, 5));
|
|
202
|
-
|
|
203
|
-
expect(mockChannel.publish).toHaveBeenCalledWith(
|
|
204
|
-
'',
|
|
205
|
-
'jobs.failed',
|
|
206
|
-
expect.any(Buffer),
|
|
207
|
-
expect.objectContaining({
|
|
208
|
-
persistent: true,
|
|
209
|
-
headers: expect.objectContaining({ 'x-retry-count': 5 }),
|
|
210
|
-
}),
|
|
211
|
-
);
|
|
212
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
213
|
-
});
|
|
214
|
-
|
|
215
|
-
it('retries on handler throw (below max retries)', async () => {
|
|
216
|
-
await consumeCallback(makeMessage({ throw: true }, 2));
|
|
217
|
-
|
|
218
|
-
expect(mockChannel.publish).toHaveBeenCalledWith(
|
|
219
|
-
'',
|
|
220
|
-
'jobs.retry',
|
|
221
|
-
expect.any(Buffer),
|
|
222
|
-
expect.objectContaining({
|
|
223
|
-
expiration: '16000', // 4000 * 2^2
|
|
224
|
-
headers: expect.objectContaining({ 'x-retry-count': 3 }),
|
|
225
|
-
}),
|
|
226
|
-
);
|
|
227
|
-
});
|
|
228
|
-
|
|
229
|
-
it('ignores null messages', async () => {
|
|
230
|
-
await consumeCallback(null as unknown as ConsumeMessage);
|
|
231
|
-
expect(mockChannel.ack).not.toHaveBeenCalled();
|
|
232
|
-
expect(mockChannel.nack).not.toHaveBeenCalled();
|
|
233
|
-
});
|
|
234
|
-
|
|
235
|
-
it('NACKs unparseable messages (poison)', async () => {
|
|
236
|
-
const poisonMsg = {
|
|
237
|
-
content: Buffer.from('not-json{{{'),
|
|
238
|
-
properties: { headers: {} },
|
|
239
|
-
fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
|
|
240
|
-
} as unknown as ConsumeMessage;
|
|
241
|
-
|
|
242
|
-
await consumeCallback(poisonMsg);
|
|
243
|
-
expect(mockChannel.nack).toHaveBeenCalledWith(poisonMsg, false, false);
|
|
244
|
-
});
|
|
245
|
-
});
|
|
246
|
-
});
|
|
247
|
-
|
|
248
|
-
describe('listen() manual-ACK mode', () => {
|
|
249
|
-
let consumeCallback: (msg: ConsumeMessage | null) => Promise<void>;
|
|
250
|
-
|
|
251
|
-
function makeMessage(payload: unknown, retryCount = 0): ConsumeMessage {
|
|
252
|
-
return {
|
|
253
|
-
content: Buffer.from(JSON.stringify(payload)),
|
|
254
|
-
properties: {
|
|
255
|
-
headers: { 'x-retry-count': retryCount },
|
|
256
|
-
},
|
|
257
|
-
fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
|
|
258
|
-
} as unknown as ConsumeMessage;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
function setupManualAckListener(
|
|
262
|
-
handler: (msg: unknown, ack: () => void, nack: () => void) => Promise<void>,
|
|
263
|
-
) {
|
|
264
|
-
(mockChannel.consume as jest.Mock).mockImplementationOnce(
|
|
265
|
-
(_queue: string, cb: (msg: ConsumeMessage | null) => Promise<void>) => {
|
|
266
|
-
consumeCallback = cb;
|
|
267
|
-
return Promise.resolve({ consumerTag: 'tag' });
|
|
268
|
-
},
|
|
269
|
-
);
|
|
270
|
-
return MessageQueueService.listen('jobs', handler, { manualAck: true });
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
it('handler receives ack and nack functions', async () => {
|
|
274
|
-
let receivedAck: (() => void) | undefined;
|
|
275
|
-
let receivedNack: (() => void) | undefined;
|
|
276
|
-
|
|
277
|
-
await setupManualAckListener(async (_msg, ack, nack) => {
|
|
278
|
-
receivedAck = ack;
|
|
279
|
-
receivedNack = nack;
|
|
280
|
-
ack();
|
|
281
|
-
});
|
|
282
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
283
|
-
|
|
284
|
-
expect(typeof receivedAck).toBe('function');
|
|
285
|
-
expect(typeof receivedNack).toBe('function');
|
|
286
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
287
|
-
});
|
|
288
|
-
|
|
289
|
-
it('guarded ack — second call is no-op', async () => {
|
|
290
|
-
await setupManualAckListener(async (_msg, ack) => {
|
|
291
|
-
ack();
|
|
292
|
-
ack(); // second call — should be no-op
|
|
293
|
-
});
|
|
294
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
295
|
-
|
|
296
|
-
expect(mockChannel.ack).toHaveBeenCalledTimes(1);
|
|
297
|
-
});
|
|
298
|
-
|
|
299
|
-
it('guarded nack — second call is no-op', async () => {
|
|
300
|
-
await setupManualAckListener(async (_msg, _ack, nack) => {
|
|
301
|
-
nack();
|
|
302
|
-
nack(); // second call — should be no-op
|
|
303
|
-
});
|
|
304
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
305
|
-
|
|
306
|
-
// retryOrDLQ publishes to retry queue then acks — only one round
|
|
307
|
-
expect(mockChannel.publish).toHaveBeenCalledTimes(1);
|
|
308
|
-
expect(mockChannel.ack).toHaveBeenCalledTimes(1);
|
|
309
|
-
});
|
|
310
|
-
|
|
311
|
-
it('ack then nack — nack is no-op (first call wins)', async () => {
|
|
312
|
-
await setupManualAckListener(async (_msg, ack, nack) => {
|
|
313
|
-
ack();
|
|
314
|
-
nack(); // should be no-op since ack already settled
|
|
315
|
-
});
|
|
316
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
317
|
-
|
|
318
|
-
expect(mockChannel.ack).toHaveBeenCalledTimes(1);
|
|
319
|
-
expect(mockChannel.publish).not.toHaveBeenCalled(); // no retry
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
it('handler throws without settling — retries via DLQ path', async () => {
|
|
323
|
-
await setupManualAckListener(async () => {
|
|
324
|
-
throw new Error('boom');
|
|
325
|
-
});
|
|
326
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
327
|
-
|
|
328
|
-
// retryOrDLQ should have been called (publishes to retry queue)
|
|
329
|
-
expect(mockChannel.publish).toHaveBeenCalledWith(
|
|
330
|
-
'',
|
|
331
|
-
'jobs.retry',
|
|
332
|
-
expect.any(Buffer),
|
|
333
|
-
expect.objectContaining({
|
|
334
|
-
persistent: true,
|
|
335
|
-
headers: expect.objectContaining({ 'x-retry-count': 1 }),
|
|
336
|
-
}),
|
|
337
|
-
);
|
|
338
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
339
|
-
});
|
|
340
|
-
|
|
341
|
-
it('handler resolves without settling — treated as failure', async () => {
|
|
342
|
-
await setupManualAckListener(async () => {
|
|
343
|
-
// deliberately not calling ack or nack
|
|
344
|
-
});
|
|
345
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
346
|
-
|
|
347
|
-
// retryOrDLQ should have been called
|
|
348
|
-
expect(mockChannel.publish).toHaveBeenCalledWith(
|
|
349
|
-
'',
|
|
350
|
-
'jobs.retry',
|
|
351
|
-
expect.any(Buffer),
|
|
352
|
-
expect.objectContaining({
|
|
353
|
-
headers: expect.objectContaining({ 'x-retry-count': 1 }),
|
|
354
|
-
}),
|
|
355
|
-
);
|
|
356
|
-
expect(mockChannel.ack).toHaveBeenCalled();
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
it('handler throws after acking — no double retry', async () => {
|
|
360
|
-
await setupManualAckListener(async (_msg, ack) => {
|
|
361
|
-
ack();
|
|
362
|
-
throw new Error('post-ack error');
|
|
363
|
-
});
|
|
364
|
-
await consumeCallback(makeMessage({ data: 1 }));
|
|
365
|
-
|
|
366
|
-
// ack was called but no retry should happen (already settled)
|
|
367
|
-
expect(mockChannel.ack).toHaveBeenCalledTimes(1);
|
|
368
|
-
expect(mockChannel.publish).not.toHaveBeenCalled();
|
|
369
|
-
});
|
|
370
|
-
});
|
|
371
|
-
|
|
372
|
-
describe('listen() prefetch', () => {
|
|
373
|
-
it('sets channel.prefetch when opts.prefetch is provided', async () => {
|
|
374
|
-
await MessageQueueService.listen('jobs', async () => true, { prefetch: 10 });
|
|
375
|
-
expect(mockChannel.prefetch).toHaveBeenCalledWith(10);
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
it('does not call prefetch when opts.prefetch is not provided', async () => {
|
|
379
|
-
await MessageQueueService.listen('jobs', async () => true);
|
|
380
|
-
expect(mockChannel.prefetch).not.toHaveBeenCalled();
|
|
381
|
-
});
|
|
382
|
-
|
|
383
|
-
it('sets prefetch before consuming', async () => {
|
|
384
|
-
const callOrder: string[] = [];
|
|
385
|
-
(mockChannel.prefetch as jest.Mock).mockImplementation(() => {
|
|
386
|
-
callOrder.push('prefetch');
|
|
387
|
-
return Promise.resolve(undefined);
|
|
388
|
-
});
|
|
389
|
-
(mockChannel.consume as jest.Mock).mockImplementation(() => {
|
|
390
|
-
callOrder.push('consume');
|
|
391
|
-
return Promise.resolve({ consumerTag: 'tag' });
|
|
392
|
-
});
|
|
393
|
-
|
|
394
|
-
await MessageQueueService.listen('jobs', async () => true, { prefetch: 5 });
|
|
395
|
-
|
|
396
|
-
expect(callOrder).toEqual(['prefetch', 'consume']);
|
|
397
|
-
});
|
|
398
|
-
});
|
|
399
|
-
|
|
400
|
-
describe('close()', () => {
|
|
401
|
-
it('closes all tracked connections', async () => {
|
|
402
|
-
// Create a listener connection
|
|
403
|
-
await MessageQueueService.listen('q', async () => true);
|
|
404
|
-
|
|
405
|
-
await MessageQueueService.close();
|
|
406
|
-
expect(mockConnection.close).toHaveBeenCalled();
|
|
407
|
-
});
|
|
408
|
-
});
|
|
409
|
-
|
|
410
|
-
describe('service shape', () => {
|
|
411
|
-
it('exports the required service interface', () => {
|
|
412
|
-
expect(MessageQueueService.name).toBe('message-queue');
|
|
413
|
-
expect(typeof MessageQueueService.init).toBe('function');
|
|
414
|
-
expect(typeof MessageQueueService.publish).toBe('function');
|
|
415
|
-
expect(typeof MessageQueueService.listen).toBe('function');
|
|
416
|
-
expect(typeof MessageQueueService.close).toBe('function');
|
|
417
|
-
});
|
|
418
|
-
});
|
|
419
|
-
});
|