@boopkit/cap-background-jobs 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/capability.json CHANGED
@@ -4,23 +4,21 @@
4
4
  "type": "application",
5
5
  "package": "@boopkit/cap-background-jobs",
6
6
  "requires": ["services"],
7
- "dependencies": ["amqplib@^0.10"],
8
- "devDependencies": ["@types/amqplib@^0.10"],
7
+ "dependencies": ["amqplib@^0.10", "minimist@^1.2"],
8
+ "devDependencies": ["@types/amqplib@^0.10", "@types/minimist@^1.2"],
9
9
  "env": [
10
10
  { "name": "CLOUDAMQP_URL", "description": "CloudAMQP connection URL (amqp://...)", "required": "boot" },
11
11
  { "name": "QUEUE_PREFIX", "description": "Queue name prefix for environment isolation (e.g. dev/mike)" },
12
12
  { "name": "MAX_MESSAGE_RETRIES", "description": "Maximum retry attempts before dead-lettering (default: 5)" }
13
13
  ],
14
14
  "scripts": {
15
- "worker": { "cmd": "ts-node src/worker.ts", "description": "Start the background job worker process" },
16
- "scheduler": { "cmd": "ts-node src/scheduler.ts", "description": "Publish trigger messages to job queues (run via Heroku Scheduler)" }
15
+ "worker": { "cmd": "tsx src/worker.ts", "description": "Start the background job worker process" },
16
+ "scheduler": { "cmd": "tsx src/scheduler.ts", "description": "Publish trigger messages to job queues supports --interval and --job filtering" }
17
17
  },
18
18
  "files": [
19
19
  "src/scheduler.ts",
20
20
  "src/worker.ts",
21
- "src/services/message-queue.service.ts",
22
21
  "src/controllers/example-job.controller.ts",
23
- "tests/services/message-queue.service.test.ts",
24
22
  "tests/controllers/example-job.controller.test.ts",
25
23
  "reference/background-jobs.ref.md"
26
24
  ],
@@ -32,7 +30,7 @@
32
30
  "Add CLOUDAMQP_URL to .env",
33
31
  "Set QUEUE_PREFIX for local dev isolation (e.g. dev/yourname)",
34
32
  "Add worker entry to Procfile: worker: npm run worker",
35
- "Configure Heroku Scheduler to run npm run scheduler on desired interval"
33
+ "Configure Heroku Scheduler to run 'npm run scheduler -- run-jobs --interval <tag>' on desired interval (e.g. 10-minutes, 60-minutes)"
36
34
  ],
37
35
  "verify": "npm run worker"
38
36
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@boopkit/cap-background-jobs",
3
- "version": "0.1.0",
3
+ "version": "0.2.1",
4
4
  "description": "Background job processing with RabbitMQ for boopkit-scaffolded projects",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -20,10 +20,12 @@
20
20
  "build": "tsc -p tsconfig.json"
21
21
  },
22
22
  "dependencies": {
23
- "amqplib": "^0.10.0"
23
+ "amqplib": "^0.10.0",
24
+ "minimist": "^1.2.8"
24
25
  },
25
26
  "devDependencies": {
26
27
  "@types/amqplib": "^0.10.0",
28
+ "@types/minimist": "^1.2.5",
27
29
  "typescript": "^5.9.3"
28
30
  },
29
31
  "license": "ISC",
@@ -33,7 +33,33 @@ Three separate OS processes share one MQ service:
33
33
  |---------|------------|------|-----------|
34
34
  | **Web** | `src/server.ts` | Publishes messages as a side effect of HTTP requests | Long-running |
35
35
  | **Worker** | `src/worker.ts` | Consumes messages via registered controllers | Long-running |
36
- | **Scheduler** | `src/scheduler.ts` | Publishes trigger messages, then exits | One-shot (cron) |
36
+ | **Scheduler** | `src/scheduler.ts` | Publishes trigger messages filtered by interval/job, then exits | One-shot (cron) |
37
+
38
+ ---
39
+
40
+ ## Scheduler CLI
41
+
42
+ The scheduler supports subcommands and options for selecting which jobs to trigger:
43
+
44
+ ```bash
45
+ # Publish all jobs tagged with the '10-minutes' interval
46
+ npm run scheduler -- run-jobs --interval 10-minutes
47
+
48
+ # Publish a specific job regardless of interval
49
+ npm run scheduler -- run-jobs --job example-job
50
+
51
+ # Combine filters
52
+ npm run scheduler -- run-jobs --interval 60-minutes --job reconciler
53
+
54
+ # List all registered jobs with their queue and interval
55
+ npm run scheduler -- list
56
+ ```
57
+
58
+ Each job in the `jobs` array has:
59
+ - `name` — human-readable identifier (for `--job` filtering)
60
+ - `queue` — target MQ queue name
61
+ - `interval` — tag for external scheduler filtering (e.g. `10-minutes`, `60-minutes`)
62
+ - `payload` — message body published to the queue
37
63
 
38
64
  ---
39
65
 
@@ -61,6 +87,12 @@ Each queue has a corresponding direct exchange for routing. Retry messages dead-
61
87
 
62
88
  The web process only opens an AMQP connection if it actually publishes a message.
63
89
 
90
+ > **Package vs scaffold:** The `MessageQueueService` implementation lives in the
91
+ > `@boopkit/cap-background-jobs` package (`src/message-queue.service.ts`). The
92
+ > scaffold's `src/services/message-queue.service.ts` is a thin re-export so
93
+ > existing local imports keep working. Package updates flow to consuming
94
+ > projects via `npm update` without re-scaffolding.
95
+
64
96
  ---
65
97
 
66
98
  ## Consumer Registration
@@ -130,7 +162,9 @@ await MessageQueueService.listen('queue', handler, { prefetch: 10 });
130
162
 
131
163
  1. **CloudAMQP addon**: `heroku addons:create cloudamqp` — sets `CLOUDAMQP_URL` automatically
132
164
  2. **Procfile**: Add `worker: npm run worker`
133
- 3. **Heroku Scheduler**: Configure `npm run scheduler` on the desired interval
165
+ 3. **Heroku Scheduler**: Configure intervals, e.g.:
166
+ - Every 10 min: `npm run scheduler -- run-jobs --interval 10-minutes`
167
+ - Every hour: `npm run scheduler -- run-jobs --interval 60-minutes`
134
168
  4. **Scale worker**: `heroku ps:scale worker=1`
135
169
 
136
170
  ---
@@ -153,8 +187,14 @@ If drain exceeds 30s, the worker force-exits with code 1.
153
187
  | File | Purpose |
154
188
  |------|---------|
155
189
  | `src/worker.ts` | Worker entry point — boots services, registers consumers, handles shutdown |
156
- | `src/scheduler.ts` | Scheduler entry point — publishes trigger messages, exits |
157
- | `src/services/message-queue.service.ts` | MQ service publish, listen (auto/manual ACK), topology, retry/DLQ |
190
+ | `src/scheduler.ts` | Scheduler entry point — CLI with interval/job filtering, publishes trigger messages, exits |
191
+ | `src/services/message-queue.service.ts` | Re-exports `MessageQueueService` from `@boopkit/cap-background-jobs` |
158
192
  | `src/controllers/example-job.controller.ts` | Example consumer — copyable template for new job handlers |
159
- | `tests/services/message-queue.service.test.ts` | MQ service unit tests |
160
193
  | `tests/controllers/example-job.controller.test.ts` | Example controller unit tests |
194
+
195
+ ### Package-level files
196
+
197
+ | File | Purpose |
198
+ |------|---------||
199
+ | `@boopkit/cap-background-jobs` `src/message-queue.service.ts` | Core MQ service — publish, listen (auto/manual ACK), topology, retry/DLQ |
200
+ | `@boopkit/cap-background-jobs` `tests/message-queue.service.test.ts` | MQ service unit tests |
@@ -1,12 +1,28 @@
1
1
  /** @boopkit/cap-background-jobs — Scheduler entry point for triggering background jobs. */
2
2
 
3
3
  import { loadServices } from '@services';
4
+ import minimist from 'minimist';
4
5
  import MessageQueueService from './services/message-queue.service';
5
6
 
6
- // Static job registration — add new {queue, payload} entries here
7
- const jobs = [
7
+ interface JobConfig {
8
+ /** Human-readable job name (used for --job filtering). */
9
+ name: string;
10
+ /** Target queue name to publish to. */
11
+ queue: string;
12
+ /** Interval tag for external scheduler filtering (e.g. '10-minutes', '60-minutes'). */
13
+ interval: string;
14
+ /** Message payload to publish. */
15
+ payload: { task: string; data?: Record<string, unknown> };
16
+ }
17
+
18
+ // Static job registration — add new entries here.
19
+ // The `interval` field tags each job so an external scheduler can run subsets:
20
+ // npm run scheduler -- run-jobs --interval 10-minutes
21
+ const jobs: JobConfig[] = [
8
22
  {
23
+ name: 'example-job',
9
24
  queue: 'example-job',
25
+ interval: '10-minutes',
10
26
  payload: {
11
27
  task: 'scheduled-run',
12
28
  data: {
@@ -17,10 +33,28 @@ const jobs = [
17
33
  },
18
34
  ];
19
35
 
20
- async function run() {
21
- await loadServices();
36
+ /**
37
+ * Publish trigger messages for jobs matching the given filters.
38
+ */
39
+ async function runJobs(
40
+ opts: { interval?: string; job?: string; await?: boolean },
41
+ ): Promise<void> {
42
+ let selected = jobs;
43
+
44
+ if (opts.interval) {
45
+ selected = selected.filter((j) => j.interval === opts.interval);
46
+ }
47
+ if (opts.job) {
48
+ selected = selected.filter((j) => j.name === opts.job);
49
+ }
22
50
 
23
- for (const job of jobs) {
51
+ console.log(
52
+ `[scheduler] ${selected.length} job(s) selected` +
53
+ (opts.interval ? ` for interval "${opts.interval}"` : '') +
54
+ (opts.job ? ` matching job "${opts.job}"` : ''),
55
+ );
56
+
57
+ for (const job of selected) {
24
58
  const ok = await MessageQueueService.publish(job.queue, job.payload);
25
59
  if (ok) {
26
60
  console.log(`[scheduler] published to "${job.queue}": ${job.payload.task}`);
@@ -28,12 +62,60 @@ async function run() {
28
62
  console.error(`[scheduler] failed to publish to "${job.queue}"`);
29
63
  }
30
64
  }
65
+ }
66
+
67
+ function printUsage(): void {
68
+ console.log(`Usage: npm run scheduler -- <command> [options]
69
+
70
+ Commands:
71
+ run-jobs Publish trigger messages to matching job queues
72
+ list List all registered jobs
73
+
74
+ Options:
75
+ --interval <tag> Filter jobs by interval tag (e.g. 10-minutes, 60-minutes)
76
+ --job <name> Filter jobs by name
77
+ `);
78
+ }
79
+
80
+ async function main(args: string[], opts: Record<string, unknown>): Promise<void> {
81
+ const command = (args[0] ?? '').toLowerCase();
82
+
83
+ if (!command) {
84
+ printUsage();
85
+ process.exit(0);
86
+ }
87
+
88
+ await loadServices();
89
+
90
+ switch (command) {
91
+ case 'run-jobs': {
92
+ await runJobs({
93
+ interval: opts.interval as string | undefined,
94
+ job: opts.job as string | undefined,
95
+ });
96
+ break;
97
+ }
98
+
99
+ case 'list': {
100
+ console.log('[scheduler] registered jobs:');
101
+ for (const job of jobs) {
102
+ console.log(` ${job.name} queue=${job.queue} interval=${job.interval}`);
103
+ }
104
+ break;
105
+ }
106
+
107
+ default:
108
+ console.error(`[scheduler] unknown command: ${command}`);
109
+ printUsage();
110
+ process.exit(1);
111
+ }
31
112
 
32
113
  await MessageQueueService.close();
33
114
  process.exit(0);
34
115
  }
35
116
 
36
- run().catch((err) => {
117
+ const argv = minimist(process.argv.slice(2));
118
+ main(argv._, argv).catch((err) => {
37
119
  console.error('[scheduler] failed:', err);
38
120
  process.exit(1);
39
121
  });
@@ -50,7 +50,7 @@ That's it. The MQ service creates the queue topology (main + retry + DLQ) automa
50
50
  Use `MessageQueueService.publish(queue, payload)` from any process:
51
51
 
52
52
  ```typescript
53
- import MessageQueueService from './services/message-queue.service';
53
+ import { MessageQueueService } from '@boopkit/cap-background-jobs';
54
54
 
55
55
  await MessageQueueService.publish('my-job', {
56
56
  task: 'process-upload',
@@ -71,13 +71,27 @@ await MessageQueueService.publish('my-job', {
71
71
  Add an entry to the `jobs` array in `src/scheduler.ts`:
72
72
 
73
73
  ```typescript
74
- const jobs = [
75
- { queue: 'example-job', payload: { task: 'scheduled-run', data: { ... } } },
76
- { queue: 'my-job', payload: { task: 'nightly-sync', data: { ... } } },
74
+ const jobs: JobConfig[] = [
75
+ { name: 'example-job', queue: 'example-job', interval: '10-minutes', payload: { task: 'scheduled-run', data: { ... } } },
76
+ { name: 'nightly-sync', queue: 'my-job', interval: '60-minutes', payload: { task: 'nightly-sync', data: { ... } } },
77
77
  ];
78
78
  ```
79
79
 
80
- The scheduler publishes all jobs and exits. Configure Heroku Scheduler to run `npm run scheduler` at the desired interval.
80
+ Each job has an `interval` tag. Configure Heroku Scheduler to run different intervals:
81
+
82
+ ```bash
83
+ # Every 10 minutes
84
+ npm run scheduler -- run-jobs --interval 10-minutes
85
+
86
+ # Every hour
87
+ npm run scheduler -- run-jobs --interval 60-minutes
88
+
89
+ # Specific job only
90
+ npm run scheduler -- run-jobs --job nightly-sync
91
+
92
+ # List all registered jobs
93
+ npm run scheduler -- list
94
+ ```
81
95
 
82
96
  ## Auto-ACK vs manual-ACK
83
97
 
File without changes
@@ -1,316 +0,0 @@
1
- /** @boopkit/cap-background-jobs — RabbitMQ message queue service. */
2
-
3
- import amqplib from 'amqplib';
4
- import type { ChannelModel, Channel, ConsumeMessage } from 'amqplib';
5
-
6
- const QUEUE_PREFIX = process.env.QUEUE_PREFIX ?? '';
7
- const MAX_RETRIES = parseInt(process.env.MAX_MESSAGE_RETRIES ?? '5', 10);
8
- const BACKOFF_BASE_MS = 4000;
9
- const BACKOFF_MAX_MS = 60000;
10
-
11
- /** All open connections tracked for shutdown cleanup. */
12
- const connections: ChannelModel[] = [];
13
-
14
- /** Lazy-initialized publish connection and channel. */
15
- let publishConnection: ChannelModel | null = null;
16
- let publishChannel: Channel | null = null;
17
-
18
- const MessageQueueService = {
19
- name: 'message-queue',
20
- init,
21
- publish,
22
- listen,
23
- close,
24
- };
25
-
26
- export default MessageQueueService;
27
-
28
- /**
29
- * Validate that CLOUDAMQP_URL is set. Does NOT open a connection —
30
- * publish connection is lazy (first publish call), listener connections
31
- * are created at listen() time.
32
- */
33
- function init(): void {
34
- if (!process.env.CLOUDAMQP_URL) {
35
- throw new Error(
36
- 'E: CLOUDAMQP_URL is not set. ' +
37
- 'R: The message queue service requires a CloudAMQP connection URL. ' +
38
- 'R: Set CLOUDAMQP_URL in your environment (e.g. .env file).',
39
- );
40
- }
41
- }
42
-
43
- /**
44
- * Publish a message to the given queue.
45
- * Lazy-initializes a persistent connection on first call, reuses it
46
- * across subsequent calls. Returns true on success, false on failure.
47
- */
48
- async function publish(queue: string, payload: unknown): Promise<boolean> {
49
- const queueName = prefixedName(queue);
50
-
51
- try {
52
- const channel = await getPublishChannel();
53
- await channel.assertQueue(queueName, { durable: true });
54
-
55
- const body = Buffer.from(JSON.stringify(payload));
56
- const ok = channel.sendToQueue(queueName, body, {
57
- persistent: true,
58
- messageId: crypto.randomUUID(),
59
- });
60
-
61
- if (!ok) {
62
- // Channel buffer full — wait for drain
63
- await new Promise<void>((resolve) => channel.once('drain', resolve));
64
- }
65
-
66
- return true;
67
- } catch (err) {
68
- console.error(`[message-queue] publish error on "${queue}":`, err);
69
- resetPublishConnection();
70
- return false;
71
- }
72
- }
73
-
74
- /** Options for listen(). */
75
- interface ListenOpts {
76
- /** When true, handler receives guarded ack/nack functions instead of returning boolean. */
77
- manualAck?: boolean;
78
- /** Channel prefetch count — limits concurrent message delivery for backpressure. */
79
- prefetch?: number;
80
- }
81
-
82
- /**
83
- * Listen on a queue with auto-ACK or manual-ACK semantics.
84
- *
85
- * Asserts full queue topology on registration:
86
- * - Main queue: `{prefix}/{name}`
87
- * - Retry queue: `{prefix}/{name}.retry` (TTL redelivery to main)
88
- * - DLQ: `{prefix}/{name}.failed` (dead-letter exchange)
89
- *
90
- * **Auto-ACK (default):** Handler returns true → ACK, false → retry with backoff,
91
- * throws → NACK to DLQ after max retries.
92
- *
93
- * **Manual-ACK (opts.manualAck = true):** Handler receives guarded ack()/nack()
94
- * functions. First call wins — subsequent calls are no-ops. If handler resolves
95
- * or throws without settling → retry/DLQ after max retries.
96
- */
97
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
98
- async function listen(queue: string, handler: (...args: any[]) => Promise<any>, opts?: ListenOpts): Promise<void> {
99
- const connection = await amqplib.connect(process.env.CLOUDAMQP_URL!);
100
- connections.push(connection);
101
-
102
- const channel = await connection.createChannel();
103
- const { mainQueue, retryQueue, failedQueue } = await assertTopology(channel, queue);
104
-
105
- if (opts?.prefetch != null) {
106
- await channel.prefetch(opts.prefetch);
107
- }
108
-
109
- await channel.consume(mainQueue, async (raw) => {
110
- if (!raw) return;
111
-
112
- const retryCount = getRetryCount(raw);
113
- let parsed: unknown;
114
-
115
- try {
116
- parsed = JSON.parse(raw.content.toString());
117
- } catch {
118
- // Poison message — ACK to prevent infinite redelivery
119
- console.error(`[message-queue] unparseable message on "${queue}", sending to DLQ`);
120
- channel.nack(raw, false, false);
121
- return;
122
- }
123
-
124
- if (opts?.manualAck) {
125
- // Manual-ACK: handler receives guarded ack/nack
126
- let settled = false;
127
-
128
- const guardedAck = (): void => {
129
- if (settled) {
130
- console.warn(`[message-queue] duplicate ack on "${queue}" — ignored`);
131
- return;
132
- }
133
- settled = true;
134
- channel.ack(raw);
135
- };
136
-
137
- const guardedNack = async (): Promise<void> => {
138
- if (settled) {
139
- console.warn(`[message-queue] duplicate nack on "${queue}" — ignored`);
140
- return;
141
- }
142
- settled = true;
143
- await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
144
- };
145
-
146
- try {
147
- await (handler as (msg: unknown, ack: () => void, nack: () => void) => Promise<void>)(parsed, guardedAck, guardedNack);
148
-
149
- if (!settled) {
150
- console.warn(`[message-queue] handler resolved without settling on "${queue}" — treating as failure`);
151
- await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
152
- }
153
- } catch (err) {
154
- console.error(`[message-queue] handler threw on "${queue}":`, err);
155
- if (!settled) {
156
- await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
157
- }
158
- }
159
- } else {
160
- // Auto-ACK: handler returns boolean
161
- try {
162
- const ok = await (handler as (msg: unknown) => Promise<boolean>)(parsed);
163
-
164
- if (ok) {
165
- channel.ack(raw);
166
- } else {
167
- await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
168
- }
169
- } catch (err) {
170
- console.error(`[message-queue] handler threw on "${queue}":`, err);
171
- await retryOrDLQ(channel, raw, retryCount, retryQueue, failedQueue);
172
- }
173
- }
174
- }, { noAck: false });
175
- }
176
-
177
- /** Close all tracked connections (publish + listeners). */
178
- async function close(): Promise<void> {
179
- resetPublishConnection();
180
-
181
- const closing = connections.map(async (conn) => {
182
- try {
183
- await conn.close();
184
- } catch {
185
- // Already closed — ignore
186
- }
187
- });
188
- await Promise.all(closing);
189
- connections.length = 0;
190
- }
191
-
192
- // ---------------------------------------------------------------------------
193
- // Internal helpers
194
- // ---------------------------------------------------------------------------
195
-
196
- function prefixedName(name: string): string {
197
- return QUEUE_PREFIX ? `${QUEUE_PREFIX}/${name}` : name;
198
- }
199
-
200
- /** Lazy-create or return the existing publish channel. */
201
- async function getPublishChannel(): Promise<Channel> {
202
- if (publishChannel) return publishChannel;
203
-
204
- const conn = await amqplib.connect(process.env.CLOUDAMQP_URL!);
205
- connections.push(conn);
206
- publishConnection = conn;
207
-
208
- conn.on('error', () => resetPublishConnection());
209
- conn.on('close', () => resetPublishConnection());
210
-
211
- publishChannel = await conn.createChannel();
212
- return publishChannel;
213
- }
214
-
215
- /** Clear cached publish connection so the next call reconnects. */
216
- function resetPublishConnection(): void {
217
- if (publishConnection) {
218
- publishConnection.close().catch(() => { /* already closing */ });
219
- const idx = connections.indexOf(publishConnection);
220
- if (idx !== -1) connections.splice(idx, 1);
221
- }
222
- publishConnection = null;
223
- publishChannel = null;
224
- }
225
-
226
- /**
227
- * Assert main + retry + DLQ queues for a given queue name.
228
- *
229
- * Topology:
230
- * main ──(dead-letter)──▸ {name}.failed (via exchange)
231
- * retry ──(TTL expiry)──▸ main (via dead-letter back to main exchange)
232
- */
233
- async function assertTopology(
234
- channel: Channel,
235
- queue: string,
236
- ): Promise<{ mainQueue: string; retryQueue: string; failedQueue: string }> {
237
- const mainQueue = prefixedName(queue);
238
- const retryQueue = `${mainQueue}.retry`;
239
- const failedQueue = `${mainQueue}.failed`;
240
-
241
- const mainExchange = `${mainQueue}.exchange`;
242
- const retryExchange = `${retryQueue}.exchange`;
243
- const failedExchange = `${failedQueue}.exchange`;
244
-
245
- // Exchanges
246
- await channel.assertExchange(mainExchange, 'direct', { durable: true });
247
- await channel.assertExchange(retryExchange, 'direct', { durable: true });
248
- await channel.assertExchange(failedExchange, 'direct', { durable: true });
249
-
250
- // Main queue — dead-letters to failed exchange.
251
- // Note: the DLX here is primarily exercised for poison messages (unparseable
252
- // JSON) which are nacked without requeue. Normal failures go through
253
- // retryOrDLQ() which manually publishes to the retry/failed queue and acks.
254
- await channel.assertQueue(mainQueue, {
255
- durable: true,
256
- deadLetterExchange: failedExchange,
257
- });
258
- await channel.bindQueue(mainQueue, mainExchange, '');
259
-
260
- // Retry queue — messages TTL then dead-letter back to main exchange
261
- await channel.assertQueue(retryQueue, {
262
- durable: true,
263
- deadLetterExchange: mainExchange,
264
- });
265
- await channel.bindQueue(retryQueue, retryExchange, '');
266
-
267
- // Failed (DLQ) — terminal resting place
268
- await channel.assertQueue(failedQueue, { durable: true });
269
- await channel.bindQueue(failedQueue, failedExchange, '');
270
-
271
- return { mainQueue, retryQueue, failedQueue };
272
- }
273
-
274
- function getRetryCount(msg: ConsumeMessage): number {
275
- const headers = msg.properties.headers ?? {};
276
- return typeof headers['x-retry-count'] === 'number' ? headers['x-retry-count'] : 0;
277
- }
278
-
279
- function computeBackoff(retryCount: number): number {
280
- return Math.min(BACKOFF_BASE_MS * Math.pow(2, retryCount), BACKOFF_MAX_MS);
281
- }
282
-
283
- /**
284
- * Retry a message (publish to retry queue with incremented count and TTL)
285
- * or send to DLQ if max retries exceeded.
286
- */
287
- async function retryOrDLQ(
288
- channel: Channel,
289
- raw: ConsumeMessage,
290
- retryCount: number,
291
- retryQueue: string,
292
- failedQueue: string,
293
- ): Promise<void> {
294
- if (retryCount >= MAX_RETRIES) {
295
- console.error(
296
- `[message-queue] max retries (${MAX_RETRIES}) exceeded, moving to DLQ: ${failedQueue}`,
297
- );
298
- channel.publish('', failedQueue, raw.content, {
299
- persistent: true,
300
- headers: { ...raw.properties.headers, 'x-retry-count': retryCount },
301
- });
302
- channel.ack(raw);
303
- return;
304
- }
305
-
306
- const ttl = computeBackoff(retryCount);
307
- channel.publish('', retryQueue, raw.content, {
308
- persistent: true,
309
- expiration: String(ttl),
310
- headers: {
311
- ...raw.properties.headers,
312
- 'x-retry-count': retryCount + 1,
313
- },
314
- });
315
- channel.ack(raw);
316
- }
File without changes
@@ -1,419 +0,0 @@
1
- import type { ConsumeMessage } from 'amqplib';
2
-
3
- // Mock amqplib before importing the service
4
- const mockChannel: Record<string, jest.Mock> = {
5
- assertExchange: jest.fn().mockResolvedValue(undefined),
6
- assertQueue: jest.fn().mockResolvedValue({ queue: 'test' }),
7
- bindQueue: jest.fn().mockResolvedValue(undefined),
8
- sendToQueue: jest.fn().mockReturnValue(true),
9
- publish: jest.fn().mockReturnValue(true),
10
- consume: jest.fn().mockResolvedValue({ consumerTag: 'test-tag' }),
11
- prefetch: jest.fn().mockResolvedValue(undefined),
12
- ack: jest.fn(),
13
- nack: jest.fn(),
14
- once: jest.fn(),
15
- };
16
-
17
- const mockConnection: Record<string, jest.Mock> = {
18
- createChannel: jest.fn().mockResolvedValue(mockChannel),
19
- close: jest.fn().mockResolvedValue(undefined),
20
- on: jest.fn(),
21
- };
22
-
23
- jest.mock('amqplib', () => ({
24
- connect: jest.fn().mockResolvedValue(mockConnection),
25
- }));
26
-
27
- import amqplib from 'amqplib';
28
- import MessageQueueService from '../../src/services/message-queue.service';
29
-
30
- describe('MessageQueueService', () => {
31
- const ORIGINAL_ENV = process.env;
32
-
33
- beforeEach(() => {
34
- jest.clearAllMocks();
35
- process.env = { ...ORIGINAL_ENV, CLOUDAMQP_URL: 'amqp://test:test@localhost/test' };
36
- });
37
-
38
- afterEach(() => {
39
- process.env = ORIGINAL_ENV;
40
- });
41
-
42
- describe('init()', () => {
43
- it('succeeds when CLOUDAMQP_URL is set', () => {
44
- expect(() => MessageQueueService.init()).not.toThrow();
45
- });
46
-
47
- it('throws when CLOUDAMQP_URL is missing', () => {
48
- delete process.env.CLOUDAMQP_URL;
49
- expect(() => MessageQueueService.init()).toThrow('CLOUDAMQP_URL is not set');
50
- });
51
- });
52
-
53
- describe('publish()', () => {
54
- it('connects lazily on first publish', async () => {
55
- const result = await MessageQueueService.publish('test-queue', { foo: 'bar' });
56
-
57
- expect(result).toBe(true);
58
- expect(amqplib.connect).toHaveBeenCalledWith(process.env.CLOUDAMQP_URL);
59
- expect(mockConnection.createChannel).toHaveBeenCalled();
60
- });
61
-
62
- it('publishes with persistent flag and messageId', async () => {
63
- await MessageQueueService.publish('test-queue', { data: 1 });
64
-
65
- expect(mockChannel.sendToQueue).toHaveBeenCalledWith(
66
- 'test-queue',
67
- expect.any(Buffer),
68
- expect.objectContaining({
69
- persistent: true,
70
- messageId: expect.any(String),
71
- }),
72
- );
73
- });
74
-
75
- it('serializes payload as JSON', async () => {
76
- const payload = { key: 'value', count: 42 };
77
- await MessageQueueService.publish('test-queue', payload);
78
-
79
- const sentBuffer = (mockChannel.sendToQueue as jest.Mock).mock.calls[0][1];
80
- expect(JSON.parse(sentBuffer.toString())).toEqual(payload);
81
- });
82
-
83
- it('returns false on error', async () => {
84
- (mockChannel.sendToQueue as jest.Mock).mockImplementationOnce(() => {
85
- throw new Error('channel closed');
86
- });
87
-
88
- const result = await MessageQueueService.publish('test-queue', {});
89
- expect(result).toBe(false);
90
- });
91
-
92
- it('waits for drain when channel buffer is full', async () => {
93
- (mockChannel.sendToQueue as jest.Mock).mockReturnValueOnce(false);
94
- (mockChannel.once as jest.Mock).mockImplementationOnce(
95
- (_event: string, cb: () => void) => { cb(); },
96
- );
97
-
98
- const result = await MessageQueueService.publish('test-queue', {});
99
-
100
- expect(result).toBe(true);
101
- expect(mockChannel.once).toHaveBeenCalledWith('drain', expect.any(Function));
102
- });
103
-
104
- // Note: QUEUE_PREFIX is captured once at module load time via const.
105
- // It cannot be tested by setting process.env after import. The prefix
106
- // behavior is verified through listen() topology assertions instead.
107
- });
108
-
109
- describe('listen()', () => {
110
- it('creates a new connection per listener', async () => {
111
- await MessageQueueService.listen('jobs', async () => true);
112
-
113
- expect(amqplib.connect).toHaveBeenCalledWith(process.env.CLOUDAMQP_URL);
114
- expect(mockConnection.createChannel).toHaveBeenCalled();
115
- });
116
-
117
- it('asserts full queue topology (main + retry + DLQ)', async () => {
118
- await MessageQueueService.listen('jobs', async () => true);
119
-
120
- // Exchanges
121
- expect(mockChannel.assertExchange).toHaveBeenCalledWith(
122
- 'jobs.exchange', 'direct', { durable: true },
123
- );
124
- expect(mockChannel.assertExchange).toHaveBeenCalledWith(
125
- 'jobs.retry.exchange', 'direct', { durable: true },
126
- );
127
- expect(mockChannel.assertExchange).toHaveBeenCalledWith(
128
- 'jobs.failed.exchange', 'direct', { durable: true },
129
- );
130
-
131
- // Queues
132
- expect(mockChannel.assertQueue).toHaveBeenCalledWith(
133
- 'jobs',
134
- expect.objectContaining({ durable: true, deadLetterExchange: 'jobs.failed.exchange' }),
135
- );
136
- expect(mockChannel.assertQueue).toHaveBeenCalledWith(
137
- 'jobs.retry',
138
- expect.objectContaining({ durable: true, deadLetterExchange: 'jobs.exchange' }),
139
- );
140
- expect(mockChannel.assertQueue).toHaveBeenCalledWith(
141
- 'jobs.failed',
142
- expect.objectContaining({ durable: true }),
143
- );
144
- });
145
-
146
- it('sets up a consumer on the main queue', async () => {
147
- await MessageQueueService.listen('jobs', async () => true);
148
- expect(mockChannel.consume).toHaveBeenCalledWith('jobs', expect.any(Function), { noAck: false });
149
- });
150
-
151
- describe('message dispatch', () => {
152
- let consumeCallback: (msg: ConsumeMessage | null) => Promise<void>;
153
-
154
- beforeEach(async () => {
155
- (mockChannel.consume as jest.Mock).mockImplementationOnce(
156
- (_queue: string, cb: (msg: ConsumeMessage | null) => Promise<void>) => {
157
- consumeCallback = cb;
158
- return Promise.resolve({ consumerTag: 'tag' });
159
- },
160
- );
161
- await MessageQueueService.listen('jobs', async (msg) => {
162
- if ((msg as any).fail) return false;
163
- if ((msg as any).throw) throw new Error('boom');
164
- return true;
165
- });
166
- });
167
-
168
- function makeMessage(payload: unknown, retryCount = 0): ConsumeMessage {
169
- return {
170
- content: Buffer.from(JSON.stringify(payload)),
171
- properties: {
172
- headers: { 'x-retry-count': retryCount },
173
- },
174
- fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
175
- } as unknown as ConsumeMessage;
176
- }
177
-
178
- it('ACKs when handler returns true', async () => {
179
- await consumeCallback(makeMessage({ ok: true }));
180
- expect(mockChannel.ack).toHaveBeenCalled();
181
- });
182
-
183
- it('retries when handler returns false', async () => {
184
- await consumeCallback(makeMessage({ fail: true }));
185
-
186
- // Should publish to retry queue with incremented retry count
187
- expect(mockChannel.publish).toHaveBeenCalledWith(
188
- '',
189
- 'jobs.retry',
190
- expect.any(Buffer),
191
- expect.objectContaining({
192
- persistent: true,
193
- expiration: '4000', // 4000 * 2^0
194
- headers: expect.objectContaining({ 'x-retry-count': 1 }),
195
- }),
196
- );
197
- expect(mockChannel.ack).toHaveBeenCalled();
198
- });
199
-
200
- it('sends to DLQ after max retries', async () => {
201
- await consumeCallback(makeMessage({ fail: true }, 5));
202
-
203
- expect(mockChannel.publish).toHaveBeenCalledWith(
204
- '',
205
- 'jobs.failed',
206
- expect.any(Buffer),
207
- expect.objectContaining({
208
- persistent: true,
209
- headers: expect.objectContaining({ 'x-retry-count': 5 }),
210
- }),
211
- );
212
- expect(mockChannel.ack).toHaveBeenCalled();
213
- });
214
-
215
- it('retries on handler throw (below max retries)', async () => {
216
- await consumeCallback(makeMessage({ throw: true }, 2));
217
-
218
- expect(mockChannel.publish).toHaveBeenCalledWith(
219
- '',
220
- 'jobs.retry',
221
- expect.any(Buffer),
222
- expect.objectContaining({
223
- expiration: '16000', // 4000 * 2^2
224
- headers: expect.objectContaining({ 'x-retry-count': 3 }),
225
- }),
226
- );
227
- });
228
-
229
- it('ignores null messages', async () => {
230
- await consumeCallback(null as unknown as ConsumeMessage);
231
- expect(mockChannel.ack).not.toHaveBeenCalled();
232
- expect(mockChannel.nack).not.toHaveBeenCalled();
233
- });
234
-
235
- it('NACKs unparseable messages (poison)', async () => {
236
- const poisonMsg = {
237
- content: Buffer.from('not-json{{{'),
238
- properties: { headers: {} },
239
- fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
240
- } as unknown as ConsumeMessage;
241
-
242
- await consumeCallback(poisonMsg);
243
- expect(mockChannel.nack).toHaveBeenCalledWith(poisonMsg, false, false);
244
- });
245
- });
246
- });
247
-
248
- describe('listen() manual-ACK mode', () => {
249
- let consumeCallback: (msg: ConsumeMessage | null) => Promise<void>;
250
-
251
- function makeMessage(payload: unknown, retryCount = 0): ConsumeMessage {
252
- return {
253
- content: Buffer.from(JSON.stringify(payload)),
254
- properties: {
255
- headers: { 'x-retry-count': retryCount },
256
- },
257
- fields: { deliveryTag: 1, redelivered: false, exchange: '', routingKey: '' },
258
- } as unknown as ConsumeMessage;
259
- }
260
-
261
- function setupManualAckListener(
262
- handler: (msg: unknown, ack: () => void, nack: () => void) => Promise<void>,
263
- ) {
264
- (mockChannel.consume as jest.Mock).mockImplementationOnce(
265
- (_queue: string, cb: (msg: ConsumeMessage | null) => Promise<void>) => {
266
- consumeCallback = cb;
267
- return Promise.resolve({ consumerTag: 'tag' });
268
- },
269
- );
270
- return MessageQueueService.listen('jobs', handler, { manualAck: true });
271
- }
272
-
273
- it('handler receives ack and nack functions', async () => {
274
- let receivedAck: (() => void) | undefined;
275
- let receivedNack: (() => void) | undefined;
276
-
277
- await setupManualAckListener(async (_msg, ack, nack) => {
278
- receivedAck = ack;
279
- receivedNack = nack;
280
- ack();
281
- });
282
- await consumeCallback(makeMessage({ data: 1 }));
283
-
284
- expect(typeof receivedAck).toBe('function');
285
- expect(typeof receivedNack).toBe('function');
286
- expect(mockChannel.ack).toHaveBeenCalled();
287
- });
288
-
289
- it('guarded ack — second call is no-op', async () => {
290
- await setupManualAckListener(async (_msg, ack) => {
291
- ack();
292
- ack(); // second call — should be no-op
293
- });
294
- await consumeCallback(makeMessage({ data: 1 }));
295
-
296
- expect(mockChannel.ack).toHaveBeenCalledTimes(1);
297
- });
298
-
299
- it('guarded nack — second call is no-op', async () => {
300
- await setupManualAckListener(async (_msg, _ack, nack) => {
301
- nack();
302
- nack(); // second call — should be no-op
303
- });
304
- await consumeCallback(makeMessage({ data: 1 }));
305
-
306
- // retryOrDLQ publishes to retry queue then acks — only one round
307
- expect(mockChannel.publish).toHaveBeenCalledTimes(1);
308
- expect(mockChannel.ack).toHaveBeenCalledTimes(1);
309
- });
310
-
311
- it('ack then nack — nack is no-op (first call wins)', async () => {
312
- await setupManualAckListener(async (_msg, ack, nack) => {
313
- ack();
314
- nack(); // should be no-op since ack already settled
315
- });
316
- await consumeCallback(makeMessage({ data: 1 }));
317
-
318
- expect(mockChannel.ack).toHaveBeenCalledTimes(1);
319
- expect(mockChannel.publish).not.toHaveBeenCalled(); // no retry
320
- });
321
-
322
- it('handler throws without settling — retries via DLQ path', async () => {
323
- await setupManualAckListener(async () => {
324
- throw new Error('boom');
325
- });
326
- await consumeCallback(makeMessage({ data: 1 }));
327
-
328
- // retryOrDLQ should have been called (publishes to retry queue)
329
- expect(mockChannel.publish).toHaveBeenCalledWith(
330
- '',
331
- 'jobs.retry',
332
- expect.any(Buffer),
333
- expect.objectContaining({
334
- persistent: true,
335
- headers: expect.objectContaining({ 'x-retry-count': 1 }),
336
- }),
337
- );
338
- expect(mockChannel.ack).toHaveBeenCalled();
339
- });
340
-
341
- it('handler resolves without settling — treated as failure', async () => {
342
- await setupManualAckListener(async () => {
343
- // deliberately not calling ack or nack
344
- });
345
- await consumeCallback(makeMessage({ data: 1 }));
346
-
347
- // retryOrDLQ should have been called
348
- expect(mockChannel.publish).toHaveBeenCalledWith(
349
- '',
350
- 'jobs.retry',
351
- expect.any(Buffer),
352
- expect.objectContaining({
353
- headers: expect.objectContaining({ 'x-retry-count': 1 }),
354
- }),
355
- );
356
- expect(mockChannel.ack).toHaveBeenCalled();
357
- });
358
-
359
- it('handler throws after acking — no double retry', async () => {
360
- await setupManualAckListener(async (_msg, ack) => {
361
- ack();
362
- throw new Error('post-ack error');
363
- });
364
- await consumeCallback(makeMessage({ data: 1 }));
365
-
366
- // ack was called but no retry should happen (already settled)
367
- expect(mockChannel.ack).toHaveBeenCalledTimes(1);
368
- expect(mockChannel.publish).not.toHaveBeenCalled();
369
- });
370
- });
371
-
372
- describe('listen() prefetch', () => {
373
- it('sets channel.prefetch when opts.prefetch is provided', async () => {
374
- await MessageQueueService.listen('jobs', async () => true, { prefetch: 10 });
375
- expect(mockChannel.prefetch).toHaveBeenCalledWith(10);
376
- });
377
-
378
- it('does not call prefetch when opts.prefetch is not provided', async () => {
379
- await MessageQueueService.listen('jobs', async () => true);
380
- expect(mockChannel.prefetch).not.toHaveBeenCalled();
381
- });
382
-
383
- it('sets prefetch before consuming', async () => {
384
- const callOrder: string[] = [];
385
- (mockChannel.prefetch as jest.Mock).mockImplementation(() => {
386
- callOrder.push('prefetch');
387
- return Promise.resolve(undefined);
388
- });
389
- (mockChannel.consume as jest.Mock).mockImplementation(() => {
390
- callOrder.push('consume');
391
- return Promise.resolve({ consumerTag: 'tag' });
392
- });
393
-
394
- await MessageQueueService.listen('jobs', async () => true, { prefetch: 5 });
395
-
396
- expect(callOrder).toEqual(['prefetch', 'consume']);
397
- });
398
- });
399
-
400
- describe('close()', () => {
401
- it('closes all tracked connections', async () => {
402
- // Create a listener connection
403
- await MessageQueueService.listen('q', async () => true);
404
-
405
- await MessageQueueService.close();
406
- expect(mockConnection.close).toHaveBeenCalled();
407
- });
408
- });
409
-
410
- describe('service shape', () => {
411
- it('exports the required service interface', () => {
412
- expect(MessageQueueService.name).toBe('message-queue');
413
- expect(typeof MessageQueueService.init).toBe('function');
414
- expect(typeof MessageQueueService.publish).toBe('function');
415
- expect(typeof MessageQueueService.listen).toBe('function');
416
- expect(typeof MessageQueueService.close).toBe('function');
417
- });
418
- });
419
- });