exp-node-server 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -3
- package/dist/cli.js +191 -103
- package/package.json +1 -1
- package/templates/mongoose/.prettierignore +1 -0
- package/templates/mongoose/_gitignore +4 -1
- package/templates/mongoose/eslint.config.js +1 -1
- package/templates/mongoose/package-lock.json +5 -16
- package/templates/mongoose/package.json +1 -2
- package/templates/mongoose/readme.md +3 -0
- package/templates/mongoose/src/app/queues/agenda.ts +16 -4
- package/templates/mongoose/src/app/queues/email.queue.ts +34 -6
- package/templates/mongoose/src/app/queues/index.ts +10 -6
- package/templates/mongoose/tests/queues/email.queue.test.ts +137 -0
- package/templates/prisma/.prettierignore +2 -1
- package/templates/prisma/_gitignore +4 -1
- package/templates/prisma/eslint.config.js +1 -1
- package/templates/prisma/package-lock.json +5 -16
- package/templates/prisma/package.json +1 -2
- package/templates/prisma/readme.md +9 -0
- package/templates/prisma/src/app/queues/boss.ts +28 -3
- package/templates/prisma/src/app/queues/email.queue.ts +23 -8
- package/templates/prisma/src/app/queues/index.ts +16 -7
- package/templates/prisma/src/app/queues/maintenance.queue.ts +2 -2
- package/templates/prisma/tests/queues/email.queue.test.ts +162 -0
package/package.json
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
"mongodb": "^7.7.0",
|
|
24
24
|
"mongoose": "^9.11.2",
|
|
25
25
|
"multer": "^2.0.2",
|
|
26
|
-
"nodemailer": "^
|
|
26
|
+
"nodemailer": "^10.1.0",
|
|
27
27
|
"passport": "^0.7.0",
|
|
28
28
|
"passport-facebook": "^3.0.0",
|
|
29
29
|
"passport-github": "^1.1.0",
|
|
@@ -44,7 +44,6 @@
|
|
|
44
44
|
"@types/jsonwebtoken": "^9.0.10",
|
|
45
45
|
"@types/multer": "^2.0.0",
|
|
46
46
|
"@types/node": "^24.12.2",
|
|
47
|
-
"@types/nodemailer": "^7.0.2",
|
|
48
47
|
"@types/passport": "^1.0.17",
|
|
49
48
|
"@types/passport-facebook": "^3.0.4",
|
|
50
49
|
"@types/passport-github": "^1.1.13",
|
|
@@ -1416,16 +1415,6 @@
|
|
|
1416
1415
|
"undici-types": ">=7.24.0 <7.24.7"
|
|
1417
1416
|
}
|
|
1418
1417
|
},
|
|
1419
|
-
"node_modules/@types/nodemailer": {
|
|
1420
|
-
"version": "7.0.12",
|
|
1421
|
-
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-7.0.12.tgz",
|
|
1422
|
-
"integrity": "sha512-80vKwiIsVSyFA1rRovH59jNPLBOuc6dRZIHEu40gXTkBkZnQv8vog1xSGEb9j5q/tdMAs5ivvDR2pLTU0hGHXA==",
|
|
1423
|
-
"dev": true,
|
|
1424
|
-
"license": "MIT",
|
|
1425
|
-
"dependencies": {
|
|
1426
|
-
"@types/node": "*"
|
|
1427
|
-
}
|
|
1428
|
-
},
|
|
1429
1418
|
"node_modules/@types/oauth": {
|
|
1430
1419
|
"version": "0.9.6",
|
|
1431
1420
|
"resolved": "https://registry.npmjs.org/@types/oauth/-/oauth-0.9.6.tgz",
|
|
@@ -4652,12 +4641,12 @@
|
|
|
4652
4641
|
}
|
|
4653
4642
|
},
|
|
4654
4643
|
"node_modules/nodemailer": {
|
|
4655
|
-
"version": "
|
|
4656
|
-
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-
|
|
4657
|
-
"integrity": "sha512-
|
|
4644
|
+
"version": "10.1.0",
|
|
4645
|
+
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.0.tgz",
|
|
4646
|
+
"integrity": "sha512-fXW8AulB25aOTuwfRYV6lqHyfz2VM4rr4SGXH03mE7x5znEUpzL5mu6anARUpZN7744Ziwkmr4AxSLPJsBQspg==",
|
|
4658
4647
|
"license": "MIT-0",
|
|
4659
4648
|
"engines": {
|
|
4660
|
-
"node": ">=
|
|
4649
|
+
"node": ">=20.0.0"
|
|
4661
4650
|
}
|
|
4662
4651
|
},
|
|
4663
4652
|
"node_modules/numbered": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"mongodb": "^7.7.0",
|
|
33
33
|
"mongoose": "^9.11.2",
|
|
34
34
|
"multer": "^2.0.2",
|
|
35
|
-
"nodemailer": "^
|
|
35
|
+
"nodemailer": "^10.1.0",
|
|
36
36
|
"passport": "^0.7.0",
|
|
37
37
|
"passport-facebook": "^3.0.0",
|
|
38
38
|
"passport-github": "^1.1.0",
|
|
@@ -53,7 +53,6 @@
|
|
|
53
53
|
"@types/jsonwebtoken": "^9.0.10",
|
|
54
54
|
"@types/multer": "^2.0.0",
|
|
55
55
|
"@types/node": "^24.12.2",
|
|
56
|
-
"@types/nodemailer": "^7.0.2",
|
|
57
56
|
"@types/passport": "^1.0.17",
|
|
58
57
|
"@types/passport-facebook": "^3.0.4",
|
|
59
58
|
"@types/passport-github": "^1.1.13",
|
|
@@ -177,6 +177,9 @@ through the route, controller and service against a real in-memory MongoDB
|
|
|
177
177
|
([mongodb-memory-server](https://github.com/typegoose/mongodb-memory-server), downloaded
|
|
178
178
|
on the first run). Only the email queue and Cloudinary are mocked.
|
|
179
179
|
|
|
180
|
+
The email queue tests run the real Agenda worker on the same in-memory MongoDB
|
|
181
|
+
(retries, giving up, several workers, graceful shutdown).
|
|
182
|
+
|
|
180
183
|
## 🧠 Tech Stack
|
|
181
184
|
|
|
182
185
|
- Express.js — Backend framework
|
|
@@ -3,18 +3,30 @@ import { Agenda } from 'agenda';
|
|
|
3
3
|
import mongoose from 'mongoose';
|
|
4
4
|
import { logger } from '../lib/logger.js';
|
|
5
5
|
|
|
6
|
+
export type AgendaOptions = {
|
|
7
|
+
/** How often a worker looks for due jobs (ms or a human interval). */
|
|
8
|
+
processEvery?: string | number;
|
|
9
|
+
};
|
|
10
|
+
|
|
6
11
|
/**
|
|
7
12
|
* Agenda stores jobs in the app's MongoDB (`agendaJobs` collection), so no Redis
|
|
8
13
|
* is needed. Created on start, once mongoose is connected, to reuse its connection.
|
|
9
14
|
*/
|
|
10
15
|
let agenda: Agenda | undefined;
|
|
11
16
|
|
|
12
|
-
|
|
13
|
-
|
|
17
|
+
/** A new Agenda instance (a worker) on the app's MongoDB connection. */
|
|
18
|
+
export const newAgenda = ({ processEvery = '5 seconds' }: AgendaOptions = {}) => {
|
|
19
|
+
const instance = new Agenda({
|
|
14
20
|
backend: new MongoBackend({ mongo: mongoose.connection.db! }),
|
|
15
|
-
processEvery
|
|
21
|
+
processEvery,
|
|
16
22
|
});
|
|
17
|
-
|
|
23
|
+
instance.on('error', (err) => logger.error(err, 'Agenda error'));
|
|
24
|
+
return instance;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Creates the app's Agenda instance, used by `enqueueEmail()` & co. */
|
|
28
|
+
export const createAgenda = (options?: AgendaOptions) => {
|
|
29
|
+
agenda = newAgenda(options);
|
|
18
30
|
return agenda;
|
|
19
31
|
};
|
|
20
32
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { exponential, type Job } from 'agenda';
|
|
1
|
+
import { type Agenda, exponential, type Job } from 'agenda';
|
|
2
|
+
import mongoose from 'mongoose';
|
|
2
3
|
import { configs } from '../configs/index.js';
|
|
3
4
|
import { logger } from '../lib/logger.js';
|
|
4
5
|
import { otpEmailTemplate } from '../templates/otpEmailTemplate.js';
|
|
@@ -7,6 +8,8 @@ import sendMail from '../utils/mail_sender.js';
|
|
|
7
8
|
import { getAgenda } from './agenda.js';
|
|
8
9
|
|
|
9
10
|
export const EMAIL_QUEUE = 'email';
|
|
11
|
+
// Agenda's default collection (see agenda.ts)
|
|
12
|
+
export const JOBS_COLLECTION = 'agendaJobs';
|
|
10
13
|
|
|
11
14
|
/** Every email the app sends goes through this one queue. */
|
|
12
15
|
export type EmailJob =
|
|
@@ -30,8 +33,18 @@ const render = (job: EmailJob) => {
|
|
|
30
33
|
}
|
|
31
34
|
};
|
|
32
35
|
|
|
33
|
-
export
|
|
34
|
-
|
|
36
|
+
export type EmailJobOptions = {
|
|
37
|
+
/** Delay before the first retry; it doubles on every further retry. */
|
|
38
|
+
retryDelayMs?: number;
|
|
39
|
+
maxRetries?: number;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** Registers the email job on a worker. */
|
|
43
|
+
export const defineEmailJob = (
|
|
44
|
+
agenda: Agenda,
|
|
45
|
+
{ retryDelayMs = 10_000, maxRetries = 3 }: EmailJobOptions = {},
|
|
46
|
+
) =>
|
|
47
|
+
agenda.define(
|
|
35
48
|
EMAIL_QUEUE,
|
|
36
49
|
async ({ attrs }: Job<EmailJob>) => {
|
|
37
50
|
const data = attrs.data;
|
|
@@ -44,9 +57,24 @@ export const defineEmailJob = () =>
|
|
|
44
57
|
}
|
|
45
58
|
},
|
|
46
59
|
{
|
|
47
|
-
backoff: exponential({ delay:
|
|
48
|
-
//
|
|
49
|
-
|
|
60
|
+
backoff: exponential({ delay: retryDelayMs, factor: 2, maxRetries }),
|
|
61
|
+
// Not `removeOnComplete`: Agenda emits "complete" before it forgets the job,
|
|
62
|
+
// and the extra await of the removal makes drain() miss it and hang until its
|
|
63
|
+
// timeout on every shutdown. A TTL index cleans up instead (below).
|
|
64
|
+
},
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Jobs carry OTPs and reset links, so finished email jobs (sent or given up on)
|
|
69
|
+
* are deleted by MongoDB an hour later. Only the email job: other jobs are kept.
|
|
70
|
+
*/
|
|
71
|
+
export const ensureEmailJobCleanup = () =>
|
|
72
|
+
mongoose.connection.db!.collection(JOBS_COLLECTION).createIndex(
|
|
73
|
+
{ lastFinishedAt: 1 },
|
|
74
|
+
{
|
|
75
|
+
name: 'email_jobs_ttl',
|
|
76
|
+
expireAfterSeconds: 60 * 60,
|
|
77
|
+
partialFilterExpression: { name: EMAIL_QUEUE },
|
|
50
78
|
},
|
|
51
79
|
);
|
|
52
80
|
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { logger } from '../lib/logger.js';
|
|
2
|
-
import { createAgenda, getAgenda } from './agenda.js';
|
|
3
|
-
import { defineEmailJob } from './email.queue.js';
|
|
2
|
+
import { type AgendaOptions, createAgenda, getAgenda } from './agenda.js';
|
|
3
|
+
import { defineEmailJob, type EmailJobOptions, ensureEmailJobCleanup } from './email.queue.js';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
/** The defaults are meant for production; tests pass shorter delays. */
|
|
6
|
+
export type QueueOptions = AgendaOptions & EmailJobOptions;
|
|
7
|
+
|
|
8
|
+
export const startQueues = async (options: QueueOptions = {}) => {
|
|
9
|
+
const agenda = createAgenda(options);
|
|
10
|
+
defineEmailJob(agenda, options);
|
|
11
|
+
await ensureEmailJobCleanup();
|
|
12
|
+
await agenda.start();
|
|
9
13
|
logger.info('Queues are running...');
|
|
10
14
|
};
|
|
11
15
|
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import type { Agenda } from 'agenda';
|
|
2
|
+
import mongoose from 'mongoose';
|
|
3
|
+
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
4
|
+
import { newAgenda } from '../../src/app/queues/agenda.js';
|
|
5
|
+
import { defineEmailJob, enqueueEmail, type EmailJob } from '../../src/app/queues/email.queue.js';
|
|
6
|
+
import { startQueues, stopQueues } from '../../src/app/queues/index.js';
|
|
7
|
+
import sendMail from '../../src/app/utils/mail_sender.js';
|
|
8
|
+
|
|
9
|
+
// the real queue (tests/setup.ts mocks it for the route tests); only SMTP is faked
|
|
10
|
+
vi.unmock('../../src/app/queues/email.queue.js');
|
|
11
|
+
vi.mock('../../src/app/utils/mail_sender.js', () => ({ default: vi.fn() }));
|
|
12
|
+
|
|
13
|
+
const send = vi.mocked(sendMail);
|
|
14
|
+
const FAST = { processEvery: 50, retryDelayMs: 50, maxRetries: 2 };
|
|
15
|
+
|
|
16
|
+
const email = (to: string): EmailJob => ({
|
|
17
|
+
type: 'reset-password',
|
|
18
|
+
email: to,
|
|
19
|
+
resetLink: `https://app.test/reset?token=${to}`,
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const jobs = () => mongoose.connection.db!.collection('agendaJobs');
|
|
23
|
+
|
|
24
|
+
/** Polls until `check` passes or `timeoutMs` is over. */
|
|
25
|
+
const waitFor = async (check: () => unknown | Promise<unknown>, timeoutMs = 5000) => {
|
|
26
|
+
const start = Date.now();
|
|
27
|
+
for (;;) {
|
|
28
|
+
try {
|
|
29
|
+
await check();
|
|
30
|
+
return;
|
|
31
|
+
} catch (err) {
|
|
32
|
+
if (Date.now() - start > timeoutMs) throw err;
|
|
33
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/** Lets workers run for a while, to show that nothing else happens. */
|
|
39
|
+
const settle = (ms = 400) => new Promise((r) => setTimeout(r, ms));
|
|
40
|
+
|
|
41
|
+
const extraWorkers: Agenda[] = [];
|
|
42
|
+
afterEach(async () => {
|
|
43
|
+
await Promise.all(extraWorkers.splice(0).map((w) => w.stop()));
|
|
44
|
+
await stopQueues().catch(() => {});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('email queue (Agenda)', () => {
|
|
48
|
+
it('sends a queued email once, and finished jobs expire', async () => {
|
|
49
|
+
send.mockResolvedValue({} as never);
|
|
50
|
+
await startQueues(FAST);
|
|
51
|
+
|
|
52
|
+
await enqueueEmail(email('a@test.com'));
|
|
53
|
+
|
|
54
|
+
await waitFor(() => expect(send).toHaveBeenCalledTimes(1));
|
|
55
|
+
expect(send).toHaveBeenCalledWith(
|
|
56
|
+
expect.objectContaining({ to: 'a@test.com', subject: 'Reset Your Password' }),
|
|
57
|
+
);
|
|
58
|
+
// jobs carry reset links and OTPs: MongoDB deletes finished email jobs after an hour
|
|
59
|
+
await waitFor(async () =>
|
|
60
|
+
expect(await jobs().findOne({ name: 'email' })).toHaveProperty('lastFinishedAt'),
|
|
61
|
+
);
|
|
62
|
+
const ttl = (await jobs().indexes()).find((i) => i.name === 'email_jobs_ttl');
|
|
63
|
+
expect(ttl).toMatchObject({
|
|
64
|
+
expireAfterSeconds: 3600,
|
|
65
|
+
partialFilterExpression: { name: 'email' },
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('retries a failed send', async () => {
|
|
70
|
+
send.mockRejectedValueOnce(new Error('SMTP down')).mockResolvedValue({} as never);
|
|
71
|
+
await startQueues(FAST);
|
|
72
|
+
|
|
73
|
+
await enqueueEmail(email('retry@test.com'));
|
|
74
|
+
|
|
75
|
+
await waitFor(() => expect(send).toHaveBeenCalledTimes(2));
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('gives up after the retry limit', async () => {
|
|
79
|
+
send.mockRejectedValue(new Error('SMTP down'));
|
|
80
|
+
await startQueues(FAST);
|
|
81
|
+
|
|
82
|
+
await enqueueEmail(email('broken@test.com'));
|
|
83
|
+
|
|
84
|
+
// the first attempt + 2 retries, and then nothing more
|
|
85
|
+
await waitFor(() => expect(send).toHaveBeenCalledTimes(3));
|
|
86
|
+
await settle();
|
|
87
|
+
expect(send).toHaveBeenCalledTimes(3);
|
|
88
|
+
const [job] = await jobs().find().toArray();
|
|
89
|
+
expect(job).toMatchObject({ failReason: 'SMTP down' });
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('never sends the same email twice, even with several workers', async () => {
|
|
93
|
+
send.mockResolvedValue({} as never);
|
|
94
|
+
await startQueues(FAST);
|
|
95
|
+
const second = newAgenda(FAST);
|
|
96
|
+
defineEmailJob(second, FAST);
|
|
97
|
+
extraWorkers.push(second);
|
|
98
|
+
await second.start();
|
|
99
|
+
|
|
100
|
+
const recipients = ['1', '2', '3', '4', '5', '6'].map((n) => `${n}@test.com`);
|
|
101
|
+
await Promise.all(recipients.map((to) => enqueueEmail(email(to))));
|
|
102
|
+
|
|
103
|
+
await waitFor(() => expect(send).toHaveBeenCalledTimes(recipients.length));
|
|
104
|
+
await settle();
|
|
105
|
+
const sentTo = send.mock.calls.map(([mail]) => mail.to).sort();
|
|
106
|
+
expect(sentTo).toEqual(recipients.sort());
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('sends the emails that were queued while no worker was running', async () => {
|
|
110
|
+
send.mockResolvedValue({} as never);
|
|
111
|
+
await startQueues(FAST);
|
|
112
|
+
await stopQueues();
|
|
113
|
+
await enqueueEmail(email('queued@test.com'));
|
|
114
|
+
await settle();
|
|
115
|
+
expect(send).not.toHaveBeenCalled();
|
|
116
|
+
|
|
117
|
+
await startQueues(FAST); // e.g. after a deploy
|
|
118
|
+
|
|
119
|
+
await waitFor(() => expect(send).toHaveBeenCalledTimes(1));
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('waits for a running send to finish before shutting down', async () => {
|
|
123
|
+
let finished = false;
|
|
124
|
+
send.mockImplementation(async () => {
|
|
125
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
126
|
+
finished = true;
|
|
127
|
+
return {} as never;
|
|
128
|
+
});
|
|
129
|
+
await startQueues(FAST);
|
|
130
|
+
await enqueueEmail(email('slow@test.com'));
|
|
131
|
+
await waitFor(() => expect(send).toHaveBeenCalled());
|
|
132
|
+
|
|
133
|
+
await stopQueues();
|
|
134
|
+
|
|
135
|
+
expect(finished).toBe(true);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"helmet": "^8.3.0",
|
|
22
22
|
"jsonwebtoken": "^9.0.2",
|
|
23
23
|
"multer": "^2.0.2",
|
|
24
|
-
"nodemailer": "^
|
|
24
|
+
"nodemailer": "^10.1.0",
|
|
25
25
|
"passport": "^0.7.0",
|
|
26
26
|
"passport-facebook": "^3.0.0",
|
|
27
27
|
"passport-github": "^1.1.0",
|
|
@@ -44,7 +44,6 @@
|
|
|
44
44
|
"@types/jsonwebtoken": "^9.0.10",
|
|
45
45
|
"@types/multer": "^2.0.0",
|
|
46
46
|
"@types/node": "^24.12.2",
|
|
47
|
-
"@types/nodemailer": "^7.0.2",
|
|
48
47
|
"@types/passport": "^1.0.17",
|
|
49
48
|
"@types/passport-facebook": "^3.0.4",
|
|
50
49
|
"@types/passport-github": "^1.1.13",
|
|
@@ -1835,16 +1834,6 @@
|
|
|
1835
1834
|
"undici-types": "~7.16.0"
|
|
1836
1835
|
}
|
|
1837
1836
|
},
|
|
1838
|
-
"node_modules/@types/nodemailer": {
|
|
1839
|
-
"version": "7.0.11",
|
|
1840
|
-
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-7.0.11.tgz",
|
|
1841
|
-
"integrity": "sha512-E+U4RzR2dKrx+u3N4DlsmLaDC6mMZOM/TPROxA0UAPiTgI0y4CEFBmZE+coGWTjakDriRsXG368lNk1u9Q0a2g==",
|
|
1842
|
-
"dev": true,
|
|
1843
|
-
"license": "MIT",
|
|
1844
|
-
"dependencies": {
|
|
1845
|
-
"@types/node": "*"
|
|
1846
|
-
}
|
|
1847
|
-
},
|
|
1848
1837
|
"node_modules/@types/oauth": {
|
|
1849
1838
|
"version": "0.9.6",
|
|
1850
1839
|
"resolved": "https://registry.npmjs.org/@types/oauth/-/oauth-0.9.6.tgz",
|
|
@@ -5244,12 +5233,12 @@
|
|
|
5244
5233
|
}
|
|
5245
5234
|
},
|
|
5246
5235
|
"node_modules/nodemailer": {
|
|
5247
|
-
"version": "
|
|
5248
|
-
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-
|
|
5249
|
-
"integrity": "sha512-
|
|
5236
|
+
"version": "10.1.0",
|
|
5237
|
+
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.0.tgz",
|
|
5238
|
+
"integrity": "sha512-fXW8AulB25aOTuwfRYV6lqHyfz2VM4rr4SGXH03mE7x5znEUpzL5mu6anARUpZN7744Ziwkmr4AxSLPJsBQspg==",
|
|
5250
5239
|
"license": "MIT-0",
|
|
5251
5240
|
"engines": {
|
|
5252
|
-
"node": ">=
|
|
5241
|
+
"node": ">=20.0.0"
|
|
5253
5242
|
}
|
|
5254
5243
|
},
|
|
5255
5244
|
"node_modules/non-error": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"helmet": "^8.3.0",
|
|
33
33
|
"jsonwebtoken": "^9.0.2",
|
|
34
34
|
"multer": "^2.0.2",
|
|
35
|
-
"nodemailer": "^
|
|
35
|
+
"nodemailer": "^10.1.0",
|
|
36
36
|
"passport": "^0.7.0",
|
|
37
37
|
"passport-facebook": "^3.0.0",
|
|
38
38
|
"passport-github": "^1.1.0",
|
|
@@ -55,7 +55,6 @@
|
|
|
55
55
|
"@types/jsonwebtoken": "^9.0.10",
|
|
56
56
|
"@types/multer": "^2.0.0",
|
|
57
57
|
"@types/node": "^24.12.2",
|
|
58
|
-
"@types/nodemailer": "^7.0.2",
|
|
59
58
|
"@types/passport": "^1.0.17",
|
|
60
59
|
"@types/passport-facebook": "^3.0.4",
|
|
61
60
|
"@types/passport-github": "^1.1.13",
|
|
@@ -173,6 +173,15 @@ Route tests with [supertest](https://github.com/ladjs/supertest) run each reques
|
|
|
173
173
|
through the route, controller and service. Prisma, the email queue and Cloudinary
|
|
174
174
|
are mocked, so no database is needed.
|
|
175
175
|
|
|
176
|
+
The email queue tests need a real PostgreSQL and are skipped otherwise:
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
docker compose up -d
|
|
180
|
+
TEST_DATABASE_URL="postgresql://postgres:change-me@localhost:5432/server_db" npm test
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
They use their own schema, which is dropped afterwards.
|
|
184
|
+
|
|
176
185
|
## 🧠 Tech Stack
|
|
177
186
|
|
|
178
187
|
- Express.js — Backend framework
|
|
@@ -2,7 +2,32 @@ import { PgBoss } from 'pg-boss';
|
|
|
2
2
|
import { configs } from '../configs/index.js';
|
|
3
3
|
import { logger } from '../lib/logger.js';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
export type BossOptions = {
|
|
6
|
+
connectionString?: string;
|
|
7
|
+
/** Postgres schema for the job tables (default `pgboss`). */
|
|
8
|
+
schema?: string;
|
|
9
|
+
};
|
|
7
10
|
|
|
8
|
-
|
|
11
|
+
/**
|
|
12
|
+
* pg-boss stores jobs in its own schema in the app's Postgres database, so no
|
|
13
|
+
* Redis is needed. Created on start, see startQueues().
|
|
14
|
+
*/
|
|
15
|
+
let boss: PgBoss | undefined;
|
|
16
|
+
|
|
17
|
+
/** A new pg-boss instance (a producer and/or worker). */
|
|
18
|
+
export const newBoss = ({ connectionString = configs.db_url, schema }: BossOptions = {}) => {
|
|
19
|
+
const instance = new PgBoss({ connectionString, ...(schema && { schema }) });
|
|
20
|
+
instance.on('error', (error) => logger.error(error, 'pg-boss error'));
|
|
21
|
+
return instance;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** Creates the app's pg-boss instance, used by `enqueueEmail()` & co. */
|
|
25
|
+
export const createBoss = (options?: BossOptions) => {
|
|
26
|
+
boss = newBoss(options);
|
|
27
|
+
return boss;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export const getBoss = (): PgBoss => {
|
|
31
|
+
if (!boss) throw new Error('Queues are not started, call startQueues() first');
|
|
32
|
+
return boss;
|
|
33
|
+
};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import type { Job } from 'pg-boss';
|
|
1
|
+
import type { Job, PgBoss } from 'pg-boss';
|
|
2
2
|
import { configs } from '../configs/index.js';
|
|
3
3
|
import { logger } from '../lib/logger.js';
|
|
4
4
|
import { otpEmailTemplate } from '../templates/otpEmailTemplate.js';
|
|
5
5
|
import { resetPasswordEmailTemplate } from '../templates/resetPasswordEmailTemplate.js';
|
|
6
6
|
import sendMail from '../utils/mail_sender.js';
|
|
7
|
-
import {
|
|
7
|
+
import { getBoss } from './boss.js';
|
|
8
8
|
|
|
9
9
|
export const EMAIL_QUEUE = 'email';
|
|
10
10
|
|
|
@@ -30,20 +30,35 @@ const render = (job: EmailJob) => {
|
|
|
30
30
|
}
|
|
31
31
|
};
|
|
32
32
|
|
|
33
|
-
export
|
|
33
|
+
export type EmailJobOptions = {
|
|
34
|
+
/** Delay before the first retry; it doubles on every further retry. */
|
|
35
|
+
retryDelaySeconds?: number;
|
|
36
|
+
retryLimit?: number;
|
|
37
|
+
/** How often an idle worker checks for new jobs. */
|
|
38
|
+
pollingIntervalSeconds?: number;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export const createEmailQueue = (
|
|
42
|
+
boss: PgBoss,
|
|
43
|
+
{ retryDelaySeconds = 10, retryLimit = 3 }: EmailJobOptions = {},
|
|
44
|
+
) =>
|
|
34
45
|
boss.createQueue(EMAIL_QUEUE, {
|
|
35
|
-
retryLimit
|
|
36
|
-
retryDelay:
|
|
46
|
+
retryLimit,
|
|
47
|
+
retryDelay: retryDelaySeconds,
|
|
37
48
|
retryBackoff: true,
|
|
38
49
|
expireInSeconds: 60,
|
|
39
50
|
// jobs carry OTPs and reset links, so don't keep them around
|
|
40
51
|
deleteAfterSeconds: 60 * 60,
|
|
41
52
|
});
|
|
42
53
|
|
|
43
|
-
export const enqueueEmail = (job: EmailJob) =>
|
|
54
|
+
export const enqueueEmail = (job: EmailJob) => getBoss().send(EMAIL_QUEUE, job);
|
|
44
55
|
|
|
45
|
-
|
|
46
|
-
|
|
56
|
+
/** Starts processing emails on `boss`. */
|
|
57
|
+
export const startEmailWorker = (
|
|
58
|
+
boss: PgBoss,
|
|
59
|
+
{ pollingIntervalSeconds = 2 }: EmailJobOptions = {},
|
|
60
|
+
) =>
|
|
61
|
+
boss.work<EmailJob>(EMAIL_QUEUE, { pollingIntervalSeconds }, async (jobs: Job<EmailJob>[]) => {
|
|
47
62
|
for (const { id, data } of jobs) {
|
|
48
63
|
try {
|
|
49
64
|
await sendMail({ to: data.email, ...render(data) });
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { logger } from '../lib/logger.js';
|
|
2
|
-
import {
|
|
3
|
-
import { createEmailQueue, startEmailWorker } from './email.queue.js';
|
|
2
|
+
import { type BossOptions, createBoss, getBoss } from './boss.js';
|
|
3
|
+
import { createEmailQueue, type EmailJobOptions, startEmailWorker } from './email.queue.js';
|
|
4
4
|
import { startMaintenance } from './maintenance.queue.js';
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
/** The defaults are meant for production; tests pass shorter delays. */
|
|
7
|
+
export type QueueOptions = BossOptions &
|
|
8
|
+
EmailJobOptions & {
|
|
9
|
+
/** Schedule the cleanup of expired rows (default true). */
|
|
10
|
+
maintenance?: boolean;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const startQueues = async (options: QueueOptions = {}) => {
|
|
14
|
+
const boss = createBoss(options);
|
|
7
15
|
await boss.start();
|
|
8
|
-
await createEmailQueue();
|
|
9
|
-
await startEmailWorker();
|
|
10
|
-
await startMaintenance();
|
|
16
|
+
await createEmailQueue(boss, options);
|
|
17
|
+
await startEmailWorker(boss, options);
|
|
18
|
+
if (options.maintenance !== false) await startMaintenance(boss);
|
|
11
19
|
logger.info('Queues are running...');
|
|
12
20
|
};
|
|
13
21
|
|
|
14
|
-
|
|
22
|
+
// waits for running jobs to finish (up to 30s) instead of abandoning them
|
|
23
|
+
export const stopQueues = () => getBoss().stop({ graceful: true, timeout: 30_000 });
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { prisma } from '../lib/prisma.js';
|
|
2
2
|
import { logger } from '../lib/logger.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { PgBoss } from 'pg-boss';
|
|
4
4
|
|
|
5
5
|
export const MAINTENANCE_QUEUE = 'maintenance';
|
|
6
6
|
|
|
@@ -22,7 +22,7 @@ const cleanup = async () => {
|
|
|
22
22
|
);
|
|
23
23
|
};
|
|
24
24
|
|
|
25
|
-
export const startMaintenance = async () => {
|
|
25
|
+
export const startMaintenance = async (boss: PgBoss) => {
|
|
26
26
|
await boss.createQueue(MAINTENANCE_QUEUE, { retryLimit: 1, deleteAfterSeconds: 60 * 60 });
|
|
27
27
|
// pg-boss runs a schedule once across all instances, every 15 minutes
|
|
28
28
|
await boss.schedule(MAINTENANCE_QUEUE, '*/15 * * * *');
|