@rvoh/psychic-workers 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +5 -0
- package/dist/cjs/src/background/BaseBackgroundedModel.js +50 -0
- package/dist/cjs/src/background/BaseBackgroundedService.js +40 -0
- package/dist/cjs/src/background/BaseScheduledService.js +31 -0
- package/dist/cjs/src/background/index.js +529 -0
- package/dist/cjs/src/background/types.js +2 -0
- package/dist/cjs/src/error/background/NoQueueForSpecifiedQueueName.js +15 -0
- package/dist/cjs/src/error/background/NoQueueForSpecifiedWorkstream.js +15 -0
- package/dist/cjs/src/helpers/EnvInternal.js +5 -0
- package/dist/cjs/src/index.js +19 -0
- package/dist/cjs/src/psychic-application-workers/cache.js +17 -0
- package/dist/cjs/src/psychic-application-workers/index.js +77 -0
- package/dist/esm/src/background/BaseBackgroundedModel.js +47 -0
- package/dist/esm/src/background/BaseBackgroundedService.js +37 -0
- package/dist/esm/src/background/BaseScheduledService.js +28 -0
- package/dist/esm/src/background/index.js +524 -0
- package/dist/esm/src/background/types.js +1 -0
- package/dist/esm/src/error/background/NoQueueForSpecifiedQueueName.js +12 -0
- package/dist/esm/src/error/background/NoQueueForSpecifiedWorkstream.js +12 -0
- package/dist/esm/src/helpers/EnvInternal.js +3 -0
- package/dist/esm/src/index.js +7 -0
- package/dist/esm/src/psychic-application-workers/cache.js +12 -0
- package/dist/esm/src/psychic-application-workers/index.js +74 -0
- package/dist/types/src/background/BaseBackgroundedModel.d.ts +15 -0
- package/dist/types/src/background/BaseBackgroundedService.d.ts +17 -0
- package/dist/types/src/background/BaseScheduledService.d.ts +11 -0
- package/dist/types/src/background/index.d.ts +101 -0
- package/dist/types/src/background/types.d.ts +7 -0
- package/dist/types/src/error/background/NoQueueForSpecifiedQueueName.d.ts +5 -0
- package/dist/types/src/error/background/NoQueueForSpecifiedWorkstream.d.ts +5 -0
- package/dist/types/src/helpers/EnvInternal.d.ts +6 -0
- package/dist/types/src/index.d.ts +7 -0
- package/dist/types/src/psychic-application-workers/cache.d.ts +4 -0
- package/dist/types/src/psychic-application-workers/index.d.ts +200 -0
- package/package.json +74 -0
@@ -0,0 +1,524 @@
|
|
1
|
+
import { closeAllDbConnections, compact, pascalize } from '@rvoh/dream';
|
2
|
+
import { PsychicApplication } from '@rvoh/psychic';
|
3
|
+
import { Job, Queue, Worker } from 'bullmq';
|
4
|
+
import { Cluster } from 'ioredis';
|
5
|
+
import NoQueueForSpecifiedQueueName from '../error/background/NoQueueForSpecifiedQueueName.js';
|
6
|
+
import NoQueueForSpecifiedWorkstream from '../error/background/NoQueueForSpecifiedWorkstream.js';
|
7
|
+
import EnvInternal from '../helpers/EnvInternal.js';
|
8
|
+
import PsychicApplicationWorkers from '../psychic-application-workers/index.js';
|
9
|
+
class DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection extends Error {
|
10
|
+
get message() {
|
11
|
+
return `
|
12
|
+
Native BullMQ options don't include a default queue connection, and the
|
13
|
+
default config does not include a queue connection
|
14
|
+
`;
|
15
|
+
}
|
16
|
+
}
|
17
|
+
class NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection extends Error {
|
18
|
+
queueName;
|
19
|
+
constructor(queueName) {
|
20
|
+
super();
|
21
|
+
this.queueName = queueName;
|
22
|
+
}
|
23
|
+
get message() {
|
24
|
+
return `
|
25
|
+
Native BullMQ options don't include a default queue connection, and the
|
26
|
+
${this.queueName} queue does not include a queue connection
|
27
|
+
`;
|
28
|
+
}
|
29
|
+
}
|
30
|
+
class ActivatingBackgroundWorkersWithoutDefaultWorkerConnection extends Error {
|
31
|
+
get message() {
|
32
|
+
return `
|
33
|
+
defaultWorkerConnection is required when activating workers. For example,
|
34
|
+
it may be omitted on webserver instances, but is required on worker instances.
|
35
|
+
`;
|
36
|
+
}
|
37
|
+
}
|
38
|
+
class ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection extends Error {
|
39
|
+
queueName;
|
40
|
+
constructor(queueName) {
|
41
|
+
super();
|
42
|
+
this.queueName = queueName;
|
43
|
+
}
|
44
|
+
get message() {
|
45
|
+
return `
|
46
|
+
defaultWorkerConnection is missing, and the ${this.queueName} queue does not
|
47
|
+
specify a workerConnection. A worker connection isrequired when activating workers.
|
48
|
+
For example, it may be omitted on webserver instances, but is required on worker instances.
|
49
|
+
`;
|
50
|
+
}
|
51
|
+
}
|
52
|
+
export class Background {
|
53
|
+
static get defaultQueueName() {
|
54
|
+
const psychicWorkersApp = PsychicApplicationWorkers.getOrFail();
|
55
|
+
return `${pascalize(psychicWorkersApp.psychicApp.appName)}BackgroundJobQueue`;
|
56
|
+
}
|
57
|
+
static get Worker() {
|
58
|
+
const psychicWorkersApp = PsychicApplicationWorkers.getOrFail();
|
59
|
+
return (psychicWorkersApp.backgroundOptions.providers?.Worker || Worker);
|
60
|
+
}
|
61
|
+
static get Queue() {
|
62
|
+
const psychicWorkersApp = PsychicApplicationWorkers.getOrFail();
|
63
|
+
return (psychicWorkersApp.backgroundOptions.providers?.Queue || Queue);
|
64
|
+
}
|
65
|
+
/**
|
66
|
+
* Used when adding jobs to the default queue
|
67
|
+
*/
|
68
|
+
defaultQueue = null;
|
69
|
+
/**
|
70
|
+
* Used when adding jobs to the default transitional queue
|
71
|
+
*/
|
72
|
+
defaultTransitionalQueue = null;
|
73
|
+
/**
|
74
|
+
* Used when adding jobs to a named queue
|
75
|
+
*/
|
76
|
+
namedQueues = {};
|
77
|
+
groupNames = {};
|
78
|
+
workstreamNames = [];
|
79
|
+
/**
|
80
|
+
* Used when adding jobs to a named transitioanl queue
|
81
|
+
*/
|
82
|
+
namedTransitionalQueues = {};
|
83
|
+
_workers = [];
|
84
|
+
redisConnections = [];
|
85
|
+
connect({ activateWorkers = false, } = {}) {
|
86
|
+
if (this.defaultQueue)
|
87
|
+
return;
|
88
|
+
const psychicWorkersApp = PsychicApplicationWorkers.getOrFail();
|
89
|
+
const defaultBullMQQueueOptions = psychicWorkersApp.backgroundOptions.defaultBullMQQueueOptions || {};
|
90
|
+
if (psychicWorkersApp.backgroundOptions.nativeBullMQ) {
|
91
|
+
this.nativeBullMQConnect(defaultBullMQQueueOptions, psychicWorkersApp.backgroundOptions, { activateWorkers });
|
92
|
+
}
|
93
|
+
else {
|
94
|
+
this.simpleConnect(defaultBullMQQueueOptions, psychicWorkersApp.backgroundOptions, { activateWorkers });
|
95
|
+
}
|
96
|
+
}
|
97
|
+
get queues() {
|
98
|
+
return compact([
|
99
|
+
this.defaultQueue,
|
100
|
+
...Object.values(this.namedQueues).map(queue => queue),
|
101
|
+
this.defaultTransitionalQueue,
|
102
|
+
...Object.values(this.namedTransitionalQueues).map(queue => queue),
|
103
|
+
]);
|
104
|
+
}
|
105
|
+
get workers() {
|
106
|
+
return [...this._workers];
|
107
|
+
}
|
108
|
+
async shutdownAndExit() {
|
109
|
+
await this.shutdown();
|
110
|
+
process.exit();
|
111
|
+
}
|
112
|
+
async shutdown() {
|
113
|
+
await Promise.all(this._workers.map(worker => worker.close()));
|
114
|
+
const psychicWorkersApp = PsychicApplicationWorkers.getOrFail();
|
115
|
+
for (const hook of psychicWorkersApp.hooks.workerShutdown) {
|
116
|
+
await hook();
|
117
|
+
}
|
118
|
+
await closeAllDbConnections();
|
119
|
+
await this.closeAllRedisConnections();
|
120
|
+
}
|
121
|
+
async closeAllRedisConnections() {
|
122
|
+
for (const queue of this.queues) {
|
123
|
+
await queue.close();
|
124
|
+
}
|
125
|
+
for (const worker of this.workers) {
|
126
|
+
await worker.close();
|
127
|
+
}
|
128
|
+
for (const connection of this.redisConnections) {
|
129
|
+
try {
|
130
|
+
connection.disconnect();
|
131
|
+
}
|
132
|
+
catch {
|
133
|
+
// noop
|
134
|
+
}
|
135
|
+
}
|
136
|
+
}
|
137
|
+
simpleConnect(defaultBullMQQueueOptions, backgroundOptions, { activateWorkers = false, activatingTransitionalWorkstreams = false, }) {
|
138
|
+
const defaultQueueConnection = backgroundOptions.defaultQueueConnection;
|
139
|
+
const defaultWorkerConnection = backgroundOptions.defaultWorkerConnection;
|
140
|
+
if (defaultQueueConnection)
|
141
|
+
this.redisConnections.push(defaultQueueConnection);
|
142
|
+
if (defaultWorkerConnection)
|
143
|
+
this.redisConnections.push(defaultWorkerConnection);
|
144
|
+
// transitional queues must have the same names they had prior to making them
|
145
|
+
// transitional since the name is what identifies the queues and enables the
|
146
|
+
// queues to be worked off
|
147
|
+
const formattedQueueName = nameToRedisQueueName(Background.defaultQueueName, defaultQueueConnection);
|
148
|
+
///////////////////////////////
|
149
|
+
// create default workstream //
|
150
|
+
///////////////////////////////
|
151
|
+
const defaultQueue = new Background.Queue(formattedQueueName, {
|
152
|
+
...defaultBullMQQueueOptions,
|
153
|
+
connection: defaultQueueConnection,
|
154
|
+
});
|
155
|
+
if (activatingTransitionalWorkstreams) {
|
156
|
+
this.defaultTransitionalQueue = defaultQueue;
|
157
|
+
}
|
158
|
+
else {
|
159
|
+
this.defaultQueue = defaultQueue;
|
160
|
+
}
|
161
|
+
////////////////////////////////////
|
162
|
+
// end: create default workstream //
|
163
|
+
////////////////////////////////////
|
164
|
+
/////////////////////////////
|
165
|
+
// create default workers //
|
166
|
+
/////////////////////////////
|
167
|
+
if (activateWorkers) {
|
168
|
+
if (!defaultWorkerConnection)
|
169
|
+
throw new ActivatingBackgroundWorkersWithoutDefaultWorkerConnection();
|
170
|
+
const workerCount = backgroundOptions.defaultWorkstream?.workerCount ?? 1;
|
171
|
+
for (let i = 0; i < workerCount; i++) {
|
172
|
+
this._workers.push(new Background.Worker(formattedQueueName, job => this.doWork(job), {
|
173
|
+
connection: defaultWorkerConnection,
|
174
|
+
concurrency: backgroundOptions.defaultWorkstream?.concurrency || 1,
|
175
|
+
}));
|
176
|
+
}
|
177
|
+
}
|
178
|
+
/////////////////////////////////
|
179
|
+
// end: create default workers //
|
180
|
+
/////////////////////////////////
|
181
|
+
//////////////////////////////
|
182
|
+
// create named workstreams //
|
183
|
+
//////////////////////////////
|
184
|
+
const namedWorkstreams = backgroundOptions.namedWorkstreams || [];
|
185
|
+
namedWorkstreams.forEach(namedWorkstream => {
|
186
|
+
if (namedWorkstream.queueConnection)
|
187
|
+
this.redisConnections.push(namedWorkstream.queueConnection);
|
188
|
+
if (namedWorkstream.workerConnection)
|
189
|
+
this.redisConnections.push(namedWorkstream.workerConnection);
|
190
|
+
const namedWorkstreamQueueConnection = namedWorkstream.queueConnection || defaultQueueConnection;
|
191
|
+
const namedWorkstreamWorkerConnection = namedWorkstream.workerConnection || defaultWorkerConnection;
|
192
|
+
// transitional queues must have the same names they had prior to making them
|
193
|
+
// transitional since the name is what identifies the queues and enables the
|
194
|
+
// queues to be worked off
|
195
|
+
const namedWorkstreamFormattedQueueName = nameToRedisQueueName(namedWorkstream.name, namedWorkstreamQueueConnection);
|
196
|
+
const namedQueue = new Background.Queue(namedWorkstreamFormattedQueueName, {
|
197
|
+
...defaultBullMQQueueOptions,
|
198
|
+
connection: namedWorkstreamQueueConnection,
|
199
|
+
});
|
200
|
+
if (activatingTransitionalWorkstreams) {
|
201
|
+
this.namedTransitionalQueues[namedWorkstream.name] = namedQueue;
|
202
|
+
}
|
203
|
+
else {
|
204
|
+
this.namedQueues[namedWorkstream.name] = namedQueue;
|
205
|
+
this.workstreamNames.push(namedWorkstream.name);
|
206
|
+
}
|
207
|
+
//////////////////////////
|
208
|
+
// create named workers //
|
209
|
+
//////////////////////////
|
210
|
+
if (activateWorkers) {
|
211
|
+
if (!namedWorkstreamWorkerConnection)
|
212
|
+
throw new ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection(namedWorkstream.name);
|
213
|
+
const workerCount = namedWorkstream.workerCount ?? 1;
|
214
|
+
for (let i = 0; i < workerCount; i++) {
|
215
|
+
this._workers.push(new Background.Worker(namedWorkstreamFormattedQueueName, job => this.doWork(job), {
|
216
|
+
group: {
|
217
|
+
id: namedWorkstream.name,
|
218
|
+
limit: namedWorkstream.rateLimit,
|
219
|
+
},
|
220
|
+
connection: namedWorkstreamWorkerConnection,
|
221
|
+
concurrency: namedWorkstream.concurrency || 1,
|
222
|
+
// explicitly typing as WorkerOptions because Psychic can't be aware of BullMQ Pro options
|
223
|
+
}));
|
224
|
+
}
|
225
|
+
}
|
226
|
+
///////////////////////////////
|
227
|
+
// end: create named workers //
|
228
|
+
///////////////////////////////
|
229
|
+
});
|
230
|
+
///////////////////////////////////
|
231
|
+
// end: create named workstreams //
|
232
|
+
///////////////////////////////////
|
233
|
+
const transitionalWorkstreams = backgroundOptions
|
234
|
+
.transitionalWorkstreams;
|
235
|
+
if (transitionalWorkstreams) {
|
236
|
+
this.simpleConnect(defaultBullMQQueueOptions, transitionalWorkstreams, {
|
237
|
+
activateWorkers,
|
238
|
+
activatingTransitionalWorkstreams: true,
|
239
|
+
});
|
240
|
+
}
|
241
|
+
}
|
242
|
+
nativeBullMQConnect(defaultBullMQQueueOptions, backgroundOptions, { activateWorkers = false, }) {
|
243
|
+
const nativeBullMQ = backgroundOptions.nativeBullMQ;
|
244
|
+
const defaultQueueConnection = nativeBullMQ.defaultQueueOptions?.queueConnection || backgroundOptions.defaultQueueConnection;
|
245
|
+
const defaultWorkerConnection = nativeBullMQ.defaultQueueOptions?.workerConnection || backgroundOptions.defaultWorkerConnection;
|
246
|
+
if (defaultQueueConnection)
|
247
|
+
this.redisConnections.push(defaultQueueConnection);
|
248
|
+
if (defaultWorkerConnection)
|
249
|
+
this.redisConnections.push(defaultWorkerConnection);
|
250
|
+
if (!defaultQueueConnection)
|
251
|
+
throw new DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection();
|
252
|
+
const formattedQueueName = nameToRedisQueueName(Background.defaultQueueName, defaultQueueConnection);
|
253
|
+
//////////////////////////
|
254
|
+
// create default queue //
|
255
|
+
//////////////////////////
|
256
|
+
this.defaultQueue = new Background.Queue(formattedQueueName, {
|
257
|
+
...defaultBullMQQueueOptions,
|
258
|
+
...(nativeBullMQ.defaultQueueOptions || {}),
|
259
|
+
connection: defaultQueueConnection,
|
260
|
+
});
|
261
|
+
///////////////////////////////
|
262
|
+
// end: create default queue //
|
263
|
+
///////////////////////////////
|
264
|
+
/////////////////////////////
|
265
|
+
// create default workers //
|
266
|
+
/////////////////////////////
|
267
|
+
if (activateWorkers) {
|
268
|
+
if (!defaultWorkerConnection)
|
269
|
+
throw new ActivatingBackgroundWorkersWithoutDefaultWorkerConnection();
|
270
|
+
const workerCount = nativeBullMQ.defaultWorkerCount ?? 1;
|
271
|
+
for (let i = 0; i < workerCount; i++) {
|
272
|
+
this._workers.push(new Background.Worker(formattedQueueName, job => this.doWork(job), {
|
273
|
+
...(backgroundOptions.nativeBullMQ.defaultWorkerOptions || {}),
|
274
|
+
connection: defaultWorkerConnection,
|
275
|
+
}));
|
276
|
+
}
|
277
|
+
}
|
278
|
+
/////////////////////////////////
|
279
|
+
// end: create default workers //
|
280
|
+
/////////////////////////////////
|
281
|
+
/////////////////////////
|
282
|
+
// create named queues //
|
283
|
+
/////////////////////////
|
284
|
+
const namedQueueOptionsMap = nativeBullMQ.namedQueueOptions || {};
|
285
|
+
Object.keys(namedQueueOptionsMap).forEach(queueName => {
|
286
|
+
const namedQueueOptions = namedQueueOptionsMap[queueName];
|
287
|
+
if (namedQueueOptions.queueConnection)
|
288
|
+
this.redisConnections.push(namedQueueOptions.queueConnection);
|
289
|
+
if (namedQueueOptions.workerConnection)
|
290
|
+
this.redisConnections.push(namedQueueOptions.workerConnection);
|
291
|
+
const namedQueueConnection = namedQueueOptions.queueConnection || defaultQueueConnection;
|
292
|
+
const namedWorkerConnection = namedQueueOptions.workerConnection || defaultWorkerConnection;
|
293
|
+
if (!namedQueueConnection)
|
294
|
+
throw new NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection(queueName);
|
295
|
+
const formattedQueuename = nameToRedisQueueName(queueName, namedQueueConnection);
|
296
|
+
this.namedQueues[queueName] = new Background.Queue(formattedQueuename, {
|
297
|
+
...defaultBullMQQueueOptions,
|
298
|
+
...namedQueueOptions,
|
299
|
+
connection: namedQueueConnection,
|
300
|
+
});
|
301
|
+
//////////////////////////
|
302
|
+
// create extra workers //
|
303
|
+
//////////////////////////
|
304
|
+
const extraWorkerOptionsMap = nativeBullMQ.namedQueueWorkers || {};
|
305
|
+
const extraWorkerOptions = extraWorkerOptionsMap[queueName];
|
306
|
+
const extraWorkerCount = extraWorkerOptions ? (extraWorkerOptions.workerCount ?? 1) : 0;
|
307
|
+
this.groupNames[queueName] ||= [];
|
308
|
+
if (extraWorkerOptions.group?.id)
|
309
|
+
this.groupNames[queueName].push(extraWorkerOptions.group.id);
|
310
|
+
if (activateWorkers) {
|
311
|
+
if (!namedWorkerConnection)
|
312
|
+
throw new ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection(queueName);
|
313
|
+
for (let i = 0; i < extraWorkerCount; i++) {
|
314
|
+
this._workers.push(new Background.Worker(formattedQueuename, job => this.doWork(job), {
|
315
|
+
...extraWorkerOptions,
|
316
|
+
connection: namedWorkerConnection,
|
317
|
+
}));
|
318
|
+
}
|
319
|
+
}
|
320
|
+
///////////////////////////////
|
321
|
+
// end: create extra workers //
|
322
|
+
///////////////////////////////
|
323
|
+
});
|
324
|
+
//////////////////////////////
|
325
|
+
// end: create named queues //
|
326
|
+
//////////////////////////////
|
327
|
+
}
|
328
|
+
work() {
|
329
|
+
this.connect({ activateWorkers: true });
|
330
|
+
process.on('SIGTERM', () => {
|
331
|
+
void this.shutdownAndExit();
|
332
|
+
});
|
333
|
+
process.on('SIGINT', () => {
|
334
|
+
void this.shutdownAndExit();
|
335
|
+
});
|
336
|
+
}
|
337
|
+
async staticMethod(ObjectClass, method, { delaySeconds, globalName, args = [], jobConfig = {}, }) {
|
338
|
+
this.connect();
|
339
|
+
await this._addToQueue(`BackgroundJobQueueStaticJob`, {
|
340
|
+
globalName,
|
341
|
+
method,
|
342
|
+
args,
|
343
|
+
}, {
|
344
|
+
delaySeconds,
|
345
|
+
jobConfig: jobConfig,
|
346
|
+
groupId: this.jobConfigToGroupId(jobConfig),
|
347
|
+
priority: this.jobConfigToPriority(jobConfig),
|
348
|
+
});
|
349
|
+
}
|
350
|
+
async scheduledMethod(ObjectClass, pattern, method, { globalName, args = [], jobConfig = {}, }) {
|
351
|
+
this.connect();
|
352
|
+
// `jobId` is used to determine uniqueness along with name and repeat pattern.
|
353
|
+
// Since the name is really a job type and never changes, the `jobId` is the only
|
354
|
+
// way to allow multiple jobs with the same cron repeat pattern. Uniqueness will
|
355
|
+
// now be enforced by combining class name, method name, and cron repeat pattern.
|
356
|
+
//
|
357
|
+
// See: https://docs.bullmq.io/guide/jobs/repeatable
|
358
|
+
const jobId = `${ObjectClass.name}:${method}`;
|
359
|
+
await this.queueInstance(jobConfig).add('BackgroundJobQueueStaticJob', {
|
360
|
+
globalName,
|
361
|
+
method,
|
362
|
+
args,
|
363
|
+
}, {
|
364
|
+
repeat: {
|
365
|
+
pattern,
|
366
|
+
},
|
367
|
+
jobId,
|
368
|
+
group: this.jobConfigToGroup(jobConfig),
|
369
|
+
priority: this.mapPriorityWordToPriorityNumber(this.jobConfigToPriority(jobConfig)),
|
370
|
+
// explicitly typing as JobsOptions because Psychic can't be aware of BullMQ Pro options
|
371
|
+
});
|
372
|
+
}
|
373
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
374
|
+
queueInstance(values) {
|
375
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
376
|
+
const workstreamConfig = values;
|
377
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
378
|
+
const queueConfig = values;
|
379
|
+
const queueInstance = typeof workstreamConfig.workstream === 'string'
|
380
|
+
? this.namedQueues[workstreamConfig.workstream]
|
381
|
+
: typeof queueConfig.queue === 'string'
|
382
|
+
? this.namedQueues[queueConfig.queue]
|
383
|
+
: this.defaultQueue;
|
384
|
+
if (!queueInstance) {
|
385
|
+
if (typeof workstreamConfig.workstream === 'string')
|
386
|
+
throw new NoQueueForSpecifiedWorkstream(workstreamConfig.workstream);
|
387
|
+
if (typeof queueConfig.queue === 'string')
|
388
|
+
throw new NoQueueForSpecifiedQueueName(queueConfig.queue);
|
389
|
+
}
|
390
|
+
return queueInstance;
|
391
|
+
}
|
392
|
+
async modelInstanceMethod(modelInstance, method, { delaySeconds, args = [], jobConfig = {}, }) {
|
393
|
+
this.connect();
|
394
|
+
await this._addToQueue('BackgroundJobQueueModelInstanceJob', {
|
395
|
+
id: modelInstance.primaryKeyValue,
|
396
|
+
globalName: modelInstance.constructor.globalName,
|
397
|
+
method,
|
398
|
+
args,
|
399
|
+
}, {
|
400
|
+
delaySeconds,
|
401
|
+
jobConfig: jobConfig,
|
402
|
+
groupId: this.jobConfigToGroupId(jobConfig),
|
403
|
+
priority: this.jobConfigToPriority(jobConfig),
|
404
|
+
});
|
405
|
+
}
|
406
|
+
// should be private, but public so we can test
|
407
|
+
async _addToQueue(jobType, jobData, { delaySeconds, jobConfig, priority, groupId, }) {
|
408
|
+
// set this variable out side of the conditional so that
|
409
|
+
// mismatches will raise exceptions even in tests
|
410
|
+
const queueInstance = this.queueInstance(jobConfig);
|
411
|
+
const delay = delaySeconds ? delaySeconds * 1000 : undefined;
|
412
|
+
if (EnvInternal.isTest && !EnvInternal.boolean('REALLY_TEST_BACKGROUND_QUEUE')) {
|
413
|
+
const queue = new Background.Queue('TestQueue', { connection: {} });
|
414
|
+
const job = new Job(queue, jobType, jobData, {});
|
415
|
+
await this.doWork(job);
|
416
|
+
//
|
417
|
+
}
|
418
|
+
else if (groupId && priority) {
|
419
|
+
await queueInstance.add(jobType, jobData, {
|
420
|
+
delay,
|
421
|
+
group: {
|
422
|
+
...this.groupIdToGroupConfig(groupId),
|
423
|
+
priority: this.mapPriorityWordToPriorityNumber(priority),
|
424
|
+
},
|
425
|
+
// explicitly typing as JobsOptions because Psychic can't be aware of BullMQ Pro options
|
426
|
+
});
|
427
|
+
//
|
428
|
+
}
|
429
|
+
else {
|
430
|
+
await queueInstance.add(jobType, jobData, {
|
431
|
+
delay,
|
432
|
+
group: this.groupIdToGroupConfig(groupId),
|
433
|
+
priority: this.mapPriorityWordToPriorityNumber(priority),
|
434
|
+
// explicitly typing as JobsOptions because Psychic can't be aware of BullMQ Pro options
|
435
|
+
});
|
436
|
+
}
|
437
|
+
}
|
438
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
439
|
+
jobConfigToPriority(jobConfig) {
|
440
|
+
if (!jobConfig)
|
441
|
+
return 'default';
|
442
|
+
return jobConfig.priority || 'default';
|
443
|
+
}
|
444
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
445
|
+
jobConfigToGroupId(jobConfig) {
|
446
|
+
if (!jobConfig)
|
447
|
+
return;
|
448
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
449
|
+
const workstreamConfig = jobConfig;
|
450
|
+
if (typeof workstreamConfig.workstream === 'string')
|
451
|
+
return workstreamConfig.workstream;
|
452
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
453
|
+
const queueConfig = jobConfig;
|
454
|
+
if (typeof queueConfig.groupId === 'string')
|
455
|
+
return queueConfig.groupId;
|
456
|
+
return;
|
457
|
+
}
|
458
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
459
|
+
jobConfigToGroup(jobConfig) {
|
460
|
+
return this.groupIdToGroupConfig(this.jobConfigToGroupId(jobConfig));
|
461
|
+
}
|
462
|
+
groupIdToGroupConfig(groupId) {
|
463
|
+
if (!groupId)
|
464
|
+
return;
|
465
|
+
return { id: groupId };
|
466
|
+
}
|
467
|
+
mapPriorityWordToPriorityNumber(priority) {
|
468
|
+
switch (priority) {
|
469
|
+
case 'urgent':
|
470
|
+
return 1;
|
471
|
+
case 'default':
|
472
|
+
return 2;
|
473
|
+
case 'not_urgent':
|
474
|
+
return 3;
|
475
|
+
case 'last':
|
476
|
+
return 4;
|
477
|
+
default:
|
478
|
+
return 2;
|
479
|
+
}
|
480
|
+
}
|
481
|
+
async doWork(job) {
|
482
|
+
const jobType = job.name;
|
483
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
484
|
+
const { id, method, args, globalName } = job.data;
|
485
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
486
|
+
let objectClass;
|
487
|
+
let dreamClass;
|
488
|
+
switch (jobType) {
|
489
|
+
case 'BackgroundJobQueueStaticJob':
|
490
|
+
if (globalName) {
|
491
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
492
|
+
objectClass = PsychicApplication.lookupClassByGlobalName(globalName);
|
493
|
+
}
|
494
|
+
if (!objectClass)
|
495
|
+
return;
|
496
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
497
|
+
await objectClass[method](...args, job);
|
498
|
+
break;
|
499
|
+
case 'BackgroundJobQueueModelInstanceJob':
|
500
|
+
if (globalName) {
|
501
|
+
dreamClass = PsychicApplication.lookupClassByGlobalName(globalName);
|
502
|
+
}
|
503
|
+
if (dreamClass) {
|
504
|
+
const modelInstance = await dreamClass.find(id);
|
505
|
+
if (!modelInstance)
|
506
|
+
return;
|
507
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
|
508
|
+
await modelInstance[method](...args, job);
|
509
|
+
}
|
510
|
+
break;
|
511
|
+
}
|
512
|
+
}
|
513
|
+
}
|
514
|
+
const background = new Background();
|
515
|
+
export default background;
|
516
|
+
export async function stopBackgroundWorkers() {
|
517
|
+
await background.shutdown();
|
518
|
+
}
|
519
|
+
function nameToRedisQueueName(queueName, redis) {
|
520
|
+
queueName = queueName.replace(/\{|\}/g, '');
|
521
|
+
if (redis instanceof Cluster)
|
522
|
+
return `{${queueName}}`;
|
523
|
+
return queueName;
|
524
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
export {};
|
@@ -0,0 +1,12 @@
|
|
1
|
+
export default class NoQueueForSpecifiedWorkstream extends Error {
|
2
|
+
workstream;
|
3
|
+
constructor(workstream) {
|
4
|
+
super();
|
5
|
+
this.workstream = workstream;
|
6
|
+
}
|
7
|
+
get message() {
|
8
|
+
return `Error enqueueing background job
|
9
|
+
No queue found for workstream "${this.workstream}"
|
10
|
+
`;
|
11
|
+
}
|
12
|
+
}
|
@@ -0,0 +1,7 @@
|
|
1
|
+
export { default as background, Background, stopBackgroundWorkers, } from './background/index.js';
|
2
|
+
export { default as BaseBackgroundedModel } from './background/BaseBackgroundedModel.js';
|
3
|
+
export { default as BaseBackgroundedService } from './background/BaseBackgroundedService.js';
|
4
|
+
export { default as BaseScheduledService } from './background/BaseScheduledService.js';
|
5
|
+
export { default as PsychicApplicationWorkers, } from './psychic-application-workers/index.js';
|
6
|
+
export { default as NoQueueForSpecifiedQueueName } from './error/background/NoQueueForSpecifiedQueueName.js';
|
7
|
+
export { default as NoQueueForSpecifiedWorkstream } from './error/background/NoQueueForSpecifiedWorkstream.js';
|
@@ -0,0 +1,12 @@
|
|
1
|
+
let _psychicWorkersApp = undefined;
|
2
|
+
export function cachePsychicWorkersApplication(psychicWorkersApp) {
|
3
|
+
_psychicWorkersApp = psychicWorkersApp;
|
4
|
+
}
|
5
|
+
export function getCachedPsychicWorkersApplication() {
|
6
|
+
return _psychicWorkersApp;
|
7
|
+
}
|
8
|
+
export function getCachedPsychicWorkersApplicationOrFail() {
|
9
|
+
if (!_psychicWorkersApp)
|
10
|
+
throw new Error('must call `cachePsychicWorkersApplication` before loading cached psychic application workers');
|
11
|
+
return _psychicWorkersApp;
|
12
|
+
}
|
@@ -0,0 +1,74 @@
|
|
1
|
+
import { Queue, Worker } from 'bullmq';
|
2
|
+
import background from '../background/index.js';
|
3
|
+
import { cachePsychicWorkersApplication, getCachedPsychicWorkersApplicationOrFail } from './cache.js';
|
4
|
+
export default class PsychicApplicationWorkers {
|
5
|
+
static async init(psychicApp, cb) {
|
6
|
+
const psychicWorkersApp = new PsychicApplicationWorkers(psychicApp);
|
7
|
+
await cb(psychicWorkersApp);
|
8
|
+
psychicApp.on('sync', () => {
|
9
|
+
background.connect();
|
10
|
+
const output = {
|
11
|
+
workstreamNames: [...background['workstreamNames']],
|
12
|
+
queueGroupMap: { ...background['groupNames'] },
|
13
|
+
};
|
14
|
+
return output;
|
15
|
+
});
|
16
|
+
psychicApp.on('server:shutdown', async () => {
|
17
|
+
await background.closeAllRedisConnections();
|
18
|
+
});
|
19
|
+
cachePsychicWorkersApplication(psychicWorkersApp);
|
20
|
+
return psychicWorkersApp;
|
21
|
+
}
|
22
|
+
/**
|
23
|
+
* Returns the cached psychic application if it has been set.
|
24
|
+
* If it has not been set, an exception is raised.
|
25
|
+
*
|
26
|
+
* The psychic application can be set by calling PsychicApplication#init
|
27
|
+
*/
|
28
|
+
static getOrFail() {
|
29
|
+
return getCachedPsychicWorkersApplicationOrFail();
|
30
|
+
}
|
31
|
+
psychicApp;
|
32
|
+
constructor(psychicApp) {
|
33
|
+
this.psychicApp = psychicApp;
|
34
|
+
}
|
35
|
+
/**
|
36
|
+
* Returns the background options provided by the user
|
37
|
+
*/
|
38
|
+
get backgroundOptions() {
|
39
|
+
return this._backgroundOptions;
|
40
|
+
}
|
41
|
+
_backgroundOptions;
|
42
|
+
_hooks = {
|
43
|
+
workerShutdown: [],
|
44
|
+
};
|
45
|
+
get hooks() {
|
46
|
+
return this._hooks;
|
47
|
+
}
|
48
|
+
on(hookEventType, cb) {
|
49
|
+
switch (hookEventType) {
|
50
|
+
case 'workers:shutdown':
|
51
|
+
this._hooks.workerShutdown.push(cb);
|
52
|
+
break;
|
53
|
+
default:
|
54
|
+
throw new Error(`unrecognized event provided to PsychicWorkersApplication#on: ${hookEventType}`);
|
55
|
+
}
|
56
|
+
}
|
57
|
+
set(option, value) {
|
58
|
+
switch (option) {
|
59
|
+
case 'background':
|
60
|
+
this._backgroundOptions = {
|
61
|
+
...{
|
62
|
+
providers: {
|
63
|
+
Queue,
|
64
|
+
Worker,
|
65
|
+
},
|
66
|
+
},
|
67
|
+
...value,
|
68
|
+
};
|
69
|
+
break;
|
70
|
+
default:
|
71
|
+
throw new Error(`Unhandled option type passed to PsychicWorkersApplication#set: ${option}`);
|
72
|
+
}
|
73
|
+
}
|
74
|
+
}
|