@rvoh/psychic-workers 2.9.1 → 2.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/src/background/helpers/backgroundQueues.js +69 -0
- package/dist/cjs/src/background/helpers/waitForClusterQueueConnectionsReady.js +106 -0
- package/dist/cjs/src/background/index.js +69 -182
- package/dist/cjs/src/psychic-app-workers/index.js +15 -0
- package/dist/esm/src/background/helpers/backgroundQueues.js +69 -0
- package/dist/esm/src/background/helpers/waitForClusterQueueConnectionsReady.js +106 -0
- package/dist/esm/src/background/index.js +69 -182
- package/dist/esm/src/psychic-app-workers/index.js +15 -0
- package/dist/types/src/background/helpers/backgroundQueues.d.ts +47 -0
- package/dist/types/src/background/helpers/waitForClusterQueueConnectionsReady.d.ts +37 -0
- package/dist/types/src/psychic-app-workers/index.d.ts +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { compact } from '@rvoh/dream/utils';
|
|
2
|
+
export default function backgroundQueues(backgroundOptions) {
|
|
3
|
+
if (!backgroundOptions)
|
|
4
|
+
return [];
|
|
5
|
+
return backgroundOptions.nativeBullMQ
|
|
6
|
+
? nativeQueues(backgroundOptions)
|
|
7
|
+
: simpleQueues(backgroundOptions);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* @internal
|
|
11
|
+
*
|
|
12
|
+
* The distinct queue connections (by object identity) the queues use: what the
|
|
13
|
+
* Cluster wait waits on. Worker connections are deliberately left out. Tolerates
|
|
14
|
+
* `undefined` options (a callback that never called `set('background')`).
|
|
15
|
+
*/
|
|
16
|
+
export function distinctQueueConnections(backgroundOptions) {
|
|
17
|
+
const connections = backgroundQueues(backgroundOptions).map(queue => queue.queueConnection);
|
|
18
|
+
return [...new Set(connections)].filter(connection => connection !== undefined);
|
|
19
|
+
}
|
|
20
|
+
function simpleQueues(level, transitional = false) {
|
|
21
|
+
const { defaultQueueConnection, defaultWorkerConnection } = level;
|
|
22
|
+
const { transitionalWorkstreams } = level;
|
|
23
|
+
return [
|
|
24
|
+
{
|
|
25
|
+
name: undefined,
|
|
26
|
+
transitional,
|
|
27
|
+
level,
|
|
28
|
+
own: undefined,
|
|
29
|
+
queueConnection: defaultQueueConnection,
|
|
30
|
+
workerConnection: defaultWorkerConnection,
|
|
31
|
+
ownConnections: compact([defaultQueueConnection, defaultWorkerConnection]),
|
|
32
|
+
},
|
|
33
|
+
...(level.namedWorkstreams || []).map(namedWorkstream => ({
|
|
34
|
+
name: namedWorkstream.name,
|
|
35
|
+
transitional,
|
|
36
|
+
level,
|
|
37
|
+
own: namedWorkstream,
|
|
38
|
+
queueConnection: namedWorkstream.queueConnection || defaultQueueConnection,
|
|
39
|
+
workerConnection: namedWorkstream.workerConnection || defaultWorkerConnection,
|
|
40
|
+
ownConnections: compact([namedWorkstream.queueConnection, namedWorkstream.workerConnection]),
|
|
41
|
+
})),
|
|
42
|
+
...(transitionalWorkstreams ? simpleQueues(transitionalWorkstreams, true) : []),
|
|
43
|
+
];
|
|
44
|
+
}
|
|
45
|
+
function nativeQueues(level) {
|
|
46
|
+
const { defaultQueueOptions, namedQueueOptions } = level.nativeBullMQ;
|
|
47
|
+
const defaultQueueConnection = defaultQueueOptions?.queueConnection || level.defaultQueueConnection;
|
|
48
|
+
const defaultWorkerConnection = defaultQueueOptions?.workerConnection || level.defaultWorkerConnection;
|
|
49
|
+
return [
|
|
50
|
+
{
|
|
51
|
+
name: undefined,
|
|
52
|
+
transitional: false,
|
|
53
|
+
level,
|
|
54
|
+
own: defaultQueueOptions,
|
|
55
|
+
queueConnection: defaultQueueConnection,
|
|
56
|
+
workerConnection: defaultWorkerConnection,
|
|
57
|
+
ownConnections: compact([defaultQueueConnection, defaultWorkerConnection]),
|
|
58
|
+
},
|
|
59
|
+
...Object.entries(namedQueueOptions || {}).map(([name, own]) => ({
|
|
60
|
+
name,
|
|
61
|
+
transitional: false,
|
|
62
|
+
level,
|
|
63
|
+
own,
|
|
64
|
+
queueConnection: own.queueConnection || defaultQueueConnection,
|
|
65
|
+
workerConnection: own.workerConnection || defaultWorkerConnection,
|
|
66
|
+
ownConnections: compact([own.queueConnection, own.workerConnection]),
|
|
67
|
+
})),
|
|
68
|
+
];
|
|
69
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { Cluster } from 'ioredis';
|
|
2
|
+
/**
|
|
3
|
+
* @internal
|
|
4
|
+
*
|
|
5
|
+
* How long the wait may take before the boot fails. It has to outlast one
|
|
6
|
+
* failed slot refresh plus a retry under the create-psychic scaffold's settings
|
|
7
|
+
* (10s `slotsRefreshTimeout`, 1-20s retry delay) with room for the next
|
|
8
|
+
* attempt to succeed. Deliberately not a public option.
|
|
9
|
+
*/
|
|
10
|
+
export const CLUSTER_READY_TIMEOUT_MS = 60_000;
|
|
11
|
+
/**
|
|
12
|
+
* @internal
|
|
13
|
+
*
|
|
14
|
+
* Resolves once every ioredis `Cluster` among `connections` is `'ready'`.
|
|
15
|
+
*
|
|
16
|
+
* A `Cluster` built with `enableOfflineQueue: false` reaches `'connect'` before
|
|
17
|
+
* `'ready'`, and BullMQ treats `'connect'` as ready, so a `Queue` built then
|
|
18
|
+
* rejects its first command. Standalone `Redis` connections are not waited on,
|
|
19
|
+
* nor is a `Cluster` already `'ready'`, nor a `lazyConnect` one still at
|
|
20
|
+
* `'wait'`: BullMQ connects that one on first use and holds the first command
|
|
21
|
+
* until it is `'ready'`, and it is never connected here.
|
|
22
|
+
*
|
|
23
|
+
* Rejects if a waited-on Cluster ends, or when `timeoutMs` passes. The error is
|
|
24
|
+
* the most recent one ioredis emitted for that Cluster (any Cluster, on a
|
|
25
|
+
* timeout), or a plain `Error` when it emitted none. A failed wait calls
|
|
26
|
+
* `disconnect()` on every Cluster it was waiting on whose status is not
|
|
27
|
+
* `'end'`, so ioredis stops reconnecting and stops logging an unhandled error
|
|
28
|
+
* on every retry now that no `'error'` listener remains. (Never `disconnect(true)`,
|
|
29
|
+
* which keeps reconnecting.)
|
|
30
|
+
*
|
|
31
|
+
* Every listener and the timer are removed when the wait settles. A Cluster
|
|
32
|
+
* that drops after reaching `'ready'` while a sibling is still pending is waited
|
|
33
|
+
* on again, under the same bound.
|
|
34
|
+
*/
|
|
35
|
+
export default async function waitForClusterQueueConnectionsReady(connections, { timeoutMs = CLUSTER_READY_TIMEOUT_MS } = {}) {
|
|
36
|
+
const clusters = connections.filter((connection) => connection instanceof Cluster && connection.status !== 'ready' && connection.status !== 'wait');
|
|
37
|
+
if (!clusters.length)
|
|
38
|
+
return;
|
|
39
|
+
await new Promise((resolve, reject) => {
|
|
40
|
+
let settled = false;
|
|
41
|
+
let lastError;
|
|
42
|
+
const lastErrors = new Map();
|
|
43
|
+
const cleanups = [];
|
|
44
|
+
function settle() {
|
|
45
|
+
settled = true;
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
cleanups.forEach(cleanup => cleanup());
|
|
48
|
+
}
|
|
49
|
+
function fail(error) {
|
|
50
|
+
if (settled)
|
|
51
|
+
return;
|
|
52
|
+
settle();
|
|
53
|
+
for (const cluster of clusters) {
|
|
54
|
+
// 'end' is final; disconnect() on it would rewrite the status
|
|
55
|
+
if (cluster.status === 'end')
|
|
56
|
+
continue;
|
|
57
|
+
try {
|
|
58
|
+
cluster.disconnect();
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// nothing more to do for a connection that cannot be closed
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
reject(error);
|
|
65
|
+
}
|
|
66
|
+
// reads each status at the moment it is called, so a Cluster that reached
|
|
67
|
+
// 'ready' and dropped again before its siblings were ready is still waited on
|
|
68
|
+
function resolveIfAllReady() {
|
|
69
|
+
if (settled)
|
|
70
|
+
return;
|
|
71
|
+
if (clusters.some(cluster => cluster.status !== 'ready'))
|
|
72
|
+
return;
|
|
73
|
+
settle();
|
|
74
|
+
resolve();
|
|
75
|
+
}
|
|
76
|
+
const timer = setTimeout(() => {
|
|
77
|
+
fail(lastError ??
|
|
78
|
+
new Error(`Timed out after ${timeoutMs}ms waiting for the Redis Cluster queue connection to be ready`));
|
|
79
|
+
}, timeoutMs);
|
|
80
|
+
for (const cluster of clusters) {
|
|
81
|
+
// an already-ended Cluster earlier in the list settled the wait: settle()
|
|
82
|
+
// ran only the cleanups pushed so far, so attaching more would leak
|
|
83
|
+
if (settled)
|
|
84
|
+
break;
|
|
85
|
+
const onError = (error) => {
|
|
86
|
+
lastError = error;
|
|
87
|
+
lastErrors.set(cluster, error);
|
|
88
|
+
};
|
|
89
|
+
const onEnd = () => {
|
|
90
|
+
fail(lastErrors.get(cluster) ?? new Error('A Redis Cluster queue connection ended before it was ready'));
|
|
91
|
+
};
|
|
92
|
+
cluster.on('error', onError);
|
|
93
|
+
cluster.on('end', onEnd);
|
|
94
|
+
cluster.on('ready', resolveIfAllReady);
|
|
95
|
+
cleanups.push(() => {
|
|
96
|
+
cluster.off('error', onError);
|
|
97
|
+
cluster.off('end', onEnd);
|
|
98
|
+
cluster.off('ready', resolveIfAllReady);
|
|
99
|
+
});
|
|
100
|
+
// ioredis never re-emits 'end', so a Cluster already there would hang until the bound
|
|
101
|
+
if (cluster.status === 'end')
|
|
102
|
+
onEnd();
|
|
103
|
+
}
|
|
104
|
+
resolveIfAllReady();
|
|
105
|
+
});
|
|
106
|
+
}
|
|
@@ -18,6 +18,7 @@ import EnvInternal from '../helpers/EnvInternal.js';
|
|
|
18
18
|
import PsychicAppWorkers from '../psychic-app-workers/index.js';
|
|
19
19
|
import AttemtedToBackgroundEntireDreamModel from '../error/background/AttemtedToBackgroundEntireDreamModel.js';
|
|
20
20
|
import DebouncedJobOutpacesRateLimit from '../error/background/DebouncedJobOutpacesRateLimit.js';
|
|
21
|
+
import backgroundQueues from './helpers/backgroundQueues.js';
|
|
21
22
|
import nameToRedisQueueName from './helpers/nameToRedisQueueName.js';
|
|
22
23
|
const DEFAULT_CONCURRENCY = 10;
|
|
23
24
|
/**
|
|
@@ -344,118 +345,55 @@ export class Background {
|
|
|
344
345
|
*
|
|
345
346
|
* connects to BullMQ using workstream-based arguments
|
|
346
347
|
*/
|
|
347
|
-
simpleConnect(defaultBullMQQueueOptions, backgroundOptions
|
|
348
|
+
simpleConnect(defaultBullMQQueueOptions, backgroundOptions) {
|
|
348
349
|
// refused before any queue or worker of this connect exists, for the current and the
|
|
349
|
-
// transitional workstreams alike
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const defaultQueueConnection = backgroundOptions.defaultQueueConnection;
|
|
355
|
-
const defaultWorkerConnection = backgroundOptions.defaultWorkerConnection;
|
|
356
|
-
if (defaultQueueConnection)
|
|
357
|
-
this.redisConnections.push(defaultQueueConnection);
|
|
358
|
-
if (defaultWorkerConnection)
|
|
359
|
-
this.redisConnections.push(defaultWorkerConnection);
|
|
360
|
-
// transitional queues must have the same names they had prior to making them
|
|
361
|
-
// transitional since the name is what identifies the queues and enables the
|
|
362
|
-
// queues to be worked off
|
|
363
|
-
const formattedQueueName = nameToRedisQueueName(Background.defaultQueueName, defaultQueueConnection);
|
|
364
|
-
///////////////////////////////
|
|
365
|
-
// create default workstream //
|
|
366
|
-
///////////////////////////////
|
|
367
|
-
const defaultQueue = new Background.Queue(formattedQueueName, {
|
|
368
|
-
...defaultBullMQQueueOptions,
|
|
369
|
-
connection: defaultQueueConnection,
|
|
370
|
-
});
|
|
371
|
-
if (activatingTransitionalWorkstreams) {
|
|
372
|
-
this.defaultTransitionalQueue = defaultQueue;
|
|
373
|
-
}
|
|
374
|
-
else {
|
|
375
|
-
this.defaultQueue = defaultQueue;
|
|
376
|
-
}
|
|
377
|
-
////////////////////////////////////
|
|
378
|
-
// end: create default workstream //
|
|
379
|
-
////////////////////////////////////
|
|
380
|
-
/////////////////////////////
|
|
381
|
-
// create default workers //
|
|
382
|
-
/////////////////////////////
|
|
383
|
-
const defaultWorkerOptions = {
|
|
384
|
-
autorun: !EnvInternal.isTest,
|
|
385
|
-
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
386
|
-
concurrency: backgroundOptions.defaultWorkstream?.concurrency || DEFAULT_CONCURRENCY,
|
|
387
|
-
};
|
|
388
|
-
this.recordQueueWorkers(defaultQueue, defaultWorkerOptions, defaultWorkerConnection, backgroundOptions.defaultWorkstream?.workerCount ?? 1, {
|
|
389
|
-
mode: 'simple',
|
|
390
|
-
isDefaultQueue: true,
|
|
391
|
-
configuredName: Background.defaultQueueName,
|
|
392
|
-
transitional: activatingTransitionalWorkstreams,
|
|
393
|
-
});
|
|
394
|
-
/////////////////////////////////
|
|
395
|
-
// end: create default workers //
|
|
396
|
-
/////////////////////////////////
|
|
397
|
-
//////////////////////////////
|
|
398
|
-
// create named workstreams //
|
|
399
|
-
//////////////////////////////
|
|
400
|
-
const namedWorkstreams = backgroundOptions.namedWorkstreams || [];
|
|
401
|
-
namedWorkstreams.forEach(namedWorkstream => {
|
|
402
|
-
if (namedWorkstream.queueConnection)
|
|
403
|
-
this.redisConnections.push(namedWorkstream.queueConnection);
|
|
404
|
-
if (namedWorkstream.workerConnection)
|
|
405
|
-
this.redisConnections.push(namedWorkstream.workerConnection);
|
|
406
|
-
const namedWorkstreamQueueConnection = namedWorkstream.queueConnection || defaultQueueConnection;
|
|
407
|
-
const namedWorkstreamWorkerConnection = namedWorkstream.workerConnection || defaultWorkerConnection;
|
|
350
|
+
// transitional workstreams alike
|
|
351
|
+
this.assertNamedWorkstreamRateLimits(backgroundOptions.namedWorkstreams, false);
|
|
352
|
+
this.assertNamedWorkstreamRateLimits(backgroundOptions.transitionalWorkstreams?.namedWorkstreams, true);
|
|
353
|
+
for (const { name, transitional, level, own: namedWorkstream, ...connections } of backgroundQueues(backgroundOptions)) {
|
|
354
|
+
this.redisConnections.push(...connections.ownConnections);
|
|
408
355
|
// transitional queues must have the same names they had prior to making them
|
|
409
356
|
// transitional since the name is what identifies the queues and enables the
|
|
410
357
|
// queues to be worked off
|
|
411
|
-
const
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
358
|
+
const queue = new Background.Queue(nameToRedisQueueName(name ?? Background.defaultQueueName, connections.queueConnection), { ...defaultBullMQQueueOptions, connection: connections.queueConnection });
|
|
359
|
+
if (name === undefined) {
|
|
360
|
+
if (transitional)
|
|
361
|
+
this.defaultTransitionalQueue = queue;
|
|
362
|
+
else
|
|
363
|
+
this.defaultQueue = queue;
|
|
364
|
+
}
|
|
365
|
+
else if (transitional) {
|
|
366
|
+
this.namedTransitionalQueues[name] = queue;
|
|
418
367
|
}
|
|
419
368
|
else {
|
|
420
|
-
this.namedQueues[
|
|
421
|
-
this.workstreamNames.push(
|
|
369
|
+
this.namedQueues[name] = queue;
|
|
370
|
+
this.workstreamNames.push(name);
|
|
422
371
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
//////////////////////////
|
|
426
|
-
const namedWorkerOptions = {
|
|
372
|
+
const workstream = namedWorkstream ?? level.defaultWorkstream;
|
|
373
|
+
const workerOptions = {
|
|
427
374
|
autorun: !EnvInternal.isTest,
|
|
428
|
-
...
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
375
|
+
...level.defaultBullMQWorkerOptions,
|
|
376
|
+
...(namedWorkstream
|
|
377
|
+
? {
|
|
378
|
+
// open-source BullMQ rate limiting (queue-wide, shared by every worker on this queue);
|
|
379
|
+
// conditional spread so the key is absent when no rateLimit is configured
|
|
380
|
+
...(namedWorkstream.rateLimit ? { limiter: namedWorkstream.rateLimit } : {}),
|
|
381
|
+
// BullMQ Pro option (ignored by open-source BullMQ); Psychic can't be aware of BullMQ Pro options
|
|
382
|
+
group: {
|
|
383
|
+
// Pro's worker `group` option is `{ limit, concurrency }` with no `id` (grouping
|
|
384
|
+
// happens at job add, see `_addToQueue`); not verified against an installed Pro build
|
|
385
|
+
id: namedWorkstream.name,
|
|
386
|
+
limit: namedWorkstream.rateLimit,
|
|
387
|
+
},
|
|
388
|
+
}
|
|
389
|
+
: {}),
|
|
390
|
+
concurrency: workstream?.concurrency || DEFAULT_CONCURRENCY,
|
|
440
391
|
};
|
|
441
|
-
this.recordQueueWorkers(
|
|
392
|
+
this.recordQueueWorkers(queue, workerOptions, connections.workerConnection, workstream?.workerCount ?? 1, {
|
|
442
393
|
mode: 'simple',
|
|
443
|
-
isDefaultQueue:
|
|
444
|
-
configuredName:
|
|
445
|
-
transitional
|
|
446
|
-
});
|
|
447
|
-
///////////////////////////////
|
|
448
|
-
// end: create named workers //
|
|
449
|
-
///////////////////////////////
|
|
450
|
-
});
|
|
451
|
-
///////////////////////////////////
|
|
452
|
-
// end: create named workstreams //
|
|
453
|
-
///////////////////////////////////
|
|
454
|
-
const transitionalWorkstreams = backgroundOptions
|
|
455
|
-
.transitionalWorkstreams;
|
|
456
|
-
if (transitionalWorkstreams) {
|
|
457
|
-
this.simpleConnect(defaultBullMQQueueOptions, transitionalWorkstreams, {
|
|
458
|
-
activatingTransitionalWorkstreams: true,
|
|
394
|
+
isDefaultQueue: name === undefined,
|
|
395
|
+
configuredName: name ?? Background.defaultQueueName,
|
|
396
|
+
transitional,
|
|
459
397
|
});
|
|
460
398
|
}
|
|
461
399
|
}
|
|
@@ -466,92 +404,41 @@ export class Background {
|
|
|
466
404
|
*/
|
|
467
405
|
nativeBullMQConnect(defaultBullMQQueueOptions, backgroundOptions) {
|
|
468
406
|
const nativeBullMQ = backgroundOptions.nativeBullMQ;
|
|
469
|
-
const
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
//////////////////////////
|
|
479
|
-
// create default queue //
|
|
480
|
-
//////////////////////////
|
|
481
|
-
const defaultQueue = new Background.Queue(formattedQueueName, {
|
|
482
|
-
...defaultBullMQQueueOptions,
|
|
483
|
-
...nativeBullMQ.defaultQueueOptions,
|
|
484
|
-
connection: defaultQueueConnection,
|
|
485
|
-
});
|
|
486
|
-
this.defaultQueue = defaultQueue;
|
|
487
|
-
///////////////////////////////
|
|
488
|
-
// end: create default queue //
|
|
489
|
-
///////////////////////////////
|
|
490
|
-
/////////////////////////////
|
|
491
|
-
// create default workers //
|
|
492
|
-
/////////////////////////////
|
|
493
|
-
const defaultWorkerOptions = {
|
|
494
|
-
autorun: !EnvInternal.isTest,
|
|
495
|
-
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
496
|
-
...backgroundOptions.nativeBullMQ.defaultWorkerOptions,
|
|
497
|
-
};
|
|
498
|
-
this.recordQueueWorkers(defaultQueue, defaultWorkerOptions, defaultWorkerConnection, nativeBullMQ.defaultWorkerCount ?? 1, {
|
|
499
|
-
mode: 'native',
|
|
500
|
-
isDefaultQueue: true,
|
|
501
|
-
configuredName: Background.defaultQueueName,
|
|
502
|
-
transitional: false,
|
|
503
|
-
});
|
|
504
|
-
/////////////////////////////////
|
|
505
|
-
// end: create default workers //
|
|
506
|
-
/////////////////////////////////
|
|
507
|
-
/////////////////////////
|
|
508
|
-
// create named queues //
|
|
509
|
-
/////////////////////////
|
|
510
|
-
const namedQueueOptionsMap = nativeBullMQ.namedQueueOptions || {};
|
|
511
|
-
Object.keys(namedQueueOptionsMap).forEach(queueName => {
|
|
512
|
-
const namedQueueOptions = namedQueueOptionsMap[queueName];
|
|
513
|
-
if (namedQueueOptions.queueConnection)
|
|
514
|
-
this.redisConnections.push(namedQueueOptions.queueConnection);
|
|
515
|
-
if (namedQueueOptions.workerConnection)
|
|
516
|
-
this.redisConnections.push(namedQueueOptions.workerConnection);
|
|
517
|
-
const namedQueueConnection = namedQueueOptions.queueConnection || defaultQueueConnection;
|
|
518
|
-
const namedWorkerConnection = namedQueueOptions.workerConnection || defaultWorkerConnection;
|
|
519
|
-
if (!namedQueueConnection)
|
|
520
|
-
throw new NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection(queueName);
|
|
521
|
-
const formattedQueuename = nameToRedisQueueName(queueName, namedQueueConnection);
|
|
522
|
-
const namedQueue = new Background.Queue(formattedQueuename, {
|
|
523
|
-
...defaultBullMQQueueOptions,
|
|
524
|
-
...namedQueueOptions,
|
|
525
|
-
connection: namedQueueConnection,
|
|
526
|
-
});
|
|
527
|
-
this.namedQueues[queueName] = namedQueue;
|
|
528
|
-
//////////////////////////
|
|
529
|
-
// create extra workers //
|
|
530
|
-
//////////////////////////
|
|
531
|
-
const extraWorkerOptionsMap = nativeBullMQ.namedQueueWorkers || {};
|
|
532
|
-
const extraWorkerOptions = extraWorkerOptionsMap[queueName];
|
|
533
|
-
const extraWorkerCount = extraWorkerOptions ? (extraWorkerOptions.workerCount ?? 1) : 0;
|
|
534
|
-
this.groupNames[queueName] ||= [];
|
|
535
|
-
if (extraWorkerOptions?.group?.id)
|
|
536
|
-
this.groupNames[queueName].push(extraWorkerOptions.group.id);
|
|
537
|
-
const namedWorkerOptions = {
|
|
407
|
+
for (const { name, own: queueOptions, ...connections } of backgroundQueues(backgroundOptions)) {
|
|
408
|
+
this.redisConnections.push(...connections.ownConnections);
|
|
409
|
+
if (!connections.queueConnection) {
|
|
410
|
+
if (name === undefined)
|
|
411
|
+
throw new DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection();
|
|
412
|
+
throw new NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection(name);
|
|
413
|
+
}
|
|
414
|
+
const queue = new Background.Queue(nameToRedisQueueName(name ?? Background.defaultQueueName, connections.queueConnection), { ...defaultBullMQQueueOptions, ...queueOptions, connection: connections.queueConnection });
|
|
415
|
+
let workerOptions = {
|
|
538
416
|
autorun: !EnvInternal.isTest,
|
|
539
417
|
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
540
|
-
...extraWorkerOptions,
|
|
541
418
|
};
|
|
542
|
-
|
|
419
|
+
let workerCount;
|
|
420
|
+
if (name === undefined) {
|
|
421
|
+
this.defaultQueue = queue;
|
|
422
|
+
workerOptions = { ...workerOptions, ...nativeBullMQ.defaultWorkerOptions };
|
|
423
|
+
workerCount = nativeBullMQ.defaultWorkerCount ?? 1;
|
|
424
|
+
}
|
|
425
|
+
else {
|
|
426
|
+
this.namedQueues[name] = queue;
|
|
427
|
+
// extra workers
|
|
428
|
+
const extraWorkerOptions = nativeBullMQ.namedQueueWorkers?.[name];
|
|
429
|
+
workerOptions = { ...workerOptions, ...extraWorkerOptions };
|
|
430
|
+
workerCount = extraWorkerOptions ? (extraWorkerOptions.workerCount ?? 1) : 0;
|
|
431
|
+
this.groupNames[name] ||= [];
|
|
432
|
+
if (extraWorkerOptions?.group?.id)
|
|
433
|
+
this.groupNames[name].push(extraWorkerOptions.group.id);
|
|
434
|
+
}
|
|
435
|
+
this.recordQueueWorkers(queue, workerOptions, connections.workerConnection, workerCount, {
|
|
543
436
|
mode: 'native',
|
|
544
|
-
isDefaultQueue:
|
|
545
|
-
configuredName:
|
|
437
|
+
isDefaultQueue: name === undefined,
|
|
438
|
+
configuredName: name ?? Background.defaultQueueName,
|
|
546
439
|
transitional: false,
|
|
547
440
|
});
|
|
548
|
-
|
|
549
|
-
// end: create extra workers //
|
|
550
|
-
///////////////////////////////
|
|
551
|
-
});
|
|
552
|
-
//////////////////////////////
|
|
553
|
-
// end: create named queues //
|
|
554
|
-
//////////////////////////////
|
|
441
|
+
}
|
|
555
442
|
}
|
|
556
443
|
/**
|
|
557
444
|
* @internal
|
|
@@ -1,12 +1,27 @@
|
|
|
1
1
|
import { Queue, Worker } from 'bullmq';
|
|
2
2
|
import background from '../background/index.js';
|
|
3
|
+
import { distinctQueueConnections } from '../background/helpers/backgroundQueues.js';
|
|
4
|
+
import waitForClusterQueueConnectionsReady from '../background/helpers/waitForClusterQueueConnectionsReady.js';
|
|
3
5
|
import ASTWorkersSchemaBuilder from '../cli/ASTWorkersSchemaBuilder.js';
|
|
4
6
|
import PsychicTypesDeprecation from '../cli/PsychicTypesDeprecation.js';
|
|
5
7
|
import { cachePsychicWorkersApp, getCachedPsychicWorkersAppOrFail } from './cache.js';
|
|
6
8
|
export default class PsychicAppWorkers {
|
|
9
|
+
/**
|
|
10
|
+
* Runs the callback that configures the workers plugin, then waits for every
|
|
11
|
+
* ioredis `Cluster` queue connection it configured to be `'ready'` before
|
|
12
|
+
* resolving, so app code is never handed a producer ioredis would reject.
|
|
13
|
+
*
|
|
14
|
+
* A Cluster that cannot become ready rejects `init` (with ioredis's last
|
|
15
|
+
* error, or a timeout) and is disconnected, so any process that inits the
|
|
16
|
+
* workers plugin, including dream CLI commands such as `db:migrate`, needs a
|
|
17
|
+
* reachable Redis Cluster to finish booting. Standalone `Redis` connections and
|
|
18
|
+
* `lazyConnect` Clusters still at `'wait'` are not waited on, and init never
|
|
19
|
+
* connects them.
|
|
20
|
+
*/
|
|
7
21
|
static async init(psychicApp, cb) {
|
|
8
22
|
const psychicWorkersApp = new PsychicAppWorkers(psychicApp);
|
|
9
23
|
await cb(psychicWorkersApp);
|
|
24
|
+
await waitForClusterQueueConnectionsReady(distinctQueueConnections(psychicWorkersApp.backgroundOptions));
|
|
10
25
|
psychicApp.on('cli:sync', async () => {
|
|
11
26
|
await new ASTWorkersSchemaBuilder().build();
|
|
12
27
|
await new PsychicTypesDeprecation().deprecate();
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { compact } from '@rvoh/dream/utils';
|
|
2
|
+
export default function backgroundQueues(backgroundOptions) {
|
|
3
|
+
if (!backgroundOptions)
|
|
4
|
+
return [];
|
|
5
|
+
return backgroundOptions.nativeBullMQ
|
|
6
|
+
? nativeQueues(backgroundOptions)
|
|
7
|
+
: simpleQueues(backgroundOptions);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* @internal
|
|
11
|
+
*
|
|
12
|
+
* The distinct queue connections (by object identity) the queues use: what the
|
|
13
|
+
* Cluster wait waits on. Worker connections are deliberately left out. Tolerates
|
|
14
|
+
* `undefined` options (a callback that never called `set('background')`).
|
|
15
|
+
*/
|
|
16
|
+
export function distinctQueueConnections(backgroundOptions) {
|
|
17
|
+
const connections = backgroundQueues(backgroundOptions).map(queue => queue.queueConnection);
|
|
18
|
+
return [...new Set(connections)].filter(connection => connection !== undefined);
|
|
19
|
+
}
|
|
20
|
+
function simpleQueues(level, transitional = false) {
|
|
21
|
+
const { defaultQueueConnection, defaultWorkerConnection } = level;
|
|
22
|
+
const { transitionalWorkstreams } = level;
|
|
23
|
+
return [
|
|
24
|
+
{
|
|
25
|
+
name: undefined,
|
|
26
|
+
transitional,
|
|
27
|
+
level,
|
|
28
|
+
own: undefined,
|
|
29
|
+
queueConnection: defaultQueueConnection,
|
|
30
|
+
workerConnection: defaultWorkerConnection,
|
|
31
|
+
ownConnections: compact([defaultQueueConnection, defaultWorkerConnection]),
|
|
32
|
+
},
|
|
33
|
+
...(level.namedWorkstreams || []).map(namedWorkstream => ({
|
|
34
|
+
name: namedWorkstream.name,
|
|
35
|
+
transitional,
|
|
36
|
+
level,
|
|
37
|
+
own: namedWorkstream,
|
|
38
|
+
queueConnection: namedWorkstream.queueConnection || defaultQueueConnection,
|
|
39
|
+
workerConnection: namedWorkstream.workerConnection || defaultWorkerConnection,
|
|
40
|
+
ownConnections: compact([namedWorkstream.queueConnection, namedWorkstream.workerConnection]),
|
|
41
|
+
})),
|
|
42
|
+
...(transitionalWorkstreams ? simpleQueues(transitionalWorkstreams, true) : []),
|
|
43
|
+
];
|
|
44
|
+
}
|
|
45
|
+
function nativeQueues(level) {
|
|
46
|
+
const { defaultQueueOptions, namedQueueOptions } = level.nativeBullMQ;
|
|
47
|
+
const defaultQueueConnection = defaultQueueOptions?.queueConnection || level.defaultQueueConnection;
|
|
48
|
+
const defaultWorkerConnection = defaultQueueOptions?.workerConnection || level.defaultWorkerConnection;
|
|
49
|
+
return [
|
|
50
|
+
{
|
|
51
|
+
name: undefined,
|
|
52
|
+
transitional: false,
|
|
53
|
+
level,
|
|
54
|
+
own: defaultQueueOptions,
|
|
55
|
+
queueConnection: defaultQueueConnection,
|
|
56
|
+
workerConnection: defaultWorkerConnection,
|
|
57
|
+
ownConnections: compact([defaultQueueConnection, defaultWorkerConnection]),
|
|
58
|
+
},
|
|
59
|
+
...Object.entries(namedQueueOptions || {}).map(([name, own]) => ({
|
|
60
|
+
name,
|
|
61
|
+
transitional: false,
|
|
62
|
+
level,
|
|
63
|
+
own,
|
|
64
|
+
queueConnection: own.queueConnection || defaultQueueConnection,
|
|
65
|
+
workerConnection: own.workerConnection || defaultWorkerConnection,
|
|
66
|
+
ownConnections: compact([own.queueConnection, own.workerConnection]),
|
|
67
|
+
})),
|
|
68
|
+
];
|
|
69
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { Cluster } from 'ioredis';
|
|
2
|
+
/**
|
|
3
|
+
* @internal
|
|
4
|
+
*
|
|
5
|
+
* How long the wait may take before the boot fails. It has to outlast one
|
|
6
|
+
* failed slot refresh plus a retry under the create-psychic scaffold's settings
|
|
7
|
+
* (10s `slotsRefreshTimeout`, 1-20s retry delay) with room for the next
|
|
8
|
+
* attempt to succeed. Deliberately not a public option.
|
|
9
|
+
*/
|
|
10
|
+
export const CLUSTER_READY_TIMEOUT_MS = 60_000;
|
|
11
|
+
/**
|
|
12
|
+
* @internal
|
|
13
|
+
*
|
|
14
|
+
* Resolves once every ioredis `Cluster` among `connections` is `'ready'`.
|
|
15
|
+
*
|
|
16
|
+
* A `Cluster` built with `enableOfflineQueue: false` reaches `'connect'` before
|
|
17
|
+
* `'ready'`, and BullMQ treats `'connect'` as ready, so a `Queue` built then
|
|
18
|
+
* rejects its first command. Standalone `Redis` connections are not waited on,
|
|
19
|
+
* nor is a `Cluster` already `'ready'`, nor a `lazyConnect` one still at
|
|
20
|
+
* `'wait'`: BullMQ connects that one on first use and holds the first command
|
|
21
|
+
* until it is `'ready'`, and it is never connected here.
|
|
22
|
+
*
|
|
23
|
+
* Rejects if a waited-on Cluster ends, or when `timeoutMs` passes. The error is
|
|
24
|
+
* the most recent one ioredis emitted for that Cluster (any Cluster, on a
|
|
25
|
+
* timeout), or a plain `Error` when it emitted none. A failed wait calls
|
|
26
|
+
* `disconnect()` on every Cluster it was waiting on whose status is not
|
|
27
|
+
* `'end'`, so ioredis stops reconnecting and stops logging an unhandled error
|
|
28
|
+
* on every retry now that no `'error'` listener remains. (Never `disconnect(true)`,
|
|
29
|
+
* which keeps reconnecting.)
|
|
30
|
+
*
|
|
31
|
+
* Every listener and the timer are removed when the wait settles. A Cluster
|
|
32
|
+
* that drops after reaching `'ready'` while a sibling is still pending is waited
|
|
33
|
+
* on again, under the same bound.
|
|
34
|
+
*/
|
|
35
|
+
export default async function waitForClusterQueueConnectionsReady(connections, { timeoutMs = CLUSTER_READY_TIMEOUT_MS } = {}) {
|
|
36
|
+
const clusters = connections.filter((connection) => connection instanceof Cluster && connection.status !== 'ready' && connection.status !== 'wait');
|
|
37
|
+
if (!clusters.length)
|
|
38
|
+
return;
|
|
39
|
+
await new Promise((resolve, reject) => {
|
|
40
|
+
let settled = false;
|
|
41
|
+
let lastError;
|
|
42
|
+
const lastErrors = new Map();
|
|
43
|
+
const cleanups = [];
|
|
44
|
+
function settle() {
|
|
45
|
+
settled = true;
|
|
46
|
+
clearTimeout(timer);
|
|
47
|
+
cleanups.forEach(cleanup => cleanup());
|
|
48
|
+
}
|
|
49
|
+
function fail(error) {
|
|
50
|
+
if (settled)
|
|
51
|
+
return;
|
|
52
|
+
settle();
|
|
53
|
+
for (const cluster of clusters) {
|
|
54
|
+
// 'end' is final; disconnect() on it would rewrite the status
|
|
55
|
+
if (cluster.status === 'end')
|
|
56
|
+
continue;
|
|
57
|
+
try {
|
|
58
|
+
cluster.disconnect();
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// nothing more to do for a connection that cannot be closed
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
reject(error);
|
|
65
|
+
}
|
|
66
|
+
// reads each status at the moment it is called, so a Cluster that reached
|
|
67
|
+
// 'ready' and dropped again before its siblings were ready is still waited on
|
|
68
|
+
function resolveIfAllReady() {
|
|
69
|
+
if (settled)
|
|
70
|
+
return;
|
|
71
|
+
if (clusters.some(cluster => cluster.status !== 'ready'))
|
|
72
|
+
return;
|
|
73
|
+
settle();
|
|
74
|
+
resolve();
|
|
75
|
+
}
|
|
76
|
+
const timer = setTimeout(() => {
|
|
77
|
+
fail(lastError ??
|
|
78
|
+
new Error(`Timed out after ${timeoutMs}ms waiting for the Redis Cluster queue connection to be ready`));
|
|
79
|
+
}, timeoutMs);
|
|
80
|
+
for (const cluster of clusters) {
|
|
81
|
+
// an already-ended Cluster earlier in the list settled the wait: settle()
|
|
82
|
+
// ran only the cleanups pushed so far, so attaching more would leak
|
|
83
|
+
if (settled)
|
|
84
|
+
break;
|
|
85
|
+
const onError = (error) => {
|
|
86
|
+
lastError = error;
|
|
87
|
+
lastErrors.set(cluster, error);
|
|
88
|
+
};
|
|
89
|
+
const onEnd = () => {
|
|
90
|
+
fail(lastErrors.get(cluster) ?? new Error('A Redis Cluster queue connection ended before it was ready'));
|
|
91
|
+
};
|
|
92
|
+
cluster.on('error', onError);
|
|
93
|
+
cluster.on('end', onEnd);
|
|
94
|
+
cluster.on('ready', resolveIfAllReady);
|
|
95
|
+
cleanups.push(() => {
|
|
96
|
+
cluster.off('error', onError);
|
|
97
|
+
cluster.off('end', onEnd);
|
|
98
|
+
cluster.off('ready', resolveIfAllReady);
|
|
99
|
+
});
|
|
100
|
+
// ioredis never re-emits 'end', so a Cluster already there would hang until the bound
|
|
101
|
+
if (cluster.status === 'end')
|
|
102
|
+
onEnd();
|
|
103
|
+
}
|
|
104
|
+
resolveIfAllReady();
|
|
105
|
+
});
|
|
106
|
+
}
|
|
@@ -18,6 +18,7 @@ import EnvInternal from '../helpers/EnvInternal.js';
|
|
|
18
18
|
import PsychicAppWorkers from '../psychic-app-workers/index.js';
|
|
19
19
|
import AttemtedToBackgroundEntireDreamModel from '../error/background/AttemtedToBackgroundEntireDreamModel.js';
|
|
20
20
|
import DebouncedJobOutpacesRateLimit from '../error/background/DebouncedJobOutpacesRateLimit.js';
|
|
21
|
+
import backgroundQueues from './helpers/backgroundQueues.js';
|
|
21
22
|
import nameToRedisQueueName from './helpers/nameToRedisQueueName.js';
|
|
22
23
|
const DEFAULT_CONCURRENCY = 10;
|
|
23
24
|
/**
|
|
@@ -344,118 +345,55 @@ export class Background {
|
|
|
344
345
|
*
|
|
345
346
|
* connects to BullMQ using workstream-based arguments
|
|
346
347
|
*/
|
|
347
|
-
simpleConnect(defaultBullMQQueueOptions, backgroundOptions
|
|
348
|
+
simpleConnect(defaultBullMQQueueOptions, backgroundOptions) {
|
|
348
349
|
// refused before any queue or worker of this connect exists, for the current and the
|
|
349
|
-
// transitional workstreams alike
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
const defaultQueueConnection = backgroundOptions.defaultQueueConnection;
|
|
355
|
-
const defaultWorkerConnection = backgroundOptions.defaultWorkerConnection;
|
|
356
|
-
if (defaultQueueConnection)
|
|
357
|
-
this.redisConnections.push(defaultQueueConnection);
|
|
358
|
-
if (defaultWorkerConnection)
|
|
359
|
-
this.redisConnections.push(defaultWorkerConnection);
|
|
360
|
-
// transitional queues must have the same names they had prior to making them
|
|
361
|
-
// transitional since the name is what identifies the queues and enables the
|
|
362
|
-
// queues to be worked off
|
|
363
|
-
const formattedQueueName = nameToRedisQueueName(Background.defaultQueueName, defaultQueueConnection);
|
|
364
|
-
///////////////////////////////
|
|
365
|
-
// create default workstream //
|
|
366
|
-
///////////////////////////////
|
|
367
|
-
const defaultQueue = new Background.Queue(formattedQueueName, {
|
|
368
|
-
...defaultBullMQQueueOptions,
|
|
369
|
-
connection: defaultQueueConnection,
|
|
370
|
-
});
|
|
371
|
-
if (activatingTransitionalWorkstreams) {
|
|
372
|
-
this.defaultTransitionalQueue = defaultQueue;
|
|
373
|
-
}
|
|
374
|
-
else {
|
|
375
|
-
this.defaultQueue = defaultQueue;
|
|
376
|
-
}
|
|
377
|
-
////////////////////////////////////
|
|
378
|
-
// end: create default workstream //
|
|
379
|
-
////////////////////////////////////
|
|
380
|
-
/////////////////////////////
|
|
381
|
-
// create default workers //
|
|
382
|
-
/////////////////////////////
|
|
383
|
-
const defaultWorkerOptions = {
|
|
384
|
-
autorun: !EnvInternal.isTest,
|
|
385
|
-
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
386
|
-
concurrency: backgroundOptions.defaultWorkstream?.concurrency || DEFAULT_CONCURRENCY,
|
|
387
|
-
};
|
|
388
|
-
this.recordQueueWorkers(defaultQueue, defaultWorkerOptions, defaultWorkerConnection, backgroundOptions.defaultWorkstream?.workerCount ?? 1, {
|
|
389
|
-
mode: 'simple',
|
|
390
|
-
isDefaultQueue: true,
|
|
391
|
-
configuredName: Background.defaultQueueName,
|
|
392
|
-
transitional: activatingTransitionalWorkstreams,
|
|
393
|
-
});
|
|
394
|
-
/////////////////////////////////
|
|
395
|
-
// end: create default workers //
|
|
396
|
-
/////////////////////////////////
|
|
397
|
-
//////////////////////////////
|
|
398
|
-
// create named workstreams //
|
|
399
|
-
//////////////////////////////
|
|
400
|
-
const namedWorkstreams = backgroundOptions.namedWorkstreams || [];
|
|
401
|
-
namedWorkstreams.forEach(namedWorkstream => {
|
|
402
|
-
if (namedWorkstream.queueConnection)
|
|
403
|
-
this.redisConnections.push(namedWorkstream.queueConnection);
|
|
404
|
-
if (namedWorkstream.workerConnection)
|
|
405
|
-
this.redisConnections.push(namedWorkstream.workerConnection);
|
|
406
|
-
const namedWorkstreamQueueConnection = namedWorkstream.queueConnection || defaultQueueConnection;
|
|
407
|
-
const namedWorkstreamWorkerConnection = namedWorkstream.workerConnection || defaultWorkerConnection;
|
|
350
|
+
// transitional workstreams alike
|
|
351
|
+
this.assertNamedWorkstreamRateLimits(backgroundOptions.namedWorkstreams, false);
|
|
352
|
+
this.assertNamedWorkstreamRateLimits(backgroundOptions.transitionalWorkstreams?.namedWorkstreams, true);
|
|
353
|
+
for (const { name, transitional, level, own: namedWorkstream, ...connections } of backgroundQueues(backgroundOptions)) {
|
|
354
|
+
this.redisConnections.push(...connections.ownConnections);
|
|
408
355
|
// transitional queues must have the same names they had prior to making them
|
|
409
356
|
// transitional since the name is what identifies the queues and enables the
|
|
410
357
|
// queues to be worked off
|
|
411
|
-
const
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
358
|
+
const queue = new Background.Queue(nameToRedisQueueName(name ?? Background.defaultQueueName, connections.queueConnection), { ...defaultBullMQQueueOptions, connection: connections.queueConnection });
|
|
359
|
+
if (name === undefined) {
|
|
360
|
+
if (transitional)
|
|
361
|
+
this.defaultTransitionalQueue = queue;
|
|
362
|
+
else
|
|
363
|
+
this.defaultQueue = queue;
|
|
364
|
+
}
|
|
365
|
+
else if (transitional) {
|
|
366
|
+
this.namedTransitionalQueues[name] = queue;
|
|
418
367
|
}
|
|
419
368
|
else {
|
|
420
|
-
this.namedQueues[
|
|
421
|
-
this.workstreamNames.push(
|
|
369
|
+
this.namedQueues[name] = queue;
|
|
370
|
+
this.workstreamNames.push(name);
|
|
422
371
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
//////////////////////////
|
|
426
|
-
const namedWorkerOptions = {
|
|
372
|
+
const workstream = namedWorkstream ?? level.defaultWorkstream;
|
|
373
|
+
const workerOptions = {
|
|
427
374
|
autorun: !EnvInternal.isTest,
|
|
428
|
-
...
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
375
|
+
...level.defaultBullMQWorkerOptions,
|
|
376
|
+
...(namedWorkstream
|
|
377
|
+
? {
|
|
378
|
+
// open-source BullMQ rate limiting (queue-wide, shared by every worker on this queue);
|
|
379
|
+
// conditional spread so the key is absent when no rateLimit is configured
|
|
380
|
+
...(namedWorkstream.rateLimit ? { limiter: namedWorkstream.rateLimit } : {}),
|
|
381
|
+
// BullMQ Pro option (ignored by open-source BullMQ); Psychic can't be aware of BullMQ Pro options
|
|
382
|
+
group: {
|
|
383
|
+
// Pro's worker `group` option is `{ limit, concurrency }` with no `id` (grouping
|
|
384
|
+
// happens at job add, see `_addToQueue`); not verified against an installed Pro build
|
|
385
|
+
id: namedWorkstream.name,
|
|
386
|
+
limit: namedWorkstream.rateLimit,
|
|
387
|
+
},
|
|
388
|
+
}
|
|
389
|
+
: {}),
|
|
390
|
+
concurrency: workstream?.concurrency || DEFAULT_CONCURRENCY,
|
|
440
391
|
};
|
|
441
|
-
this.recordQueueWorkers(
|
|
392
|
+
this.recordQueueWorkers(queue, workerOptions, connections.workerConnection, workstream?.workerCount ?? 1, {
|
|
442
393
|
mode: 'simple',
|
|
443
|
-
isDefaultQueue:
|
|
444
|
-
configuredName:
|
|
445
|
-
transitional
|
|
446
|
-
});
|
|
447
|
-
///////////////////////////////
|
|
448
|
-
// end: create named workers //
|
|
449
|
-
///////////////////////////////
|
|
450
|
-
});
|
|
451
|
-
///////////////////////////////////
|
|
452
|
-
// end: create named workstreams //
|
|
453
|
-
///////////////////////////////////
|
|
454
|
-
const transitionalWorkstreams = backgroundOptions
|
|
455
|
-
.transitionalWorkstreams;
|
|
456
|
-
if (transitionalWorkstreams) {
|
|
457
|
-
this.simpleConnect(defaultBullMQQueueOptions, transitionalWorkstreams, {
|
|
458
|
-
activatingTransitionalWorkstreams: true,
|
|
394
|
+
isDefaultQueue: name === undefined,
|
|
395
|
+
configuredName: name ?? Background.defaultQueueName,
|
|
396
|
+
transitional,
|
|
459
397
|
});
|
|
460
398
|
}
|
|
461
399
|
}
|
|
@@ -466,92 +404,41 @@ export class Background {
|
|
|
466
404
|
*/
|
|
467
405
|
nativeBullMQConnect(defaultBullMQQueueOptions, backgroundOptions) {
|
|
468
406
|
const nativeBullMQ = backgroundOptions.nativeBullMQ;
|
|
469
|
-
const
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
//////////////////////////
|
|
479
|
-
// create default queue //
|
|
480
|
-
//////////////////////////
|
|
481
|
-
const defaultQueue = new Background.Queue(formattedQueueName, {
|
|
482
|
-
...defaultBullMQQueueOptions,
|
|
483
|
-
...nativeBullMQ.defaultQueueOptions,
|
|
484
|
-
connection: defaultQueueConnection,
|
|
485
|
-
});
|
|
486
|
-
this.defaultQueue = defaultQueue;
|
|
487
|
-
///////////////////////////////
|
|
488
|
-
// end: create default queue //
|
|
489
|
-
///////////////////////////////
|
|
490
|
-
/////////////////////////////
|
|
491
|
-
// create default workers //
|
|
492
|
-
/////////////////////////////
|
|
493
|
-
const defaultWorkerOptions = {
|
|
494
|
-
autorun: !EnvInternal.isTest,
|
|
495
|
-
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
496
|
-
...backgroundOptions.nativeBullMQ.defaultWorkerOptions,
|
|
497
|
-
};
|
|
498
|
-
this.recordQueueWorkers(defaultQueue, defaultWorkerOptions, defaultWorkerConnection, nativeBullMQ.defaultWorkerCount ?? 1, {
|
|
499
|
-
mode: 'native',
|
|
500
|
-
isDefaultQueue: true,
|
|
501
|
-
configuredName: Background.defaultQueueName,
|
|
502
|
-
transitional: false,
|
|
503
|
-
});
|
|
504
|
-
/////////////////////////////////
|
|
505
|
-
// end: create default workers //
|
|
506
|
-
/////////////////////////////////
|
|
507
|
-
/////////////////////////
|
|
508
|
-
// create named queues //
|
|
509
|
-
/////////////////////////
|
|
510
|
-
const namedQueueOptionsMap = nativeBullMQ.namedQueueOptions || {};
|
|
511
|
-
Object.keys(namedQueueOptionsMap).forEach(queueName => {
|
|
512
|
-
const namedQueueOptions = namedQueueOptionsMap[queueName];
|
|
513
|
-
if (namedQueueOptions.queueConnection)
|
|
514
|
-
this.redisConnections.push(namedQueueOptions.queueConnection);
|
|
515
|
-
if (namedQueueOptions.workerConnection)
|
|
516
|
-
this.redisConnections.push(namedQueueOptions.workerConnection);
|
|
517
|
-
const namedQueueConnection = namedQueueOptions.queueConnection || defaultQueueConnection;
|
|
518
|
-
const namedWorkerConnection = namedQueueOptions.workerConnection || defaultWorkerConnection;
|
|
519
|
-
if (!namedQueueConnection)
|
|
520
|
-
throw new NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection(queueName);
|
|
521
|
-
const formattedQueuename = nameToRedisQueueName(queueName, namedQueueConnection);
|
|
522
|
-
const namedQueue = new Background.Queue(formattedQueuename, {
|
|
523
|
-
...defaultBullMQQueueOptions,
|
|
524
|
-
...namedQueueOptions,
|
|
525
|
-
connection: namedQueueConnection,
|
|
526
|
-
});
|
|
527
|
-
this.namedQueues[queueName] = namedQueue;
|
|
528
|
-
//////////////////////////
|
|
529
|
-
// create extra workers //
|
|
530
|
-
//////////////////////////
|
|
531
|
-
const extraWorkerOptionsMap = nativeBullMQ.namedQueueWorkers || {};
|
|
532
|
-
const extraWorkerOptions = extraWorkerOptionsMap[queueName];
|
|
533
|
-
const extraWorkerCount = extraWorkerOptions ? (extraWorkerOptions.workerCount ?? 1) : 0;
|
|
534
|
-
this.groupNames[queueName] ||= [];
|
|
535
|
-
if (extraWorkerOptions?.group?.id)
|
|
536
|
-
this.groupNames[queueName].push(extraWorkerOptions.group.id);
|
|
537
|
-
const namedWorkerOptions = {
|
|
407
|
+
for (const { name, own: queueOptions, ...connections } of backgroundQueues(backgroundOptions)) {
|
|
408
|
+
this.redisConnections.push(...connections.ownConnections);
|
|
409
|
+
if (!connections.queueConnection) {
|
|
410
|
+
if (name === undefined)
|
|
411
|
+
throw new DefaultBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection();
|
|
412
|
+
throw new NamedBullMQNativeOptionsMissingQueueConnectionAndDefaultQueueConnection(name);
|
|
413
|
+
}
|
|
414
|
+
const queue = new Background.Queue(nameToRedisQueueName(name ?? Background.defaultQueueName, connections.queueConnection), { ...defaultBullMQQueueOptions, ...queueOptions, connection: connections.queueConnection });
|
|
415
|
+
let workerOptions = {
|
|
538
416
|
autorun: !EnvInternal.isTest,
|
|
539
417
|
...backgroundOptions.defaultBullMQWorkerOptions,
|
|
540
|
-
...extraWorkerOptions,
|
|
541
418
|
};
|
|
542
|
-
|
|
419
|
+
let workerCount;
|
|
420
|
+
if (name === undefined) {
|
|
421
|
+
this.defaultQueue = queue;
|
|
422
|
+
workerOptions = { ...workerOptions, ...nativeBullMQ.defaultWorkerOptions };
|
|
423
|
+
workerCount = nativeBullMQ.defaultWorkerCount ?? 1;
|
|
424
|
+
}
|
|
425
|
+
else {
|
|
426
|
+
this.namedQueues[name] = queue;
|
|
427
|
+
// extra workers
|
|
428
|
+
const extraWorkerOptions = nativeBullMQ.namedQueueWorkers?.[name];
|
|
429
|
+
workerOptions = { ...workerOptions, ...extraWorkerOptions };
|
|
430
|
+
workerCount = extraWorkerOptions ? (extraWorkerOptions.workerCount ?? 1) : 0;
|
|
431
|
+
this.groupNames[name] ||= [];
|
|
432
|
+
if (extraWorkerOptions?.group?.id)
|
|
433
|
+
this.groupNames[name].push(extraWorkerOptions.group.id);
|
|
434
|
+
}
|
|
435
|
+
this.recordQueueWorkers(queue, workerOptions, connections.workerConnection, workerCount, {
|
|
543
436
|
mode: 'native',
|
|
544
|
-
isDefaultQueue:
|
|
545
|
-
configuredName:
|
|
437
|
+
isDefaultQueue: name === undefined,
|
|
438
|
+
configuredName: name ?? Background.defaultQueueName,
|
|
546
439
|
transitional: false,
|
|
547
440
|
});
|
|
548
|
-
|
|
549
|
-
// end: create extra workers //
|
|
550
|
-
///////////////////////////////
|
|
551
|
-
});
|
|
552
|
-
//////////////////////////////
|
|
553
|
-
// end: create named queues //
|
|
554
|
-
//////////////////////////////
|
|
441
|
+
}
|
|
555
442
|
}
|
|
556
443
|
/**
|
|
557
444
|
* @internal
|
|
@@ -1,12 +1,27 @@
|
|
|
1
1
|
import { Queue, Worker } from 'bullmq';
|
|
2
2
|
import background from '../background/index.js';
|
|
3
|
+
import { distinctQueueConnections } from '../background/helpers/backgroundQueues.js';
|
|
4
|
+
import waitForClusterQueueConnectionsReady from '../background/helpers/waitForClusterQueueConnectionsReady.js';
|
|
3
5
|
import ASTWorkersSchemaBuilder from '../cli/ASTWorkersSchemaBuilder.js';
|
|
4
6
|
import PsychicTypesDeprecation from '../cli/PsychicTypesDeprecation.js';
|
|
5
7
|
import { cachePsychicWorkersApp, getCachedPsychicWorkersAppOrFail } from './cache.js';
|
|
6
8
|
export default class PsychicAppWorkers {
|
|
9
|
+
/**
|
|
10
|
+
* Runs the callback that configures the workers plugin, then waits for every
|
|
11
|
+
* ioredis `Cluster` queue connection it configured to be `'ready'` before
|
|
12
|
+
* resolving, so app code is never handed a producer ioredis would reject.
|
|
13
|
+
*
|
|
14
|
+
* A Cluster that cannot become ready rejects `init` (with ioredis's last
|
|
15
|
+
* error, or a timeout) and is disconnected, so any process that inits the
|
|
16
|
+
* workers plugin, including dream CLI commands such as `db:migrate`, needs a
|
|
17
|
+
* reachable Redis Cluster to finish booting. Standalone `Redis` connections and
|
|
18
|
+
* `lazyConnect` Clusters still at `'wait'` are not waited on, and init never
|
|
19
|
+
* connects them.
|
|
20
|
+
*/
|
|
7
21
|
static async init(psychicApp, cb) {
|
|
8
22
|
const psychicWorkersApp = new PsychicAppWorkers(psychicApp);
|
|
9
23
|
await cb(psychicWorkersApp);
|
|
24
|
+
await waitForClusterQueueConnectionsReady(distinctQueueConnections(psychicWorkersApp.backgroundOptions));
|
|
10
25
|
psychicApp.on('cli:sync', async () => {
|
|
11
26
|
await new ASTWorkersSchemaBuilder().build();
|
|
12
27
|
await new PsychicTypesDeprecation().deprecate();
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { PsychicBackgroundNativeBullMQOptions, PsychicBackgroundSimpleOptions, PsychicBackgroundWorkstreamOptions, QueueOptionsWithConnectionInstance, RedisOrRedisClusterConnection, TransitionalPsychicBackgroundSimpleOptions } from '../../psychic-app-workers/index.js';
|
|
2
|
+
import type { PsychicBackgroundOptions } from '../../types/background.js';
|
|
3
|
+
type Connection = RedisOrRedisClusterConnection | undefined;
|
|
4
|
+
type SimpleLevel = PsychicBackgroundSimpleOptions | TransitionalPsychicBackgroundSimpleOptions;
|
|
5
|
+
/**
|
|
6
|
+
* @internal
|
|
7
|
+
*
|
|
8
|
+
* One queue `Background#connect` builds, with the connections it resolved to.
|
|
9
|
+
* `level` is the options object the queue is configured in (the transitional
|
|
10
|
+
* options for a transitional queue) and `own` the options of that one queue (the
|
|
11
|
+
* named workstream, or `nativeBullMQ.defaultQueueOptions` / the named queue's
|
|
12
|
+
* options in native mode; `undefined` for simple mode's default queue).
|
|
13
|
+
*/
|
|
14
|
+
export interface BackgroundQueue<Level, Own, QueueConnection extends Connection = Connection> {
|
|
15
|
+
/** the configured name; `undefined` for the default queue */
|
|
16
|
+
name: string | undefined;
|
|
17
|
+
transitional: boolean;
|
|
18
|
+
level: Level;
|
|
19
|
+
own: Own | undefined;
|
|
20
|
+
queueConnection: QueueConnection;
|
|
21
|
+
workerConnection: Connection;
|
|
22
|
+
/** the connections to close on shutdown: the resolved ones for the default queue, only the queue's own otherwise */
|
|
23
|
+
ownConnections: RedisOrRedisClusterConnection[];
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* @internal
|
|
27
|
+
*
|
|
28
|
+
* The one walk over the background options, in the order `Background#connect`
|
|
29
|
+
* builds queues: the default queue, the named ones, then simple mode's
|
|
30
|
+
* transitional default and named. It holds every rule for which connections a
|
|
31
|
+
* queue uses (a named queue falls back to the default's; native
|
|
32
|
+
* `defaultQueueOptions` beat the top-level defaults), so `connect()` and
|
|
33
|
+
* `distinctQueueConnections` cannot disagree. A missing connection stays
|
|
34
|
+
* `undefined`; deciding whether that is an error is the caller's.
|
|
35
|
+
*/
|
|
36
|
+
export default function backgroundQueues(backgroundOptions: PsychicBackgroundNativeBullMQOptions): BackgroundQueue<PsychicBackgroundNativeBullMQOptions, QueueOptionsWithConnectionInstance>[];
|
|
37
|
+
export default function backgroundQueues(backgroundOptions: PsychicBackgroundSimpleOptions): BackgroundQueue<SimpleLevel, PsychicBackgroundWorkstreamOptions, RedisOrRedisClusterConnection>[];
|
|
38
|
+
export default function backgroundQueues(backgroundOptions: PsychicBackgroundOptions | undefined): BackgroundQueue<unknown, unknown>[];
|
|
39
|
+
/**
|
|
40
|
+
* @internal
|
|
41
|
+
*
|
|
42
|
+
* The distinct queue connections (by object identity) the queues use: what the
|
|
43
|
+
* Cluster wait waits on. Worker connections are deliberately left out. Tolerates
|
|
44
|
+
* `undefined` options (a callback that never called `set('background')`).
|
|
45
|
+
*/
|
|
46
|
+
export declare function distinctQueueConnections(backgroundOptions: PsychicBackgroundOptions | undefined): RedisOrRedisClusterConnection[];
|
|
47
|
+
export {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { RedisOrRedisClusterConnection } from '../../psychic-app-workers/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* @internal
|
|
4
|
+
*
|
|
5
|
+
* How long the wait may take before the boot fails. It has to outlast one
|
|
6
|
+
* failed slot refresh plus a retry under the create-psychic scaffold's settings
|
|
7
|
+
* (10s `slotsRefreshTimeout`, 1-20s retry delay) with room for the next
|
|
8
|
+
* attempt to succeed. Deliberately not a public option.
|
|
9
|
+
*/
|
|
10
|
+
export declare const CLUSTER_READY_TIMEOUT_MS = 60000;
|
|
11
|
+
/**
|
|
12
|
+
* @internal
|
|
13
|
+
*
|
|
14
|
+
* Resolves once every ioredis `Cluster` among `connections` is `'ready'`.
|
|
15
|
+
*
|
|
16
|
+
* A `Cluster` built with `enableOfflineQueue: false` reaches `'connect'` before
|
|
17
|
+
* `'ready'`, and BullMQ treats `'connect'` as ready, so a `Queue` built then
|
|
18
|
+
* rejects its first command. Standalone `Redis` connections are not waited on,
|
|
19
|
+
* nor is a `Cluster` already `'ready'`, nor a `lazyConnect` one still at
|
|
20
|
+
* `'wait'`: BullMQ connects that one on first use and holds the first command
|
|
21
|
+
* until it is `'ready'`, and it is never connected here.
|
|
22
|
+
*
|
|
23
|
+
* Rejects if a waited-on Cluster ends, or when `timeoutMs` passes. The error is
|
|
24
|
+
* the most recent one ioredis emitted for that Cluster (any Cluster, on a
|
|
25
|
+
* timeout), or a plain `Error` when it emitted none. A failed wait calls
|
|
26
|
+
* `disconnect()` on every Cluster it was waiting on whose status is not
|
|
27
|
+
* `'end'`, so ioredis stops reconnecting and stops logging an unhandled error
|
|
28
|
+
* on every retry now that no `'error'` listener remains. (Never `disconnect(true)`,
|
|
29
|
+
* which keeps reconnecting.)
|
|
30
|
+
*
|
|
31
|
+
* Every listener and the timer are removed when the wait settles. A Cluster
|
|
32
|
+
* that drops after reaching `'ready'` while a sibling is still pending is waited
|
|
33
|
+
* on again, under the same bound.
|
|
34
|
+
*/
|
|
35
|
+
export default function waitForClusterQueueConnectionsReady(connections: RedisOrRedisClusterConnection[], { timeoutMs }?: {
|
|
36
|
+
timeoutMs?: number;
|
|
37
|
+
}): Promise<void>;
|
|
@@ -3,6 +3,18 @@ import { QueueOptions, WorkerOptions } from 'bullmq';
|
|
|
3
3
|
import { Cluster, Redis } from 'ioredis';
|
|
4
4
|
import { PsychicBackgroundOptions } from '../types/background.js';
|
|
5
5
|
export default class PsychicAppWorkers {
|
|
6
|
+
/**
|
|
7
|
+
* Runs the callback that configures the workers plugin, then waits for every
|
|
8
|
+
* ioredis `Cluster` queue connection it configured to be `'ready'` before
|
|
9
|
+
* resolving, so app code is never handed a producer ioredis would reject.
|
|
10
|
+
*
|
|
11
|
+
* A Cluster that cannot become ready rejects `init` (with ioredis's last
|
|
12
|
+
* error, or a timeout) and is disconnected, so any process that inits the
|
|
13
|
+
* workers plugin, including dream CLI commands such as `db:migrate`, needs a
|
|
14
|
+
* reachable Redis Cluster to finish booting. Standalone `Redis` connections and
|
|
15
|
+
* `lazyConnect` Clusters still at `'wait'` are not waited on, and init never
|
|
16
|
+
* connects them.
|
|
17
|
+
*/
|
|
6
18
|
static init(psychicApp: PsychicApp, cb: (app: PsychicAppWorkers) => void | Promise<void>): Promise<PsychicAppWorkers>;
|
|
7
19
|
/**
|
|
8
20
|
* Returns the cached psychic application if it has been set.
|