@rvoh/psychic-workers 0.4.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +0 -0
- package/dist/cjs/src/background/BaseBackgroundedService.js +1 -1
- package/dist/cjs/src/background/helpers/nameToRedisQueueName.js +2 -1
- package/dist/cjs/src/background/helpers/parallelTestSafeQueueName.js +10 -0
- package/dist/cjs/src/background/index.js +9 -1
- package/dist/cjs/src/index.js +3 -1
- package/dist/cjs/src/psychic-app-workers/index.js +21 -1
- package/dist/cjs/src/test-utils/WorkerTestUtils.js +124 -0
- package/dist/esm/src/background/BaseBackgroundedService.js +1 -1
- package/dist/esm/src/background/helpers/nameToRedisQueueName.js +2 -1
- package/dist/esm/src/background/helpers/parallelTestSafeQueueName.js +7 -0
- package/dist/esm/src/background/index.js +9 -1
- package/dist/esm/src/index.js +1 -0
- package/dist/esm/src/psychic-app-workers/index.js +21 -1
- package/dist/esm/src/test-utils/WorkerTestUtils.js +121 -0
- package/dist/types/src/background/helpers/nameToRedisQueueName.d.ts +1 -1
- package/dist/types/src/background/helpers/parallelTestSafeQueueName.d.ts +1 -0
- package/dist/types/src/helpers/EnvInternal.d.ts +2 -1
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/psychic-app-workers/index.d.ts +15 -2
- package/dist/types/src/test-utils/WorkerTestUtils.d.ts +19 -0
- package/dist/types/src/types/background.d.ts +1 -1
- package/package.json +3 -3
package/CHANGELOG.md
ADDED
File without changes
|
@@ -1,8 +1,8 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
const dream_1 = require("@rvoh/dream");
|
4
|
-
const index_js_1 = require("./index.js");
|
5
4
|
const durationToSeconds_js_1 = require("../helpers/durationToSeconds.js");
|
5
|
+
const index_js_1 = require("./index.js");
|
6
6
|
class BaseBackgroundedService {
|
7
7
|
/**
|
8
8
|
* A getter meant to be overridden in child classes. This does
|
@@ -2,9 +2,10 @@
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.default = nameToRedisQueueName;
|
4
4
|
const ioredis_1 = require("ioredis");
|
5
|
+
const parallelTestSafeQueueName_js_1 = require("./parallelTestSafeQueueName.js");
|
5
6
|
function nameToRedisQueueName(queueName, redis) {
|
6
7
|
queueName = queueName.replace(/\{|\}/g, '');
|
7
8
|
if (redis instanceof ioredis_1.Cluster)
|
8
9
|
return `{${queueName}}`;
|
9
|
-
return queueName;
|
10
|
+
return (0, parallelTestSafeQueueName_js_1.default)(queueName);
|
10
11
|
}
|
@@ -0,0 +1,10 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.default = parallelTestSafeQueueName;
|
4
|
+
const EnvInternal_js_1 = require("../../helpers/EnvInternal.js");
|
5
|
+
function parallelTestSafeQueueName(queueName) {
|
6
|
+
if (EnvInternal_js_1.default.isTest && (EnvInternal_js_1.default.integer('VITEST_POOL_ID', { optional: true }) || 0) > 1) {
|
7
|
+
queueName = `${queueName}-${EnvInternal_js_1.default.integer('VITEST_POOL_ID')}`;
|
8
|
+
}
|
9
|
+
return queueName;
|
10
|
+
}
|
@@ -207,6 +207,7 @@ class Background {
|
|
207
207
|
const workerCount = backgroundOptions.defaultWorkstream?.workerCount ?? 1;
|
208
208
|
for (let i = 0; i < workerCount; i++) {
|
209
209
|
this._workers.push(new Background.Worker(formattedQueueName, async (job) => await this.doWork(job), {
|
210
|
+
autorun: !EnvInternal_js_1.default.isTest,
|
210
211
|
connection: defaultWorkerConnection,
|
211
212
|
concurrency: backgroundOptions.defaultWorkstream?.concurrency || DEFAULT_CONCURRENCY,
|
212
213
|
}));
|
@@ -250,6 +251,7 @@ class Background {
|
|
250
251
|
const workerCount = namedWorkstream.workerCount ?? 1;
|
251
252
|
for (let i = 0; i < workerCount; i++) {
|
252
253
|
this._workers.push(new Background.Worker(namedWorkstreamFormattedQueueName, async (job) => await this.doWork(job), {
|
254
|
+
autorun: !EnvInternal_js_1.default.isTest,
|
253
255
|
group: {
|
254
256
|
id: namedWorkstream.name,
|
255
257
|
limit: namedWorkstream.rateLimit,
|
@@ -312,6 +314,7 @@ class Background {
|
|
312
314
|
const workerCount = nativeBullMQ.defaultWorkerCount ?? 1;
|
313
315
|
for (let i = 0; i < workerCount; i++) {
|
314
316
|
this._workers.push(new Background.Worker(formattedQueueName, async (job) => await this.doWork(job), {
|
317
|
+
autorun: !EnvInternal_js_1.default.isTest,
|
315
318
|
...(backgroundOptions.nativeBullMQ.defaultWorkerOptions || {}),
|
316
319
|
connection: defaultWorkerConnection,
|
317
320
|
}));
|
@@ -354,6 +357,7 @@ class Background {
|
|
354
357
|
throw new ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection_js_1.default(queueName);
|
355
358
|
for (let i = 0; i < extraWorkerCount; i++) {
|
356
359
|
this._workers.push(new Background.Worker(formattedQueuename, async (job) => await this.doWork(job), {
|
360
|
+
autorun: !EnvInternal_js_1.default.isTest,
|
357
361
|
...extraWorkerOptions,
|
358
362
|
connection: namedWorkerConnection,
|
359
363
|
}));
|
@@ -504,7 +508,11 @@ class Background {
|
|
504
508
|
// if delaySeconds is 0, we will intentionally treat
|
505
509
|
// this as `undefined`
|
506
510
|
const delay = delaySeconds ? delaySeconds * 1000 : undefined;
|
507
|
-
|
511
|
+
const workersApp = index_js_1.default.getOrFail();
|
512
|
+
// in test environments, this block will short-circuit adding to the queue,
|
513
|
+
// causing the job to immediately invoke instead. This behavior can be bypassed
|
514
|
+
// by setting `testInvocation=manual` in the workers config.
|
515
|
+
if (EnvInternal_js_1.default.isTest && workersApp.testInvocation === 'automatic') {
|
508
516
|
const queue = new Background.Queue('TestQueue', { connection: {} });
|
509
517
|
const job = new bullmq_1.Job(queue, jobType, jobData, {});
|
510
518
|
await this.doWork(job);
|
package/dist/cjs/src/index.js
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.NoQueueForSpecifiedWorkstream = exports.NoQueueForSpecifiedQueueName = exports.PsychicAppWorkers = exports.BaseScheduledService = exports.BaseBackgroundedService = exports.BaseBackgroundedModel = exports.stopBackgroundWorkers = exports.Background = exports.background = void 0;
|
3
|
+
exports.WorkerTestUtils = exports.NoQueueForSpecifiedWorkstream = exports.NoQueueForSpecifiedQueueName = exports.PsychicAppWorkers = exports.BaseScheduledService = exports.BaseBackgroundedService = exports.BaseBackgroundedModel = exports.stopBackgroundWorkers = exports.Background = exports.background = void 0;
|
4
4
|
var index_js_1 = require("./background/index.js");
|
5
5
|
Object.defineProperty(exports, "background", { enumerable: true, get: function () { return index_js_1.default; } });
|
6
6
|
Object.defineProperty(exports, "Background", { enumerable: true, get: function () { return index_js_1.Background; } });
|
@@ -17,3 +17,5 @@ var NoQueueForSpecifiedQueueName_js_1 = require("./error/background/NoQueueForSp
|
|
17
17
|
Object.defineProperty(exports, "NoQueueForSpecifiedQueueName", { enumerable: true, get: function () { return NoQueueForSpecifiedQueueName_js_1.default; } });
|
18
18
|
var NoQueueForSpecifiedWorkstream_js_1 = require("./error/background/NoQueueForSpecifiedWorkstream.js");
|
19
19
|
Object.defineProperty(exports, "NoQueueForSpecifiedWorkstream", { enumerable: true, get: function () { return NoQueueForSpecifiedWorkstream_js_1.default; } });
|
20
|
+
var WorkerTestUtils_js_1 = require("./test-utils/WorkerTestUtils.js");
|
21
|
+
Object.defineProperty(exports, "WorkerTestUtils", { enumerable: true, get: function () { return WorkerTestUtils_js_1.default; } });
|
@@ -7,7 +7,7 @@ class PsychicAppWorkers {
|
|
7
7
|
static async init(psychicApp, cb) {
|
8
8
|
const psychicWorkersApp = new PsychicAppWorkers(psychicApp);
|
9
9
|
await cb(psychicWorkersApp);
|
10
|
-
psychicApp.on('sync', () => {
|
10
|
+
psychicApp.on('cli:sync', () => {
|
11
11
|
index_js_1.default.connect();
|
12
12
|
const output = {
|
13
13
|
workstreamNames: [...index_js_1.default['workstreamNames']],
|
@@ -18,6 +18,9 @@ class PsychicAppWorkers {
|
|
18
18
|
psychicApp.on('server:shutdown', async () => {
|
19
19
|
await index_js_1.default.closeAllRedisConnections();
|
20
20
|
});
|
21
|
+
psychicApp.on('server:init:after-routes', () => {
|
22
|
+
index_js_1.default.connect();
|
23
|
+
});
|
21
24
|
(0, cache_js_1.cachePsychicWorkersApp)(psychicWorkersApp);
|
22
25
|
return psychicWorkersApp;
|
23
26
|
}
|
@@ -41,6 +44,20 @@ class PsychicAppWorkers {
|
|
41
44
|
return this._backgroundOptions;
|
42
45
|
}
|
43
46
|
_backgroundOptions;
|
47
|
+
/**
|
48
|
+
* Returns the testInvocation option provided by the user
|
49
|
+
*
|
50
|
+
* when "automatic", any backgrounded job will be immediately
|
51
|
+
* invoked during tests. This is the default behavior
|
52
|
+
*
|
53
|
+
* when "manual", this will enable the dev to manually interact with
|
54
|
+
* queues, enabling them to target jobs and run them at specific
|
55
|
+
* code points.
|
56
|
+
*/
|
57
|
+
get testInvocation() {
|
58
|
+
return this._testInvocation;
|
59
|
+
}
|
60
|
+
_testInvocation = 'automatic';
|
44
61
|
_hooks = {
|
45
62
|
workerShutdown: [],
|
46
63
|
};
|
@@ -69,6 +86,9 @@ class PsychicAppWorkers {
|
|
69
86
|
...value,
|
70
87
|
};
|
71
88
|
break;
|
89
|
+
case 'testInvocation':
|
90
|
+
this._testInvocation = value;
|
91
|
+
break;
|
72
92
|
default:
|
73
93
|
throw new Error(`Unhandled option type passed to PsychicWorkersApp#set: ${option}`);
|
74
94
|
}
|
@@ -0,0 +1,124 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
const index_js_1 = require("../background/index.js");
|
4
|
+
const parallelTestSafeQueueName_js_1 = require("../background/helpers/parallelTestSafeQueueName.js");
|
5
|
+
const LOCK_TOKEN = 'psychic-test-worker';
|
6
|
+
class WorkerTestUtils {
|
7
|
+
/*
|
8
|
+
* Safely encapsulate's a queue name in parallel test runs,
|
9
|
+
* capturing the VITEST_POOL_ID and appending it to the
|
10
|
+
* end of the queue name if VITEST_POOL_ID > 1
|
11
|
+
*/
|
12
|
+
static parallelTestSafeQueueName(queueName) {
|
13
|
+
return (0, parallelTestSafeQueueName_js_1.default)(queueName);
|
14
|
+
}
|
15
|
+
/*
|
16
|
+
* Works off all of the jobs in all queues. The jobs
|
17
|
+
* are worked off in a round-robin fashion until every queue
|
18
|
+
* is empty, at which point the promise will resolve
|
19
|
+
*
|
20
|
+
* ```ts
|
21
|
+
* await MyService.background('someMethod', ...)
|
22
|
+
* await WorkerTestUtils.work()
|
23
|
+
* // now you can safely assert the results of your background job
|
24
|
+
* ```
|
25
|
+
*
|
26
|
+
* NOTE: this is only useful if you are running with `testInvocation=manual`.
|
27
|
+
* make sure to set this in your workers config, or else in your test,
|
28
|
+
* so that jobs are successfully commited to the queue and can be worked off.
|
29
|
+
*/
|
30
|
+
static async work(opts = {}) {
|
31
|
+
index_js_1.default.connect();
|
32
|
+
const queues = index_js_1.default.queues;
|
33
|
+
let workWasDone = true;
|
34
|
+
do {
|
35
|
+
workWasDone = false;
|
36
|
+
for (const queue of queues) {
|
37
|
+
if (opts.queue && !this.queueNamesMatch(queue, opts.queue))
|
38
|
+
continue;
|
39
|
+
workWasDone ||= await this.workOne(queue);
|
40
|
+
}
|
41
|
+
} while (workWasDone);
|
42
|
+
}
|
43
|
+
static async workScheduled(opts = {}) {
|
44
|
+
index_js_1.default.connect();
|
45
|
+
const queues = opts.queue
|
46
|
+
? index_js_1.default.queues.filter(queue => queue.name === opts.queue)
|
47
|
+
: index_js_1.default.queues;
|
48
|
+
if (opts.queue && !queues.length)
|
49
|
+
throw new Error(`Expected to find queue with name: ${opts.queue}, but none were found by that name. The queue names available are: ${index_js_1.default.queues.map(queue => queue.name).join(', ')}`);
|
50
|
+
for (const queue of queues) {
|
51
|
+
const jobs = (await queue.getDelayed());
|
52
|
+
for (const job of jobs) {
|
53
|
+
const data = job.data;
|
54
|
+
if (opts.for) {
|
55
|
+
if (data.globalName === opts.for.globalName) {
|
56
|
+
await index_js_1.default.doWork(job);
|
57
|
+
}
|
58
|
+
}
|
59
|
+
else {
|
60
|
+
await index_js_1.default.doWork(job);
|
61
|
+
}
|
62
|
+
}
|
63
|
+
}
|
64
|
+
}
|
65
|
+
/*
|
66
|
+
* iterates through each registered queue, and cleans out all
|
67
|
+
* jobs, including completed, failed, and scheduled jobs. This
|
68
|
+
* is especially useful before a test where you plan to exercise
|
69
|
+
* background jobs manually.
|
70
|
+
*
|
71
|
+
* If your entire app is continuously exercising background jobs
|
72
|
+
* manually, you may want to do this in your spec/setup/hooks.ts file,
|
73
|
+
* so that it can be called before every test.
|
74
|
+
*
|
75
|
+
* ```ts
|
76
|
+
* beforeEach(async () => {
|
77
|
+
* await WorkerTestUtils.clean()
|
78
|
+
* })
|
79
|
+
* ```
|
80
|
+
*/
|
81
|
+
static async clean() {
|
82
|
+
index_js_1.default.connect();
|
83
|
+
for (const queue of index_js_1.default.queues) {
|
84
|
+
// clears all non-scheduled, non-completed, and non-failed jobs
|
85
|
+
await queue.drain();
|
86
|
+
// clear out completed and failed jobs
|
87
|
+
await queue.clean(0, 10000, 'completed');
|
88
|
+
await queue.clean(0, 10000, 'failed');
|
89
|
+
// clear out scheduled jobs
|
90
|
+
const schedulers = await queue.getJobSchedulers();
|
91
|
+
for (const scheduler of schedulers) {
|
92
|
+
await queue.removeJobScheduler(scheduler.key);
|
93
|
+
}
|
94
|
+
}
|
95
|
+
}
|
96
|
+
static async workOne(queue) {
|
97
|
+
const worker = new index_js_1.Background.Worker(queue.name, async (job) => await index_js_1.default.doWork(job), {
|
98
|
+
autorun: false,
|
99
|
+
connection: queue.client,
|
100
|
+
concurrency: 1,
|
101
|
+
});
|
102
|
+
if (!worker)
|
103
|
+
throw new Error(`Failed to find worker for queue: ${queue.name}`);
|
104
|
+
const job = await worker.getNextJob(LOCK_TOKEN);
|
105
|
+
if (!job)
|
106
|
+
return false;
|
107
|
+
await this.processJob(job);
|
108
|
+
return true;
|
109
|
+
}
|
110
|
+
static queueNamesMatch(queue, compareQueueName) {
|
111
|
+
return (queue.name === (0, parallelTestSafeQueueName_js_1.default)(compareQueueName) ||
|
112
|
+
queue.name === `{${(0, parallelTestSafeQueueName_js_1.default)(compareQueueName)}`);
|
113
|
+
}
|
114
|
+
static async processJob(job) {
|
115
|
+
try {
|
116
|
+
const res = await index_js_1.default.doWork(job);
|
117
|
+
await job.moveToCompleted(res, LOCK_TOKEN, false);
|
118
|
+
}
|
119
|
+
catch (err) {
|
120
|
+
await job.moveToFailed(err, LOCK_TOKEN, false);
|
121
|
+
}
|
122
|
+
}
|
123
|
+
}
|
124
|
+
exports.default = WorkerTestUtils;
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import { GlobalNameNotSet } from '@rvoh/dream';
|
2
|
-
import background from './index.js';
|
3
2
|
import durationToSeconds from '../helpers/durationToSeconds.js';
|
3
|
+
import background from './index.js';
|
4
4
|
export default class BaseBackgroundedService {
|
5
5
|
/**
|
6
6
|
* A getter meant to be overridden in child classes. This does
|
@@ -1,7 +1,8 @@
|
|
1
1
|
import { Cluster } from 'ioredis';
|
2
|
+
import parallelTestSafeQueueName from './parallelTestSafeQueueName.js';
|
2
3
|
export default function nameToRedisQueueName(queueName, redis) {
|
3
4
|
queueName = queueName.replace(/\{|\}/g, '');
|
4
5
|
if (redis instanceof Cluster)
|
5
6
|
return `{${queueName}}`;
|
6
|
-
return queueName;
|
7
|
+
return parallelTestSafeQueueName(queueName);
|
7
8
|
}
|
@@ -0,0 +1,7 @@
|
|
1
|
+
import EnvInternal from '../../helpers/EnvInternal.js';
|
2
|
+
export default function parallelTestSafeQueueName(queueName) {
|
3
|
+
if (EnvInternal.isTest && (EnvInternal.integer('VITEST_POOL_ID', { optional: true }) || 0) > 1) {
|
4
|
+
queueName = `${queueName}-${EnvInternal.integer('VITEST_POOL_ID')}`;
|
5
|
+
}
|
6
|
+
return queueName;
|
7
|
+
}
|
@@ -203,6 +203,7 @@ export class Background {
|
|
203
203
|
const workerCount = backgroundOptions.defaultWorkstream?.workerCount ?? 1;
|
204
204
|
for (let i = 0; i < workerCount; i++) {
|
205
205
|
this._workers.push(new Background.Worker(formattedQueueName, async (job) => await this.doWork(job), {
|
206
|
+
autorun: !EnvInternal.isTest,
|
206
207
|
connection: defaultWorkerConnection,
|
207
208
|
concurrency: backgroundOptions.defaultWorkstream?.concurrency || DEFAULT_CONCURRENCY,
|
208
209
|
}));
|
@@ -246,6 +247,7 @@ export class Background {
|
|
246
247
|
const workerCount = namedWorkstream.workerCount ?? 1;
|
247
248
|
for (let i = 0; i < workerCount; i++) {
|
248
249
|
this._workers.push(new Background.Worker(namedWorkstreamFormattedQueueName, async (job) => await this.doWork(job), {
|
250
|
+
autorun: !EnvInternal.isTest,
|
249
251
|
group: {
|
250
252
|
id: namedWorkstream.name,
|
251
253
|
limit: namedWorkstream.rateLimit,
|
@@ -308,6 +310,7 @@ export class Background {
|
|
308
310
|
const workerCount = nativeBullMQ.defaultWorkerCount ?? 1;
|
309
311
|
for (let i = 0; i < workerCount; i++) {
|
310
312
|
this._workers.push(new Background.Worker(formattedQueueName, async (job) => await this.doWork(job), {
|
313
|
+
autorun: !EnvInternal.isTest,
|
311
314
|
...(backgroundOptions.nativeBullMQ.defaultWorkerOptions || {}),
|
312
315
|
connection: defaultWorkerConnection,
|
313
316
|
}));
|
@@ -350,6 +353,7 @@ export class Background {
|
|
350
353
|
throw new ActivatingNamedQueueBackgroundWorkersWithoutWorkerConnection(queueName);
|
351
354
|
for (let i = 0; i < extraWorkerCount; i++) {
|
352
355
|
this._workers.push(new Background.Worker(formattedQueuename, async (job) => await this.doWork(job), {
|
356
|
+
autorun: !EnvInternal.isTest,
|
353
357
|
...extraWorkerOptions,
|
354
358
|
connection: namedWorkerConnection,
|
355
359
|
}));
|
@@ -500,7 +504,11 @@ export class Background {
|
|
500
504
|
// if delaySeconds is 0, we will intentionally treat
|
501
505
|
// this as `undefined`
|
502
506
|
const delay = delaySeconds ? delaySeconds * 1000 : undefined;
|
503
|
-
|
507
|
+
const workersApp = PsychicAppWorkers.getOrFail();
|
508
|
+
// in test environments, this block will short-circuit adding to the queue,
|
509
|
+
// causing the job to immediately invoke instead. This behavior can be bypassed
|
510
|
+
// by setting `testInvocation=manual` in the workers config.
|
511
|
+
if (EnvInternal.isTest && workersApp.testInvocation === 'automatic') {
|
504
512
|
const queue = new Background.Queue('TestQueue', { connection: {} });
|
505
513
|
const job = new Job(queue, jobType, jobData, {});
|
506
514
|
await this.doWork(job);
|
package/dist/esm/src/index.js
CHANGED
@@ -5,3 +5,4 @@ export { default as BaseScheduledService } from './background/BaseScheduledServi
|
|
5
5
|
export { default as PsychicAppWorkers, } from './psychic-app-workers/index.js';
|
6
6
|
export { default as NoQueueForSpecifiedQueueName } from './error/background/NoQueueForSpecifiedQueueName.js';
|
7
7
|
export { default as NoQueueForSpecifiedWorkstream } from './error/background/NoQueueForSpecifiedWorkstream.js';
|
8
|
+
export { default as WorkerTestUtils } from './test-utils/WorkerTestUtils.js';
|
@@ -5,7 +5,7 @@ export default class PsychicAppWorkers {
|
|
5
5
|
static async init(psychicApp, cb) {
|
6
6
|
const psychicWorkersApp = new PsychicAppWorkers(psychicApp);
|
7
7
|
await cb(psychicWorkersApp);
|
8
|
-
psychicApp.on('sync', () => {
|
8
|
+
psychicApp.on('cli:sync', () => {
|
9
9
|
background.connect();
|
10
10
|
const output = {
|
11
11
|
workstreamNames: [...background['workstreamNames']],
|
@@ -16,6 +16,9 @@ export default class PsychicAppWorkers {
|
|
16
16
|
psychicApp.on('server:shutdown', async () => {
|
17
17
|
await background.closeAllRedisConnections();
|
18
18
|
});
|
19
|
+
psychicApp.on('server:init:after-routes', () => {
|
20
|
+
background.connect();
|
21
|
+
});
|
19
22
|
cachePsychicWorkersApp(psychicWorkersApp);
|
20
23
|
return psychicWorkersApp;
|
21
24
|
}
|
@@ -39,6 +42,20 @@ export default class PsychicAppWorkers {
|
|
39
42
|
return this._backgroundOptions;
|
40
43
|
}
|
41
44
|
_backgroundOptions;
|
45
|
+
/**
|
46
|
+
* Returns the testInvocation option provided by the user
|
47
|
+
*
|
48
|
+
* when "automatic", any backgrounded job will be immediately
|
49
|
+
* invoked during tests. This is the default behavior
|
50
|
+
*
|
51
|
+
* when "manual", this will enable the dev to manually interact with
|
52
|
+
* queues, enabling them to target jobs and run them at specific
|
53
|
+
* code points.
|
54
|
+
*/
|
55
|
+
get testInvocation() {
|
56
|
+
return this._testInvocation;
|
57
|
+
}
|
58
|
+
_testInvocation = 'automatic';
|
42
59
|
_hooks = {
|
43
60
|
workerShutdown: [],
|
44
61
|
};
|
@@ -67,6 +84,9 @@ export default class PsychicAppWorkers {
|
|
67
84
|
...value,
|
68
85
|
};
|
69
86
|
break;
|
87
|
+
case 'testInvocation':
|
88
|
+
this._testInvocation = value;
|
89
|
+
break;
|
70
90
|
default:
|
71
91
|
throw new Error(`Unhandled option type passed to PsychicWorkersApp#set: ${option}`);
|
72
92
|
}
|
@@ -0,0 +1,121 @@
|
|
1
|
+
import background, { Background } from '../background/index.js';
|
2
|
+
import parallelTestSafeQueueName from '../background/helpers/parallelTestSafeQueueName.js';
|
3
|
+
const LOCK_TOKEN = 'psychic-test-worker';
|
4
|
+
export default class WorkerTestUtils {
|
5
|
+
/*
|
6
|
+
* Safely encapsulate's a queue name in parallel test runs,
|
7
|
+
* capturing the VITEST_POOL_ID and appending it to the
|
8
|
+
* end of the queue name if VITEST_POOL_ID > 1
|
9
|
+
*/
|
10
|
+
static parallelTestSafeQueueName(queueName) {
|
11
|
+
return parallelTestSafeQueueName(queueName);
|
12
|
+
}
|
13
|
+
/*
|
14
|
+
* Works off all of the jobs in all queues. The jobs
|
15
|
+
* are worked off in a round-robin fashion until every queue
|
16
|
+
* is empty, at which point the promise will resolve
|
17
|
+
*
|
18
|
+
* ```ts
|
19
|
+
* await MyService.background('someMethod', ...)
|
20
|
+
* await WorkerTestUtils.work()
|
21
|
+
* // now you can safely assert the results of your background job
|
22
|
+
* ```
|
23
|
+
*
|
24
|
+
* NOTE: this is only useful if you are running with `testInvocation=manual`.
|
25
|
+
* make sure to set this in your workers config, or else in your test,
|
26
|
+
* so that jobs are successfully commited to the queue and can be worked off.
|
27
|
+
*/
|
28
|
+
static async work(opts = {}) {
|
29
|
+
background.connect();
|
30
|
+
const queues = background.queues;
|
31
|
+
let workWasDone = true;
|
32
|
+
do {
|
33
|
+
workWasDone = false;
|
34
|
+
for (const queue of queues) {
|
35
|
+
if (opts.queue && !this.queueNamesMatch(queue, opts.queue))
|
36
|
+
continue;
|
37
|
+
workWasDone ||= await this.workOne(queue);
|
38
|
+
}
|
39
|
+
} while (workWasDone);
|
40
|
+
}
|
41
|
+
static async workScheduled(opts = {}) {
|
42
|
+
background.connect();
|
43
|
+
const queues = opts.queue
|
44
|
+
? background.queues.filter(queue => queue.name === opts.queue)
|
45
|
+
: background.queues;
|
46
|
+
if (opts.queue && !queues.length)
|
47
|
+
throw new Error(`Expected to find queue with name: ${opts.queue}, but none were found by that name. The queue names available are: ${background.queues.map(queue => queue.name).join(', ')}`);
|
48
|
+
for (const queue of queues) {
|
49
|
+
const jobs = (await queue.getDelayed());
|
50
|
+
for (const job of jobs) {
|
51
|
+
const data = job.data;
|
52
|
+
if (opts.for) {
|
53
|
+
if (data.globalName === opts.for.globalName) {
|
54
|
+
await background.doWork(job);
|
55
|
+
}
|
56
|
+
}
|
57
|
+
else {
|
58
|
+
await background.doWork(job);
|
59
|
+
}
|
60
|
+
}
|
61
|
+
}
|
62
|
+
}
|
63
|
+
/*
|
64
|
+
* iterates through each registered queue, and cleans out all
|
65
|
+
* jobs, including completed, failed, and scheduled jobs. This
|
66
|
+
* is especially useful before a test where you plan to exercise
|
67
|
+
* background jobs manually.
|
68
|
+
*
|
69
|
+
* If your entire app is continuously exercising background jobs
|
70
|
+
* manually, you may want to do this in your spec/setup/hooks.ts file,
|
71
|
+
* so that it can be called before every test.
|
72
|
+
*
|
73
|
+
* ```ts
|
74
|
+
* beforeEach(async () => {
|
75
|
+
* await WorkerTestUtils.clean()
|
76
|
+
* })
|
77
|
+
* ```
|
78
|
+
*/
|
79
|
+
static async clean() {
|
80
|
+
background.connect();
|
81
|
+
for (const queue of background.queues) {
|
82
|
+
// clears all non-scheduled, non-completed, and non-failed jobs
|
83
|
+
await queue.drain();
|
84
|
+
// clear out completed and failed jobs
|
85
|
+
await queue.clean(0, 10000, 'completed');
|
86
|
+
await queue.clean(0, 10000, 'failed');
|
87
|
+
// clear out scheduled jobs
|
88
|
+
const schedulers = await queue.getJobSchedulers();
|
89
|
+
for (const scheduler of schedulers) {
|
90
|
+
await queue.removeJobScheduler(scheduler.key);
|
91
|
+
}
|
92
|
+
}
|
93
|
+
}
|
94
|
+
static async workOne(queue) {
|
95
|
+
const worker = new Background.Worker(queue.name, async (job) => await background.doWork(job), {
|
96
|
+
autorun: false,
|
97
|
+
connection: queue.client,
|
98
|
+
concurrency: 1,
|
99
|
+
});
|
100
|
+
if (!worker)
|
101
|
+
throw new Error(`Failed to find worker for queue: ${queue.name}`);
|
102
|
+
const job = await worker.getNextJob(LOCK_TOKEN);
|
103
|
+
if (!job)
|
104
|
+
return false;
|
105
|
+
await this.processJob(job);
|
106
|
+
return true;
|
107
|
+
}
|
108
|
+
static queueNamesMatch(queue, compareQueueName) {
|
109
|
+
return (queue.name === parallelTestSafeQueueName(compareQueueName) ||
|
110
|
+
queue.name === `{${parallelTestSafeQueueName(compareQueueName)}`);
|
111
|
+
}
|
112
|
+
static async processJob(job) {
|
113
|
+
try {
|
114
|
+
const res = await background.doWork(job);
|
115
|
+
await job.moveToCompleted(res, LOCK_TOKEN, false);
|
116
|
+
}
|
117
|
+
catch (err) {
|
118
|
+
await job.moveToFailed(err, LOCK_TOKEN, false);
|
119
|
+
}
|
120
|
+
}
|
121
|
+
}
|
@@ -1,2 +1,2 @@
|
|
1
|
-
import {
|
1
|
+
import { Cluster, Redis } from 'ioredis';
|
2
2
|
export default function nameToRedisQueueName(queueName: string, redis: Redis | Cluster): string;
|
@@ -0,0 +1 @@
|
|
1
|
+
export default function parallelTestSafeQueueName(queueName: string): string;
|
@@ -1,6 +1,7 @@
|
|
1
1
|
import { Env } from '@rvoh/dream';
|
2
2
|
declare const EnvInternal: Env<{
|
3
3
|
string: "NODE_ENV" | "PSYCHIC_CORE_DEVELOPMENT";
|
4
|
+
integer: "VITEST_POOL_ID";
|
4
5
|
boolean: "REALLY_TEST_BACKGROUND_QUEUE";
|
5
|
-
}, "NODE_ENV" | "PSYCHIC_CORE_DEVELOPMENT",
|
6
|
+
}, "NODE_ENV" | "PSYCHIC_CORE_DEVELOPMENT", "VITEST_POOL_ID", "REALLY_TEST_BACKGROUND_QUEUE">;
|
6
7
|
export default EnvInternal;
|
@@ -6,3 +6,4 @@ export { default as BaseScheduledService } from './background/BaseScheduledServi
|
|
6
6
|
export { default as PsychicAppWorkers, type BullMQNativeWorkerOptions, type PsychicBackgroundNativeBullMQOptions, type PsychicBackgroundSimpleOptions, type PsychicBackgroundWorkstreamOptions, type QueueOptionsWithConnectionInstance, type RedisOrRedisClusterConnection, type TransitionalPsychicBackgroundSimpleOptions, } from './psychic-app-workers/index.js';
|
7
7
|
export { default as NoQueueForSpecifiedQueueName } from './error/background/NoQueueForSpecifiedQueueName.js';
|
8
8
|
export { default as NoQueueForSpecifiedWorkstream } from './error/background/NoQueueForSpecifiedWorkstream.js';
|
9
|
+
export { default as WorkerTestUtils } from './test-utils/WorkerTestUtils.js';
|
@@ -18,16 +18,29 @@ export default class PsychicAppWorkers {
|
|
18
18
|
*/
|
19
19
|
get backgroundOptions(): PsychicBackgroundOptions;
|
20
20
|
private _backgroundOptions;
|
21
|
+
/**
|
22
|
+
* Returns the testInvocation option provided by the user
|
23
|
+
*
|
24
|
+
* when "automatic", any backgrounded job will be immediately
|
25
|
+
* invoked during tests. This is the default behavior
|
26
|
+
*
|
27
|
+
* when "manual", this will enable the dev to manually interact with
|
28
|
+
* queues, enabling them to target jobs and run them at specific
|
29
|
+
* code points.
|
30
|
+
*/
|
31
|
+
get testInvocation(): PsychicWorkersAppTestInvocationType;
|
32
|
+
private _testInvocation;
|
21
33
|
private _hooks;
|
22
34
|
get hooks(): PsychicWorkersAppHooks;
|
23
35
|
on<T extends PsychicWorkersHookEventType>(hookEventType: T, cb: T extends 'workers:shutdown' ? () => void | Promise<void> : never): void;
|
24
|
-
set<Opt extends PsychicWorkersAppOption>(option: Opt, value: Opt extends 'background' ? PsychicBackgroundOptions : unknown): void;
|
36
|
+
set<Opt extends PsychicWorkersAppOption>(option: Opt, value: Opt extends 'background' ? PsychicBackgroundOptions : Opt extends 'testInvocation' ? PsychicWorkersAppTestInvocationType : unknown): void;
|
25
37
|
}
|
26
38
|
export interface PsychicWorkersTypeSync {
|
27
39
|
workstreamNames: string[];
|
28
40
|
queueGroupMap: Record<string, string[]>;
|
29
41
|
}
|
30
|
-
export type PsychicWorkersAppOption = 'background';
|
42
|
+
export type PsychicWorkersAppOption = 'background' | 'testInvocation';
|
43
|
+
export type PsychicWorkersAppTestInvocationType = 'automatic' | 'manual';
|
31
44
|
export type PsychicWorkersHookEventType = 'workers:shutdown';
|
32
45
|
export interface PsychicWorkersAppHooks {
|
33
46
|
workerShutdown: (() => void | Promise<void>)[];
|
@@ -0,0 +1,19 @@
|
|
1
|
+
export default class WorkerTestUtils {
|
2
|
+
static parallelTestSafeQueueName(queueName: string): string;
|
3
|
+
static work(opts?: TestWorkerWorkOffOpts): Promise<void>;
|
4
|
+
static workScheduled(opts?: TestWorkerScheduledWorkOffOpts): Promise<void>;
|
5
|
+
static clean(): Promise<void>;
|
6
|
+
private static workOne;
|
7
|
+
private static queueNamesMatch;
|
8
|
+
private static processJob;
|
9
|
+
}
|
10
|
+
interface TestWorkerWorkOffOpts {
|
11
|
+
queue?: string;
|
12
|
+
}
|
13
|
+
interface TestWorkerScheduledWorkOffOpts {
|
14
|
+
queue?: string;
|
15
|
+
for?: {
|
16
|
+
globalName: string;
|
17
|
+
};
|
18
|
+
}
|
19
|
+
export {};
|
@@ -52,7 +52,7 @@ interface BaseBackgroundJobConfig {
|
|
52
52
|
export interface WorkstreamBackgroundJobConfig<T extends BaseScheduledService | BaseBackgroundedService> extends BaseBackgroundJobConfig {
|
53
53
|
workstream?: T['psychicTypes']['workstreamNames'][number];
|
54
54
|
}
|
55
|
-
export interface QueueBackgroundJobConfig<T extends BaseScheduledService | BaseBackgroundedService, PsyTypes extends T['psychicTypes'] = T['psychicTypes'], QueueGroupMap = PsyTypes['queueGroupMap'], Queue extends keyof QueueGroupMap = keyof QueueGroupMap, Groups extends QueueGroupMap[Queue] = QueueGroupMap[Queue], GroupId = Groups[number & keyof Groups]> extends BaseBackgroundJobConfig {
|
55
|
+
export interface QueueBackgroundJobConfig<T extends BaseScheduledService | BaseBackgroundedService, PsyTypes extends T['psychicTypes'] = T['psychicTypes'], QueueGroupMap = PsyTypes['queueGroupMap'], Queue extends keyof QueueGroupMap & string = keyof QueueGroupMap & string, Groups extends QueueGroupMap[Queue] = QueueGroupMap[Queue], GroupId = Groups[number & keyof Groups]> extends BaseBackgroundJobConfig {
|
56
56
|
groupId?: GroupId;
|
57
57
|
queue?: Queue;
|
58
58
|
}
|
package/package.json
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
"type": "module",
|
3
3
|
"name": "@rvoh/psychic-workers",
|
4
4
|
"description": "Background job system for Psychic applications",
|
5
|
-
"version": "0.
|
5
|
+
"version": "1.0.0",
|
6
6
|
"author": "RVO Health",
|
7
7
|
"repository": {
|
8
8
|
"type": "git",
|
@@ -45,7 +45,7 @@
|
|
45
45
|
"@eslint/js": "=9.0.0",
|
46
46
|
"@rvoh/dream": "^0.39.0",
|
47
47
|
"@rvoh/dream-spec-helpers": "^0.2.4",
|
48
|
-
"@rvoh/psychic": "^0.
|
48
|
+
"@rvoh/psychic": "^0.35.1",
|
49
49
|
"@rvoh/psychic-spec-helpers": "^0.6.0",
|
50
50
|
"@socket.io/redis-adapter": "^8.3.0",
|
51
51
|
"@socket.io/redis-emitter": "^5.1.0",
|
@@ -70,7 +70,7 @@
|
|
70
70
|
"typedoc": "^0.26.6",
|
71
71
|
"typescript": "^5.8.2",
|
72
72
|
"typescript-eslint": "=7.18.0",
|
73
|
-
"vitest": "^3.1.
|
73
|
+
"vitest": "^3.1.3"
|
74
74
|
},
|
75
75
|
"packageManager": "yarn@4.7.0"
|
76
76
|
}
|