@venizia/ignis-helpers 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,12 +1,59 @@
|
|
|
1
1
|
import { BaseHelper } from '../../../helpers/base';
|
|
2
|
+
import { DefaultRedisHelper } from '../../../helpers/redis';
|
|
2
3
|
import { Job, Queue, Worker } from 'bullmq';
|
|
3
|
-
import Redis from 'ioredis';
|
|
4
4
|
import { TBullQueueRole } from '../common';
|
|
5
|
+
/**
|
|
6
|
+
* BullMQ Helper for queue and worker management.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* // When using Redis Cluster, initialize with recommended options:
|
|
10
|
+
* import { Cluster } from 'ioredis';
|
|
11
|
+
*
|
|
12
|
+
* const cluster = new Cluster(
|
|
13
|
+
* [
|
|
14
|
+
* { host: 'node1.redis.example.com', port: 6379 },
|
|
15
|
+
* { host: 'node2.redis.example.com', port: 6379 },
|
|
16
|
+
* { host: 'node3.redis.example.com', port: 6379 },
|
|
17
|
+
* ],
|
|
18
|
+
* {
|
|
19
|
+
* // Recommended options for BullMQ:
|
|
20
|
+
* maxRetriesPerRequest: null, // Required by BullMQ (disables retry limit)
|
|
21
|
+
* enableReadyCheck: true, // Wait until cluster is ready
|
|
22
|
+
* scaleReads: 'slave', // Optional: read from replicas to reduce master load
|
|
23
|
+
*
|
|
24
|
+
* // If behind NAT/proxy:
|
|
25
|
+
* // natMap: {
|
|
26
|
+
* // 'internal-ip:6379': { host: 'external-ip', port: 6379 }
|
|
27
|
+
* // },
|
|
28
|
+
*
|
|
29
|
+
* redisOptions: {
|
|
30
|
+
* password: 'your-password', // If auth required
|
|
31
|
+
* tls: {}, // If TLS required
|
|
32
|
+
* },
|
|
33
|
+
* }
|
|
34
|
+
* );
|
|
35
|
+
*
|
|
36
|
+
* const redisHelper = new DefaultRedisHelper({
|
|
37
|
+
* scope: 'BullMQ',
|
|
38
|
+
* identifier: 'my-redis',
|
|
39
|
+
* client: cluster,
|
|
40
|
+
* });
|
|
41
|
+
*
|
|
42
|
+
* const helper = BullMQHelper.newInstance({
|
|
43
|
+
* queueName: 'my-queue',
|
|
44
|
+
* identifier: 'my-worker',
|
|
45
|
+
* role: 'worker',
|
|
46
|
+
* redisConnection: redisHelper,
|
|
47
|
+
* onWorkerData: async (job) => { ... },
|
|
48
|
+
* });
|
|
49
|
+
*
|
|
50
|
+
* @note `maxRetriesPerRequest: null` is required by BullMQ for both Redis and Cluster connections to prevent blocking issues.
|
|
51
|
+
*/
|
|
5
52
|
interface IBullMQOptions<TQueueElement = any, TQueueResult = any> {
|
|
6
53
|
queueName: string;
|
|
7
54
|
identifier: string;
|
|
8
55
|
role: TBullQueueRole;
|
|
9
|
-
|
|
56
|
+
redisConnection: DefaultRedisHelper;
|
|
10
57
|
numberOfWorker?: number;
|
|
11
58
|
lockDuration?: number;
|
|
12
59
|
onWorkerData?: (job: Job<TQueueElement, TQueueResult>) => Promise<any>;
|
|
@@ -14,21 +61,22 @@ interface IBullMQOptions<TQueueElement = any, TQueueResult = any> {
|
|
|
14
61
|
onWorkerDataFail?: (job: Job<TQueueElement, TQueueResult> | undefined, error: Error) => Promise<void>;
|
|
15
62
|
}
|
|
16
63
|
export declare class BullMQHelper<TQueueElement = any, TQueueResult = any> extends BaseHelper {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
64
|
+
protected queueName: string;
|
|
65
|
+
protected role: TBullQueueRole;
|
|
66
|
+
protected redisConnection: DefaultRedisHelper;
|
|
20
67
|
queue: Queue<TQueueElement, TQueueResult>;
|
|
21
68
|
worker: Worker<TQueueElement, TQueueResult>;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
69
|
+
protected numberOfWorker: number;
|
|
70
|
+
protected lockDuration: number;
|
|
71
|
+
protected onWorkerData?: (job: Job<TQueueElement, TQueueResult>) => Promise<any>;
|
|
72
|
+
protected onWorkerDataCompleted?: (job: Job<TQueueElement, TQueueResult>, result: any) => Promise<void>;
|
|
73
|
+
protected onWorkerDataFail?: (job: Job<TQueueElement, TQueueResult> | undefined, error: Error) => Promise<void>;
|
|
27
74
|
constructor(options: IBullMQOptions<TQueueElement, TQueueResult>);
|
|
28
75
|
static newInstance<T = any, R = any>(opts: IBullMQOptions<T, R>): BullMQHelper<T, R>;
|
|
29
76
|
configureQueue(): void;
|
|
30
77
|
configureWorker(): void;
|
|
31
78
|
configure(): void;
|
|
79
|
+
close(): Promise<void>;
|
|
32
80
|
}
|
|
33
81
|
export {};
|
|
34
82
|
//# sourceMappingURL=helper.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../../src/helpers/queue/bullmq/helper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,
|
|
1
|
+
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../../../../src/helpers/queue/bullmq/helper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,UAAU,cAAc,CAAC,aAAa,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,cAAc,CAAC;IACrB,eAAe,EAAE,kBAAkB,CAAC;IAEpC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;IACvE,qBAAqB,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9F,gBAAgB,CAAC,EAAE,CACjB,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,SAAS,EACjD,KAAK,EAAE,KAAK,KACT,OAAO,CAAC,IAAI,CAAC,CAAC;CACpB;AAED,qBAAa,YAAY,CAAC,aAAa,GAAG,GAAG,EAAE,YAAY,GAAG,GAAG,CAAE,SAAQ,UAAU;IACnF,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,IAAI,EAAE,cAAc,CAAC;IAC/B,SAAS,CAAC,eAAe,EAAE,kBAAkB,CAAC;IAE9C,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IAC1C,MAAM,EAAE,MAAM,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IAE5C,SAAS,CAAC,cAAc,SAAK;IAC7B,SAAS,CAAC,YAAY,SAAkB;IAExC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,CAAC;IACjF,SAAS,CAAC,qBAAqB,CAAC,EAAE,CAChC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,EACrC,MAAM,EAAE,GAAG,KACR,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,SAAS,CAAC,gBAAgB,CAAC,EAAE,CAC3B,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,SAAS,EACjD,KAAK,EAAE,KAAK,KACT,OAAO,CAAC,IAAI,CAAC,CAAC;gBAEP,OAAO,EAAE,cAAc,CAAC,aAAa,EAAE,YAAY,CAAC;IA2BhE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC;IAI/D,cAAc;IAed,eAAe;IA8Df,SAAS;IAqBH,KAAK;CAUZ"}
|
|
@@ -8,10 +8,10 @@ class BullMQHelper extends base_1.BaseHelper {
|
|
|
8
8
|
super({ scope: BullMQHelper.name, identifier: options.identifier });
|
|
9
9
|
this.numberOfWorker = 1;
|
|
10
10
|
this.lockDuration = 90 * 60 * 1000;
|
|
11
|
-
const { queueName,
|
|
11
|
+
const { queueName, redisConnection, role, numberOfWorker = 1, lockDuration = 90 * 60 * 1000, onWorkerData, onWorkerDataCompleted, onWorkerDataFail, } = options;
|
|
12
12
|
this.queueName = queueName;
|
|
13
13
|
this.role = role;
|
|
14
|
-
this.
|
|
14
|
+
this.redisConnection = redisConnection;
|
|
15
15
|
this.numberOfWorker = numberOfWorker;
|
|
16
16
|
this.lockDuration = lockDuration;
|
|
17
17
|
this.onWorkerData = onWorkerData;
|
|
@@ -28,7 +28,7 @@ class BullMQHelper extends base_1.BaseHelper {
|
|
|
28
28
|
return;
|
|
29
29
|
}
|
|
30
30
|
this.queue = new bullmq_1.Queue(this.queueName, {
|
|
31
|
-
connection: this.
|
|
31
|
+
connection: this.redisConnection.getClient().duplicate(),
|
|
32
32
|
defaultJobOptions: {
|
|
33
33
|
removeOnComplete: true,
|
|
34
34
|
removeOnFail: true,
|
|
@@ -48,7 +48,7 @@ class BullMQHelper extends base_1.BaseHelper {
|
|
|
48
48
|
const { id, name, data } = job;
|
|
49
49
|
this.logger.info('[onWorkerData][%s] queue: %s | id: %s | name: %s | data: %j', this.identifier, this.queueName, id, name, data);
|
|
50
50
|
}, {
|
|
51
|
-
connection: this.
|
|
51
|
+
connection: this.redisConnection.getClient().duplicate(),
|
|
52
52
|
concurrency: this.numberOfWorker,
|
|
53
53
|
lockDuration: this.lockDuration,
|
|
54
54
|
});
|
|
@@ -87,6 +87,17 @@ class BullMQHelper extends base_1.BaseHelper {
|
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
|
+
async close() {
|
|
91
|
+
try {
|
|
92
|
+
await this.worker?.close();
|
|
93
|
+
await this.queue?.close();
|
|
94
|
+
this.logger.info('[close][%s] BullMQ helper closed successfully', this.identifier);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
this.logger.error('[close][%s] Error closing BullMQ helper: %s', this.identifier, error);
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
90
101
|
}
|
|
91
102
|
exports.BullMQHelper = BullMQHelper;
|
|
92
103
|
//# sourceMappingURL=helper.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helper.js","sourceRoot":"","sources":["../../../../src/helpers/queue/bullmq/helper.ts"],"names":[],"mappings":";;;AAAA,yCAA4C;
|
|
1
|
+
{"version":3,"file":"helper.js","sourceRoot":"","sources":["../../../../src/helpers/queue/bullmq/helper.ts"],"names":[],"mappings":";;;AAAA,yCAA4C;AAE5C,mCAA4C;AAmE5C,MAAa,YAAsD,SAAQ,iBAAU;IAqBnF,YAAY,OAAoD;QAC9D,KAAK,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;QAd5D,mBAAc,GAAG,CAAC,CAAC;QACnB,iBAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QActC,MAAM,EACJ,SAAS,EACT,eAAe,EACf,IAAI,EACJ,cAAc,GAAG,CAAC,EAClB,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAC7B,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,GACjB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QAEjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAEzC,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,WAAW,CAAmB,IAA0B;QAC7D,OAAO,IAAI,YAAY,CAAO,IAAI,CAAC,CAAC;IACtC,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,cAAK,CAA8B,IAAI,CAAC,SAAS,EAAE;YAClE,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE;YACxD,iBAAiB,EAAE;gBACjB,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,IAAI;aACnB;SACF,CAAC,CAAC;IACL,CAAC;IAED,eAAe;QACb,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACjF,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,eAAM,CACtB,IAAI,CAAC,SAAS,EACd,KAAK,EAAC,GAAG,EAAC,EAAE;YACV,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;gBACxC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,6DAA6D,EAC7D,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,EAAE,EACF,IAAI,EACJ,IAAI,CACL,CAAC;QACJ,CAAC,EACD;YACE,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE;YACxD,WAAW,EAAE,IAAI,CAAC,cAAc;YAChC,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CACF,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;YAC1C,IAAI,CAAC,qBAAqB,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC;iBACtC,IAAI,CAAC,GAAG,EAAE;gBACT,8CAA8C;YAChD,CAAC,CAAC;iBACD,KAAK,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qFAAqF,EACrF,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,KAAK,CACN,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;YACvC,IAAI,CAAC,gBAAgB,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC;iBACjC,IAAI,CAAC,GAAG,EAAE;gBACT,2CAA2C;YAC7C,CAAC,CAAC;iBACD,KAAK,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,kFAAkF,EAClF,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,SAAS,EACd,KAAK,CACN,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,gFAAgF,EAChF,IAAI,CAAC,UAAU,CAChB,CAAC;YACF,OAAO;QACT,CAAC;QAED,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACrF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACzF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF;AAhKD,oCAgKC"}
|