@motiadev/adapter-bullmq-events 0.13.0-beta.162-717198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +93 -0
- package/README.md +222 -0
- package/dist/bullmq-event-adapter.d.ts +27 -0
- package/dist/bullmq-event-adapter.d.ts.map +1 -0
- package/dist/bullmq-event-adapter.js +75 -0
- package/dist/config-builder.d.ts +6 -0
- package/dist/config-builder.d.ts.map +1 -0
- package/dist/config-builder.js +29 -0
- package/dist/connection-manager.d.ts +10 -0
- package/dist/connection-manager.d.ts.map +1 -0
- package/dist/connection-manager.js +39 -0
- package/dist/constants.d.ts +14 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +16 -0
- package/dist/dlq-manager.d.ts +22 -0
- package/dist/dlq-manager.d.ts.map +1 -0
- package/dist/dlq-manager.js +112 -0
- package/dist/errors.d.ts +14 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +35 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/queue-manager.d.ts +20 -0
- package/dist/queue-manager.d.ts.map +1 -0
- package/dist/queue-manager.js +85 -0
- package/dist/types.d.ts +23 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/worker-manager.d.ts +38 -0
- package/dist/worker-manager.d.ts.map +1 -0
- package/dist/worker-manager.js +136 -0
- package/package.json +25 -0
- package/src/bullmq-event-adapter.ts +105 -0
- package/src/config-builder.ts +41 -0
- package/src/connection-manager.ts +40 -0
- package/src/constants.ts +14 -0
- package/src/dlq-manager.ts +151 -0
- package/src/errors.ts +33 -0
- package/src/index.ts +6 -0
- package/src/queue-manager.ts +107 -0
- package/src/types.ts +24 -0
- package/src/worker-manager.ts +200 -0
- package/tsconfig.json +19 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DLQManager = void 0;
|
|
4
|
+
const bullmq_1 = require("bullmq");
|
|
5
|
+
const constants_1 = require("./constants");
|
|
6
|
+
const errors_1 = require("./errors");
|
|
7
|
+
class DLQManager {
|
|
8
|
+
constructor(connection, config) {
|
|
9
|
+
this.dlqQueues = new Map();
|
|
10
|
+
this.connection = connection;
|
|
11
|
+
this.config = config;
|
|
12
|
+
}
|
|
13
|
+
getDLQQueueName(topic, stepName) {
|
|
14
|
+
const baseQueueName = `${topic}.${stepName}`;
|
|
15
|
+
return `${baseQueueName}${this.config.dlq.suffix}`;
|
|
16
|
+
}
|
|
17
|
+
getOrCreateDLQQueue(queueName) {
|
|
18
|
+
if (!this.dlqQueues.has(queueName)) {
|
|
19
|
+
const ttlMs = this.config.dlq.ttl * constants_1.MILLISECONDS_PER_SECOND;
|
|
20
|
+
const queue = new bullmq_1.Queue(queueName, {
|
|
21
|
+
connection: this.connection,
|
|
22
|
+
prefix: this.config.prefix,
|
|
23
|
+
defaultJobOptions: {
|
|
24
|
+
removeOnComplete: {
|
|
25
|
+
age: ttlMs,
|
|
26
|
+
},
|
|
27
|
+
removeOnFail: {
|
|
28
|
+
age: ttlMs,
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
queue.on('error', (err) => {
|
|
33
|
+
console.error(`${constants_1.LOG_PREFIX} DLQ error for ${queueName}:`, err);
|
|
34
|
+
});
|
|
35
|
+
this.dlqQueues.set(queueName, queue);
|
|
36
|
+
}
|
|
37
|
+
const queue = this.dlqQueues.get(queueName);
|
|
38
|
+
if (!queue) {
|
|
39
|
+
throw new errors_1.QueueCreationError(queueName);
|
|
40
|
+
}
|
|
41
|
+
return queue;
|
|
42
|
+
}
|
|
43
|
+
async moveToDLQ(topic, stepName, event, error, attemptsMade, originalJobId) {
|
|
44
|
+
if (!this.config.dlq.enabled) {
|
|
45
|
+
console.warn(`${constants_1.LOG_PREFIX} DLQ is disabled, skipping move to DLQ for topic ${topic}, step ${stepName}`);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
const dlqQueueName = this.getDLQQueueName(topic, stepName);
|
|
50
|
+
const dlqQueue = this.getOrCreateDLQQueue(dlqQueueName);
|
|
51
|
+
const sanitizedEvent = {
|
|
52
|
+
topic: event.topic,
|
|
53
|
+
data: event.data,
|
|
54
|
+
traceId: event.traceId || 'unknown',
|
|
55
|
+
...(event.flows && { flows: event.flows }),
|
|
56
|
+
...(event.messageGroupId && { messageGroupId: event.messageGroupId }),
|
|
57
|
+
};
|
|
58
|
+
const dlqJobData = {
|
|
59
|
+
originalEvent: sanitizedEvent,
|
|
60
|
+
failureReason: error.message || 'Unknown error',
|
|
61
|
+
failureTimestamp: Date.now(),
|
|
62
|
+
attemptsMade,
|
|
63
|
+
...(originalJobId && { originalJobId }),
|
|
64
|
+
};
|
|
65
|
+
const jobOptions = originalJobId ? { jobId: `${constants_1.DLQ_JOB_PREFIX}${originalJobId}` } : {};
|
|
66
|
+
await dlqQueue.add(`${topic}.dlq`, dlqJobData, jobOptions);
|
|
67
|
+
console.warn(`${constants_1.LOG_PREFIX} Moved failed job to DLQ: ${dlqQueueName}`, {
|
|
68
|
+
topic,
|
|
69
|
+
stepName,
|
|
70
|
+
attemptsMade,
|
|
71
|
+
error: error.message,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
const dlqError = err instanceof Error ? err : new Error(String(err));
|
|
76
|
+
console.error(`${constants_1.LOG_PREFIX} Failed to move job to DLQ for topic ${topic}, step ${stepName}:`, dlqError);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async closeDLQQueue(queueName) {
|
|
80
|
+
const queue = this.dlqQueues.get(queueName);
|
|
81
|
+
if (queue) {
|
|
82
|
+
await queue.close();
|
|
83
|
+
this.dlqQueues.delete(queueName);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async closeAll() {
|
|
87
|
+
const promises = Array.from(this.dlqQueues.values()).map((queue) => queue.close().catch((err) => {
|
|
88
|
+
console.error(`${constants_1.LOG_PREFIX} Error closing DLQ queue:`, err);
|
|
89
|
+
}));
|
|
90
|
+
await Promise.allSettled(promises);
|
|
91
|
+
this.dlqQueues.clear();
|
|
92
|
+
}
|
|
93
|
+
getDLQQueue(queueName) {
|
|
94
|
+
return this.dlqQueues.get(queueName);
|
|
95
|
+
}
|
|
96
|
+
getOrCreateDLQ(queueName) {
|
|
97
|
+
return this.getOrCreateDLQQueue(queueName);
|
|
98
|
+
}
|
|
99
|
+
listDLQQueueNames() {
|
|
100
|
+
return Array.from(this.dlqQueues.keys());
|
|
101
|
+
}
|
|
102
|
+
getDLQSuffix() {
|
|
103
|
+
return this.config.dlq.suffix;
|
|
104
|
+
}
|
|
105
|
+
getPrefix() {
|
|
106
|
+
return this.config.prefix;
|
|
107
|
+
}
|
|
108
|
+
getConnection() {
|
|
109
|
+
return this.connection;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.DLQManager = DLQManager;
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare class BullMQAdapterError extends Error {
|
|
2
|
+
readonly cause?: Error | undefined;
|
|
3
|
+
constructor(message: string, cause?: Error | undefined);
|
|
4
|
+
}
|
|
5
|
+
export declare class QueueCreationError extends BullMQAdapterError {
|
|
6
|
+
constructor(queueName: string, cause?: Error);
|
|
7
|
+
}
|
|
8
|
+
export declare class WorkerCreationError extends BullMQAdapterError {
|
|
9
|
+
constructor(topic: string, stepName: string, cause?: Error);
|
|
10
|
+
}
|
|
11
|
+
export declare class ConnectionError extends BullMQAdapterError {
|
|
12
|
+
constructor(message: string, cause?: Error);
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qBAAa,kBAAmB,SAAQ,KAAK;aAGzB,KAAK,CAAC,EAAE,KAAK;gBAD7B,OAAO,EAAE,MAAM,EACC,KAAK,CAAC,EAAE,KAAK,YAAA;CAQhC;AAED,qBAAa,kBAAmB,SAAQ,kBAAkB;gBAC5C,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI7C;AAED,qBAAa,mBAAoB,SAAQ,kBAAkB;gBAC7C,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3D;AAED,qBAAa,eAAgB,SAAQ,kBAAkB;gBACzC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C"}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ConnectionError = exports.WorkerCreationError = exports.QueueCreationError = exports.BullMQAdapterError = void 0;
|
|
4
|
+
class BullMQAdapterError extends Error {
|
|
5
|
+
constructor(message, cause) {
|
|
6
|
+
super(message);
|
|
7
|
+
this.cause = cause;
|
|
8
|
+
this.name = 'BullMQAdapterError';
|
|
9
|
+
if (cause) {
|
|
10
|
+
this.stack = `${this.stack}\nCaused by: ${cause.stack}`;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
exports.BullMQAdapterError = BullMQAdapterError;
|
|
15
|
+
class QueueCreationError extends BullMQAdapterError {
|
|
16
|
+
constructor(queueName, cause) {
|
|
17
|
+
super(`Failed to create queue: ${queueName}`, cause);
|
|
18
|
+
this.name = 'QueueCreationError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
exports.QueueCreationError = QueueCreationError;
|
|
22
|
+
class WorkerCreationError extends BullMQAdapterError {
|
|
23
|
+
constructor(topic, stepName, cause) {
|
|
24
|
+
super(`Failed to create worker for topic ${topic}, step ${stepName}`, cause);
|
|
25
|
+
this.name = 'WorkerCreationError';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
exports.WorkerCreationError = WorkerCreationError;
|
|
29
|
+
class ConnectionError extends BullMQAdapterError {
|
|
30
|
+
constructor(message, cause) {
|
|
31
|
+
super(`Connection error: ${message}`, cause);
|
|
32
|
+
this.name = 'ConnectionError';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
exports.ConnectionError = ConnectionError;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { BullMQEventAdapter } from './bullmq-event-adapter';
|
|
2
|
+
export { DLQManager } from './dlq-manager';
|
|
3
|
+
export { QueueManager } from './queue-manager';
|
|
4
|
+
export type { BullMQConnectionConfig, BullMQEventAdapterConfig } from './types';
|
|
5
|
+
export type { SubscriberInfo } from './worker-manager';
|
|
6
|
+
export { WorkerManager } from './worker-manager';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,YAAY,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAA;AAC/E,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WorkerManager = exports.QueueManager = exports.DLQManager = exports.BullMQEventAdapter = void 0;
|
|
4
|
+
var bullmq_event_adapter_1 = require("./bullmq-event-adapter");
|
|
5
|
+
Object.defineProperty(exports, "BullMQEventAdapter", { enumerable: true, get: function () { return bullmq_event_adapter_1.BullMQEventAdapter; } });
|
|
6
|
+
var dlq_manager_1 = require("./dlq-manager");
|
|
7
|
+
Object.defineProperty(exports, "DLQManager", { enumerable: true, get: function () { return dlq_manager_1.DLQManager; } });
|
|
8
|
+
var queue_manager_1 = require("./queue-manager");
|
|
9
|
+
Object.defineProperty(exports, "QueueManager", { enumerable: true, get: function () { return queue_manager_1.QueueManager; } });
|
|
10
|
+
var worker_manager_1 = require("./worker-manager");
|
|
11
|
+
Object.defineProperty(exports, "WorkerManager", { enumerable: true, get: function () { return worker_manager_1.WorkerManager; } });
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Event } from '@motiadev/core';
|
|
2
|
+
import { Queue } from 'bullmq';
|
|
3
|
+
import type { Redis } from 'ioredis';
|
|
4
|
+
import type { MergedConfig } from './config-builder';
|
|
5
|
+
import type { SubscriberInfo } from './worker-manager';
|
|
6
|
+
export declare class QueueManager {
|
|
7
|
+
private readonly queues;
|
|
8
|
+
private readonly connection;
|
|
9
|
+
private readonly config;
|
|
10
|
+
constructor(connection: Redis, config: MergedConfig);
|
|
11
|
+
getQueue(queueName: string): Queue;
|
|
12
|
+
getQueueName(topic: string, stepName: string): string;
|
|
13
|
+
enqueueToAll<TData>(event: Event<TData>, subscribers: SubscriberInfo[]): Promise<void>;
|
|
14
|
+
closeQueue(queueName: string): Promise<void>;
|
|
15
|
+
closeAll(): Promise<void>;
|
|
16
|
+
listQueueNames(): string[];
|
|
17
|
+
getPrefix(): string;
|
|
18
|
+
getConnection(): Redis;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=queue-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue-manager.d.ts","sourceRoot":"","sources":["../src/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAC9B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGpD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAEtD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgC;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;gBAEzB,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAKnD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK;IAsBlC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAI/C,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCtF,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,cAAc,IAAI,MAAM,EAAE;IAI1B,SAAS,IAAI,MAAM;IAInB,aAAa,IAAI,KAAK;CAGvB"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QueueManager = void 0;
|
|
4
|
+
const bullmq_1 = require("bullmq");
|
|
5
|
+
const constants_1 = require("./constants");
|
|
6
|
+
const errors_1 = require("./errors");
|
|
7
|
+
class QueueManager {
|
|
8
|
+
constructor(connection, config) {
|
|
9
|
+
this.queues = new Map();
|
|
10
|
+
this.connection = connection;
|
|
11
|
+
this.config = config;
|
|
12
|
+
}
|
|
13
|
+
getQueue(queueName) {
|
|
14
|
+
if (!this.queues.has(queueName)) {
|
|
15
|
+
const queue = new bullmq_1.Queue(queueName, {
|
|
16
|
+
connection: this.connection,
|
|
17
|
+
prefix: this.config.prefix,
|
|
18
|
+
defaultJobOptions: this.config.defaultJobOptions,
|
|
19
|
+
});
|
|
20
|
+
queue.on('error', (err) => {
|
|
21
|
+
console.error(`[BullMQ] Queue error for ${queueName}:`, err);
|
|
22
|
+
});
|
|
23
|
+
this.queues.set(queueName, queue);
|
|
24
|
+
}
|
|
25
|
+
const queue = this.queues.get(queueName);
|
|
26
|
+
if (!queue) {
|
|
27
|
+
throw new errors_1.QueueCreationError(queueName);
|
|
28
|
+
}
|
|
29
|
+
return queue;
|
|
30
|
+
}
|
|
31
|
+
getQueueName(topic, stepName) {
|
|
32
|
+
return `${topic}.${stepName}`;
|
|
33
|
+
}
|
|
34
|
+
async enqueueToAll(event, subscribers) {
|
|
35
|
+
const promises = subscribers.map((subscriber) => {
|
|
36
|
+
const queueName = this.getQueueName(subscriber.topic, subscriber.stepName);
|
|
37
|
+
const queue = this.getQueue(queueName);
|
|
38
|
+
const jobId = event.messageGroupId ? `${queueName}.${event.messageGroupId}` : undefined;
|
|
39
|
+
const jobData = {
|
|
40
|
+
topic: event.topic,
|
|
41
|
+
data: event.data,
|
|
42
|
+
traceId: event.traceId,
|
|
43
|
+
flows: event.flows,
|
|
44
|
+
messageGroupId: event.messageGroupId,
|
|
45
|
+
};
|
|
46
|
+
const maxRetries = subscriber.queueConfig?.maxRetries;
|
|
47
|
+
const attempts = maxRetries != null ? maxRetries + 1 : this.config.defaultJobOptions.attempts;
|
|
48
|
+
const delay = subscriber.queueConfig?.delaySeconds
|
|
49
|
+
? subscriber.queueConfig.delaySeconds * constants_1.MILLISECONDS_PER_SECOND
|
|
50
|
+
: undefined;
|
|
51
|
+
const jobOptions = {
|
|
52
|
+
jobId,
|
|
53
|
+
attempts,
|
|
54
|
+
backoff: this.config.defaultJobOptions.backoff,
|
|
55
|
+
delay,
|
|
56
|
+
};
|
|
57
|
+
return queue.add(event.topic, jobData, jobOptions).then(() => undefined);
|
|
58
|
+
});
|
|
59
|
+
await Promise.all(promises);
|
|
60
|
+
}
|
|
61
|
+
async closeQueue(queueName) {
|
|
62
|
+
const queue = this.queues.get(queueName);
|
|
63
|
+
if (queue) {
|
|
64
|
+
await queue.close();
|
|
65
|
+
this.queues.delete(queueName);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async closeAll() {
|
|
69
|
+
const promises = Array.from(this.queues.values()).map((queue) => queue.close().catch((err) => {
|
|
70
|
+
console.error(`[BullMQ] Error closing queue:`, err);
|
|
71
|
+
}));
|
|
72
|
+
await Promise.allSettled(promises);
|
|
73
|
+
this.queues.clear();
|
|
74
|
+
}
|
|
75
|
+
listQueueNames() {
|
|
76
|
+
return Array.from(this.queues.keys());
|
|
77
|
+
}
|
|
78
|
+
getPrefix() {
|
|
79
|
+
return this.config.prefix;
|
|
80
|
+
}
|
|
81
|
+
getConnection() {
|
|
82
|
+
return this.connection;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
exports.QueueManager = QueueManager;
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { KeepJobs } from 'bullmq';
|
|
2
|
+
import type { Redis, RedisOptions } from 'ioredis';
|
|
3
|
+
export type BullMQConnectionConfig = Redis | RedisOptions;
|
|
4
|
+
export interface BullMQEventAdapterConfig {
|
|
5
|
+
connection: BullMQConnectionConfig;
|
|
6
|
+
concurrency?: number;
|
|
7
|
+
defaultJobOptions?: {
|
|
8
|
+
attempts?: number;
|
|
9
|
+
backoff?: {
|
|
10
|
+
type: 'fixed' | 'exponential';
|
|
11
|
+
delay: number;
|
|
12
|
+
};
|
|
13
|
+
removeOnComplete?: KeepJobs;
|
|
14
|
+
removeOnFail?: KeepJobs;
|
|
15
|
+
};
|
|
16
|
+
prefix?: string;
|
|
17
|
+
dlq?: {
|
|
18
|
+
enabled?: boolean;
|
|
19
|
+
ttl?: number;
|
|
20
|
+
suffix?: string;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACtC,OAAO,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAElD,MAAM,MAAM,sBAAsB,GAAG,KAAK,GAAG,YAAY,CAAA;AAEzD,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,sBAAsB,CAAA;IAClC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,iBAAiB,CAAC,EAAE;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,OAAO,GAAG,aAAa,CAAA;YAC7B,KAAK,EAAE,MAAM,CAAA;SACd,CAAA;QACD,gBAAgB,CAAC,EAAE,QAAQ,CAAA;QAC3B,YAAY,CAAC,EAAE,QAAQ,CAAA;KACxB,CAAA;IACD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,GAAG,CAAC,EAAE;QACJ,OAAO,CAAC,EAAE,OAAO,CAAA;QACjB,GAAG,CAAC,EAAE,MAAM,CAAA;QACZ,MAAM,CAAC,EAAE,MAAM,CAAA;KAChB,CAAA;CACF"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { Event, QueueConfig, SubscriptionHandle } from '@motiadev/core';
|
|
2
|
+
import { Worker } from 'bullmq';
|
|
3
|
+
import type { Redis } from 'ioredis';
|
|
4
|
+
import type { MergedConfig } from './config-builder';
|
|
5
|
+
import type { DLQManager } from './dlq-manager';
|
|
6
|
+
export type SubscriberInfo = {
|
|
7
|
+
topic: string;
|
|
8
|
+
stepName: string;
|
|
9
|
+
queueConfig?: QueueConfig;
|
|
10
|
+
};
|
|
11
|
+
type WorkerInfo = {
|
|
12
|
+
worker: Worker;
|
|
13
|
+
topic: string;
|
|
14
|
+
stepName: string;
|
|
15
|
+
handle: SubscriptionHandle;
|
|
16
|
+
queueConfig?: QueueConfig;
|
|
17
|
+
};
|
|
18
|
+
export declare class WorkerManager {
|
|
19
|
+
private readonly workers;
|
|
20
|
+
private readonly topicSubscriptions;
|
|
21
|
+
private readonly connection;
|
|
22
|
+
private readonly config;
|
|
23
|
+
private readonly getQueueName;
|
|
24
|
+
private readonly dlqManager;
|
|
25
|
+
constructor(connection: Redis, config: MergedConfig, getQueueName: (topic: string, stepName: string) => string, dlqManager?: DLQManager);
|
|
26
|
+
createWorker<TData>(topic: string, stepName: string, handler: (event: Event<TData>) => void | Promise<void>, options?: QueueConfig): SubscriptionHandle;
|
|
27
|
+
getSubscribers(topic: string): SubscriberInfo[];
|
|
28
|
+
getWorkerInfo(id: string): WorkerInfo | undefined;
|
|
29
|
+
removeWorker(id: string): Promise<void>;
|
|
30
|
+
closeAll(): Promise<void>;
|
|
31
|
+
getSubscriptionCount(topic: string): number;
|
|
32
|
+
listTopics(): string[];
|
|
33
|
+
private addTopicSubscription;
|
|
34
|
+
private removeTopicSubscription;
|
|
35
|
+
private setupWorkerHandlers;
|
|
36
|
+
}
|
|
37
|
+
export {};
|
|
38
|
+
//# sourceMappingURL=worker-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-manager.d.ts","sourceRoot":"","sources":["../src/worker-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAC5E,OAAO,EAAY,MAAM,EAAE,MAAM,QAAQ,CAAA;AACzC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAG/C,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B,CAAA;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,kBAAkB,CAAA;IAC1B,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B,CAAA;AAUD,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqC;IAC7D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAsC;IACzE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6C;IAC1E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;gBAG5C,UAAU,EAAE,KAAK,EACjB,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,EACzD,UAAU,CAAC,EAAE,UAAU;IAQzB,YAAY,CAAC,KAAK,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACtD,OAAO,CAAC,EAAE,WAAW,GACpB,kBAAkB;IAuDrB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,EAAE;IAY/C,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAI3C,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWvC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAW/B,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI3C,UAAU,IAAI,MAAM,EAAE;IAItB,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,uBAAuB;IAU/B,OAAO,CAAC,mBAAmB;CAyB5B"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WorkerManager = void 0;
|
|
4
|
+
const bullmq_1 = require("bullmq");
|
|
5
|
+
const uuid_1 = require("uuid");
|
|
6
|
+
const constants_1 = require("./constants");
|
|
7
|
+
const errors_1 = require("./errors");
|
|
8
|
+
class WorkerManager {
|
|
9
|
+
constructor(connection, config, getQueueName, dlqManager) {
|
|
10
|
+
this.workers = new Map();
|
|
11
|
+
this.topicSubscriptions = new Map();
|
|
12
|
+
this.connection = connection;
|
|
13
|
+
this.config = config;
|
|
14
|
+
this.getQueueName = getQueueName;
|
|
15
|
+
this.dlqManager = dlqManager ?? null;
|
|
16
|
+
}
|
|
17
|
+
createWorker(topic, stepName, handler, options) {
|
|
18
|
+
const id = (0, uuid_1.v4)();
|
|
19
|
+
const queueName = this.getQueueName(topic, stepName);
|
|
20
|
+
this.addTopicSubscription(topic, id);
|
|
21
|
+
const concurrency = options?.type === 'fifo' ? constants_1.FIFO_CONCURRENCY : this.config.concurrency;
|
|
22
|
+
const attempts = options?.maxRetries != null ? options.maxRetries + 1 : this.config.defaultJobOptions.attempts;
|
|
23
|
+
const lockDuration = options?.visibilityTimeout ? options.visibilityTimeout * constants_1.MILLISECONDS_PER_SECOND : undefined;
|
|
24
|
+
const worker = new bullmq_1.Worker(queueName, async (job) => {
|
|
25
|
+
const eventData = job.data;
|
|
26
|
+
const event = {
|
|
27
|
+
topic: eventData.topic,
|
|
28
|
+
data: eventData.data,
|
|
29
|
+
traceId: eventData.traceId,
|
|
30
|
+
flows: eventData.flows,
|
|
31
|
+
messageGroupId: eventData.messageGroupId,
|
|
32
|
+
};
|
|
33
|
+
await handler(event);
|
|
34
|
+
}, {
|
|
35
|
+
connection: this.connection,
|
|
36
|
+
prefix: this.config.prefix,
|
|
37
|
+
concurrency,
|
|
38
|
+
lockDuration,
|
|
39
|
+
removeOnComplete: this.config.defaultJobOptions.removeOnComplete,
|
|
40
|
+
removeOnFail: this.config.defaultJobOptions.removeOnFail,
|
|
41
|
+
});
|
|
42
|
+
this.setupWorkerHandlers(worker, topic, stepName, attempts ?? 3);
|
|
43
|
+
const handle = {
|
|
44
|
+
topic,
|
|
45
|
+
id,
|
|
46
|
+
unsubscribe: async () => {
|
|
47
|
+
await this.removeWorker(handle.id);
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
const workerInfo = {
|
|
51
|
+
worker,
|
|
52
|
+
topic,
|
|
53
|
+
stepName,
|
|
54
|
+
handle,
|
|
55
|
+
queueConfig: options,
|
|
56
|
+
};
|
|
57
|
+
this.workers.set(id, workerInfo);
|
|
58
|
+
return handle;
|
|
59
|
+
}
|
|
60
|
+
getSubscribers(topic) {
|
|
61
|
+
const subscriptionIds = this.topicSubscriptions.get(topic);
|
|
62
|
+
if (!subscriptionIds || subscriptionIds.size === 0) {
|
|
63
|
+
return [];
|
|
64
|
+
}
|
|
65
|
+
return Array.from(subscriptionIds)
|
|
66
|
+
.map((id) => this.workers.get(id))
|
|
67
|
+
.filter((info) => info !== undefined)
|
|
68
|
+
.map((info) => ({ topic: info.topic, stepName: info.stepName, queueConfig: info.queueConfig }));
|
|
69
|
+
}
|
|
70
|
+
getWorkerInfo(id) {
|
|
71
|
+
return this.workers.get(id);
|
|
72
|
+
}
|
|
73
|
+
async removeWorker(id) {
|
|
74
|
+
const workerInfo = this.workers.get(id);
|
|
75
|
+
if (!workerInfo) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
this.removeTopicSubscription(workerInfo.topic, id);
|
|
79
|
+
await workerInfo.worker.close();
|
|
80
|
+
this.workers.delete(id);
|
|
81
|
+
}
|
|
82
|
+
async closeAll() {
|
|
83
|
+
const promises = Array.from(this.workers.values()).map((info) => info.worker.close().catch((err) => {
|
|
84
|
+
console.error(`[BullMQ] Error closing worker for topic ${info.topic}, step ${info.stepName}:`, err);
|
|
85
|
+
}));
|
|
86
|
+
await Promise.allSettled(promises);
|
|
87
|
+
this.workers.clear();
|
|
88
|
+
this.topicSubscriptions.clear();
|
|
89
|
+
}
|
|
90
|
+
getSubscriptionCount(topic) {
|
|
91
|
+
return Array.from(this.workers.values()).filter((w) => w.topic === topic).length;
|
|
92
|
+
}
|
|
93
|
+
listTopics() {
|
|
94
|
+
return Array.from(new Set(Array.from(this.workers.values()).map((w) => w.topic)));
|
|
95
|
+
}
|
|
96
|
+
addTopicSubscription(topic, id) {
|
|
97
|
+
if (!this.topicSubscriptions.has(topic)) {
|
|
98
|
+
this.topicSubscriptions.set(topic, new Set());
|
|
99
|
+
}
|
|
100
|
+
this.topicSubscriptions.get(topic)?.add(id);
|
|
101
|
+
}
|
|
102
|
+
removeTopicSubscription(topic, id) {
|
|
103
|
+
const subscriptions = this.topicSubscriptions.get(topic);
|
|
104
|
+
if (subscriptions) {
|
|
105
|
+
subscriptions.delete(id);
|
|
106
|
+
if (subscriptions.size === 0) {
|
|
107
|
+
this.topicSubscriptions.delete(topic);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
setupWorkerHandlers(worker, topic, stepName, attempts) {
|
|
112
|
+
worker.on('error', (err) => {
|
|
113
|
+
const error = new errors_1.WorkerCreationError(topic, stepName, err);
|
|
114
|
+
console.error(`[BullMQ] Worker error for topic ${topic}, step ${stepName}:`, error);
|
|
115
|
+
});
|
|
116
|
+
worker.on('failed', async (job, err) => {
|
|
117
|
+
if (job) {
|
|
118
|
+
const attemptsMade = job.attemptsMade || 0;
|
|
119
|
+
if (attemptsMade >= attempts) {
|
|
120
|
+
if (this.dlqManager) {
|
|
121
|
+
const eventData = job.data;
|
|
122
|
+
const event = {
|
|
123
|
+
topic: eventData.topic || topic,
|
|
124
|
+
data: eventData.data,
|
|
125
|
+
traceId: eventData.traceId || 'unknown',
|
|
126
|
+
...(eventData.flows && { flows: eventData.flows }),
|
|
127
|
+
...(eventData.messageGroupId && { messageGroupId: eventData.messageGroupId }),
|
|
128
|
+
};
|
|
129
|
+
await this.dlqManager.moveToDLQ(topic, stepName, event, err, attemptsMade, job.id);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
exports.WorkerManager = WorkerManager;
|
package/package.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@motiadev/adapter-bullmq-events",
|
|
3
|
+
"description": "BullMQ event adapter for Motia framework, enabling distributed event handling with advanced features like retries, priorities, delays, and FIFO queues.",
|
|
4
|
+
"main": "dist/index.js",
|
|
5
|
+
"types": "dist/index.d.ts",
|
|
6
|
+
"version": "0.13.0-beta.162-717198",
|
|
7
|
+
"dependencies": {
|
|
8
|
+
"bullmq": "^5.63.0",
|
|
9
|
+
"ioredis": "^5.8.2",
|
|
10
|
+
"uuid": "^11.1.0",
|
|
11
|
+
"@motiadev/core": "0.13.0-beta.162-717198"
|
|
12
|
+
},
|
|
13
|
+
"devDependencies": {
|
|
14
|
+
"@types/node": "^22.10.2",
|
|
15
|
+
"typescript": "^5.7.2"
|
|
16
|
+
},
|
|
17
|
+
"peerDependencies": {
|
|
18
|
+
"@motiadev/core": "^0.8.0"
|
|
19
|
+
},
|
|
20
|
+
"scripts": {
|
|
21
|
+
"build": "rm -rf dist && tsc",
|
|
22
|
+
"lint": "biome check .",
|
|
23
|
+
"watch": "tsc --watch"
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import type { Event, EventAdapter, QueueConfig, SubscriptionHandle } from '@motiadev/core'
|
|
2
|
+
import type { Redis } from 'ioredis'
|
|
3
|
+
import { buildConfig, type MergedConfig } from './config-builder'
|
|
4
|
+
import { ConnectionManager } from './connection-manager'
|
|
5
|
+
import { DLQManager } from './dlq-manager'
|
|
6
|
+
import { QueueManager } from './queue-manager'
|
|
7
|
+
import type { BullMQEventAdapterConfig } from './types'
|
|
8
|
+
import { WorkerManager } from './worker-manager'
|
|
9
|
+
|
|
10
|
+
export class BullMQEventAdapter implements EventAdapter {
|
|
11
|
+
private readonly connectionManager: ConnectionManager
|
|
12
|
+
private readonly _queueManager: QueueManager
|
|
13
|
+
private readonly _workerManager: WorkerManager
|
|
14
|
+
private readonly _dlqManager: DLQManager
|
|
15
|
+
private readonly _config: MergedConfig
|
|
16
|
+
|
|
17
|
+
constructor(config: BullMQEventAdapterConfig) {
|
|
18
|
+
this._config = buildConfig(config)
|
|
19
|
+
this.connectionManager = new ConnectionManager(config.connection)
|
|
20
|
+
this._queueManager = new QueueManager(this.connectionManager.connection, this._config)
|
|
21
|
+
this._dlqManager = new DLQManager(this.connectionManager.connection, this._config)
|
|
22
|
+
this._workerManager = new WorkerManager(
|
|
23
|
+
this.connectionManager.connection,
|
|
24
|
+
this._config,
|
|
25
|
+
(topic, stepName) => this._queueManager.getQueueName(topic, stepName),
|
|
26
|
+
this._dlqManager,
|
|
27
|
+
)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
get connection(): Redis {
|
|
31
|
+
return this.connectionManager.connection
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
get prefix(): string {
|
|
35
|
+
return this._config.prefix
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
get dlqSuffix(): string {
|
|
39
|
+
return this._config.dlq.suffix
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
get queueManager(): QueueManager {
|
|
43
|
+
return this._queueManager
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
get workerManager(): WorkerManager {
|
|
47
|
+
return this._workerManager
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
get dlqManager(): DLQManager {
|
|
51
|
+
return this._dlqManager
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async emit<TData>(event: Event<TData>): Promise<void> {
|
|
55
|
+
const subscribers = this._workerManager.getSubscribers(event.topic)
|
|
56
|
+
if (subscribers.length === 0) {
|
|
57
|
+
return
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
await this._queueManager.enqueueToAll(event, subscribers)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async subscribe<TData>(
|
|
64
|
+
topic: string,
|
|
65
|
+
stepName: string,
|
|
66
|
+
handler: (event: Event<TData>) => void | Promise<void>,
|
|
67
|
+
options?: QueueConfig,
|
|
68
|
+
): Promise<SubscriptionHandle> {
|
|
69
|
+
const queueName = this._queueManager.getQueueName(topic, stepName)
|
|
70
|
+
this._queueManager.getQueue(queueName)
|
|
71
|
+
|
|
72
|
+
return this._workerManager.createWorker(topic, stepName, handler, options)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async unsubscribe(handle: SubscriptionHandle): Promise<void> {
|
|
76
|
+
const workerInfo = this._workerManager.getWorkerInfo(handle.id)
|
|
77
|
+
if (workerInfo) {
|
|
78
|
+
const queueName = this._queueManager.getQueueName(workerInfo.topic, workerInfo.stepName)
|
|
79
|
+
await this._queueManager.closeQueue(queueName)
|
|
80
|
+
await this._workerManager.removeWorker(handle.id)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async shutdown(): Promise<void> {
|
|
85
|
+
await Promise.allSettled([
|
|
86
|
+
this._workerManager.closeAll(),
|
|
87
|
+
this._queueManager.closeAll(),
|
|
88
|
+
this._dlqManager.closeAll(),
|
|
89
|
+
])
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
await this.connectionManager.close()
|
|
93
|
+
} catch (err) {
|
|
94
|
+
console.error('[BullMQ] Error closing connection:', err)
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async getSubscriptionCount(topic: string): Promise<number> {
|
|
99
|
+
return this._workerManager.getSubscriptionCount(topic)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async listTopics(): Promise<string[]> {
|
|
103
|
+
return this._workerManager.listTopics()
|
|
104
|
+
}
|
|
105
|
+
}
|