@motiadev/adapter-bullmq-events 0.14.0-beta.165-256670 → 0.14.0-beta.165-246284
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bullmq-event-adapter.d.mts +31 -0
- package/dist/bullmq-event-adapter.d.mts.map +1 -0
- package/dist/bullmq-event-adapter.mjs +74 -0
- package/dist/bullmq-event-adapter.mjs.map +1 -0
- package/dist/config-builder.d.mts +9 -0
- package/dist/config-builder.d.mts.map +1 -0
- package/dist/config-builder.mjs +28 -0
- package/dist/config-builder.mjs.map +1 -0
- package/dist/connection-manager.mjs +35 -0
- package/dist/connection-manager.mjs.map +1 -0
- package/dist/constants.mjs +18 -0
- package/dist/constants.mjs.map +1 -0
- package/dist/dlq-manager.d.mts +26 -0
- package/dist/dlq-manager.d.mts.map +1 -0
- package/dist/dlq-manager.mjs +105 -0
- package/dist/dlq-manager.mjs.map +1 -0
- package/dist/errors.mjs +31 -0
- package/dist/errors.mjs.map +1 -0
- package/dist/index.d.mts +6 -0
- package/dist/index.mjs +6 -0
- package/dist/queue-manager.d.mts +24 -0
- package/dist/queue-manager.d.mts.map +1 -0
- package/dist/queue-manager.mjs +83 -0
- package/dist/queue-manager.mjs.map +1 -0
- package/dist/types.d.mts +27 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/worker-manager.d.mts +41 -0
- package/dist/worker-manager.d.mts.map +1 -0
- package/dist/worker-manager.mjs +130 -0
- package/dist/worker-manager.mjs.map +1 -0
- package/package.json +11 -7
- package/tsconfig.json +2 -2
- package/tsdown.config.ts +17 -0
- package/dist/bullmq-event-adapter.d.ts +0 -27
- package/dist/bullmq-event-adapter.d.ts.map +0 -1
- package/dist/bullmq-event-adapter.js +0 -75
- package/dist/config-builder.d.ts +0 -6
- package/dist/config-builder.d.ts.map +0 -1
- package/dist/config-builder.js +0 -29
- package/dist/connection-manager.d.ts +0 -10
- package/dist/connection-manager.d.ts.map +0 -1
- package/dist/connection-manager.js +0 -39
- package/dist/constants.d.ts +0 -14
- package/dist/constants.d.ts.map +0 -1
- package/dist/constants.js +0 -16
- package/dist/dlq-manager.d.ts +0 -22
- package/dist/dlq-manager.d.ts.map +0 -1
- package/dist/dlq-manager.js +0 -112
- package/dist/errors.d.ts +0 -14
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js +0 -35
- package/dist/index.d.ts +0 -7
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -11
- package/dist/queue-manager.d.ts +0 -20
- package/dist/queue-manager.d.ts.map +0 -1
- package/dist/queue-manager.js +0 -85
- package/dist/types.d.ts +0 -23
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -2
- package/dist/worker-manager.d.ts +0 -38
- package/dist/worker-manager.d.ts.map +0 -1
- package/dist/worker-manager.js +0 -136
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { FIFO_CONCURRENCY, MILLISECONDS_PER_SECOND } from "./constants.mjs";
|
|
2
|
+
import { WorkerCreationError } from "./errors.mjs";
|
|
3
|
+
import { Worker } from "bullmq";
|
|
4
|
+
import { v4 } from "uuid";
|
|
5
|
+
|
|
6
|
+
//#region src/worker-manager.ts
|
|
7
|
+
var WorkerManager = class {
|
|
8
|
+
constructor(connection, config, getQueueName, dlqManager) {
|
|
9
|
+
this.workers = /* @__PURE__ */ new Map();
|
|
10
|
+
this.topicSubscriptions = /* @__PURE__ */ new Map();
|
|
11
|
+
this.connection = connection;
|
|
12
|
+
this.config = config;
|
|
13
|
+
this.getQueueName = getQueueName;
|
|
14
|
+
this.dlqManager = dlqManager ?? null;
|
|
15
|
+
}
|
|
16
|
+
createWorker(topic, stepName, handler, options) {
|
|
17
|
+
const id = v4();
|
|
18
|
+
const queueName = this.getQueueName(topic, stepName);
|
|
19
|
+
this.addTopicSubscription(topic, id);
|
|
20
|
+
const concurrency = options?.type === "fifo" ? FIFO_CONCURRENCY : this.config.concurrency;
|
|
21
|
+
const attempts = options?.maxRetries != null ? options.maxRetries + 1 : this.config.defaultJobOptions.attempts;
|
|
22
|
+
const lockDuration = options?.visibilityTimeout ? options.visibilityTimeout * MILLISECONDS_PER_SECOND : void 0;
|
|
23
|
+
const worker = new Worker(queueName, async (job) => {
|
|
24
|
+
const eventData = job.data;
|
|
25
|
+
await handler({
|
|
26
|
+
topic: eventData.topic,
|
|
27
|
+
data: eventData.data,
|
|
28
|
+
traceId: eventData.traceId,
|
|
29
|
+
flows: eventData.flows,
|
|
30
|
+
messageGroupId: eventData.messageGroupId
|
|
31
|
+
});
|
|
32
|
+
}, {
|
|
33
|
+
connection: this.connection,
|
|
34
|
+
prefix: this.config.prefix,
|
|
35
|
+
concurrency,
|
|
36
|
+
lockDuration,
|
|
37
|
+
removeOnComplete: this.config.defaultJobOptions.removeOnComplete,
|
|
38
|
+
removeOnFail: this.config.defaultJobOptions.removeOnFail
|
|
39
|
+
});
|
|
40
|
+
this.setupWorkerHandlers(worker, topic, stepName, attempts ?? 3);
|
|
41
|
+
const handle = {
|
|
42
|
+
topic,
|
|
43
|
+
id,
|
|
44
|
+
unsubscribe: async () => {
|
|
45
|
+
await this.removeWorker(handle.id);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
const workerInfo = {
|
|
49
|
+
worker,
|
|
50
|
+
topic,
|
|
51
|
+
stepName,
|
|
52
|
+
handle,
|
|
53
|
+
queueConfig: options
|
|
54
|
+
};
|
|
55
|
+
this.workers.set(id, workerInfo);
|
|
56
|
+
return handle;
|
|
57
|
+
}
|
|
58
|
+
getSubscribers(topic) {
|
|
59
|
+
const subscriptionIds = this.topicSubscriptions.get(topic);
|
|
60
|
+
if (!subscriptionIds || subscriptionIds.size === 0) return [];
|
|
61
|
+
return Array.from(subscriptionIds).map((id) => this.workers.get(id)).filter((info) => info !== void 0).map((info) => ({
|
|
62
|
+
topic: info.topic,
|
|
63
|
+
stepName: info.stepName,
|
|
64
|
+
queueConfig: info.queueConfig
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
getWorkerInfo(id) {
|
|
68
|
+
return this.workers.get(id);
|
|
69
|
+
}
|
|
70
|
+
async removeWorker(id) {
|
|
71
|
+
const workerInfo = this.workers.get(id);
|
|
72
|
+
if (!workerInfo) return;
|
|
73
|
+
this.removeTopicSubscription(workerInfo.topic, id);
|
|
74
|
+
await workerInfo.worker.close();
|
|
75
|
+
this.workers.delete(id);
|
|
76
|
+
}
|
|
77
|
+
async closeAll() {
|
|
78
|
+
const promises = Array.from(this.workers.values()).map((info) => info.worker.close().catch((err) => {
|
|
79
|
+
console.error(`[BullMQ] Error closing worker for topic ${info.topic}, step ${info.stepName}:`, err);
|
|
80
|
+
}));
|
|
81
|
+
await Promise.allSettled(promises);
|
|
82
|
+
this.workers.clear();
|
|
83
|
+
this.topicSubscriptions.clear();
|
|
84
|
+
}
|
|
85
|
+
getSubscriptionCount(topic) {
|
|
86
|
+
return Array.from(this.workers.values()).filter((w) => w.topic === topic).length;
|
|
87
|
+
}
|
|
88
|
+
listTopics() {
|
|
89
|
+
return Array.from(new Set(Array.from(this.workers.values()).map((w) => w.topic)));
|
|
90
|
+
}
|
|
91
|
+
addTopicSubscription(topic, id) {
|
|
92
|
+
if (!this.topicSubscriptions.has(topic)) this.topicSubscriptions.set(topic, /* @__PURE__ */ new Set());
|
|
93
|
+
this.topicSubscriptions.get(topic)?.add(id);
|
|
94
|
+
}
|
|
95
|
+
removeTopicSubscription(topic, id) {
|
|
96
|
+
const subscriptions = this.topicSubscriptions.get(topic);
|
|
97
|
+
if (subscriptions) {
|
|
98
|
+
subscriptions.delete(id);
|
|
99
|
+
if (subscriptions.size === 0) this.topicSubscriptions.delete(topic);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
setupWorkerHandlers(worker, topic, stepName, attempts) {
|
|
103
|
+
worker.on("error", (err) => {
|
|
104
|
+
const error = new WorkerCreationError(topic, stepName, err);
|
|
105
|
+
console.error(`[BullMQ] Worker error for topic ${topic}, step ${stepName}:`, error);
|
|
106
|
+
});
|
|
107
|
+
worker.on("failed", async (job, err) => {
|
|
108
|
+
if (job) {
|
|
109
|
+
const attemptsMade = job.attemptsMade || 0;
|
|
110
|
+
if (attemptsMade >= attempts) {
|
|
111
|
+
if (this.dlqManager) {
|
|
112
|
+
const eventData = job.data;
|
|
113
|
+
const event = {
|
|
114
|
+
topic: eventData.topic || topic,
|
|
115
|
+
data: eventData.data,
|
|
116
|
+
traceId: eventData.traceId || "unknown",
|
|
117
|
+
...eventData.flows && { flows: eventData.flows },
|
|
118
|
+
...eventData.messageGroupId && { messageGroupId: eventData.messageGroupId }
|
|
119
|
+
};
|
|
120
|
+
await this.dlqManager.moveToDLQ(topic, stepName, event, err, attemptsMade, job.id);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
//#endregion
|
|
129
|
+
export { WorkerManager };
|
|
130
|
+
//# sourceMappingURL=worker-manager.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-manager.mjs","names":["uuidv4","handle: SubscriptionHandle","workerInfo: WorkerInfo"],"sources":["../src/worker-manager.ts"],"sourcesContent":["import type { Event, QueueConfig, SubscriptionHandle } from '@motiadev/core'\nimport { type Job, Worker } from 'bullmq'\nimport type { Redis } from 'ioredis'\nimport { v4 as uuidv4 } from 'uuid'\nimport type { MergedConfig } from './config-builder'\nimport { FIFO_CONCURRENCY, MILLISECONDS_PER_SECOND } from './constants'\nimport type { DLQManager } from './dlq-manager'\nimport { WorkerCreationError } from './errors'\n\nexport type SubscriberInfo = {\n topic: string\n stepName: string\n queueConfig?: QueueConfig\n}\n\ntype WorkerInfo = {\n worker: Worker\n topic: string\n stepName: string\n handle: SubscriptionHandle\n queueConfig?: QueueConfig\n}\n\ntype JobData<TData> = {\n topic: string\n data: TData\n traceId: string\n flows?: string[]\n messageGroupId?: string\n}\n\nexport class WorkerManager {\n private readonly workers: Map<string, WorkerInfo> = new Map()\n private readonly topicSubscriptions: Map<string, Set<string>> = new Map()\n private readonly connection: Redis\n private readonly config: MergedConfig\n private readonly getQueueName: (topic: string, stepName: string) => string\n private readonly dlqManager: DLQManager | null\n\n constructor(\n connection: Redis,\n config: MergedConfig,\n getQueueName: (topic: string, stepName: string) => string,\n dlqManager?: DLQManager,\n ) {\n this.connection = connection\n this.config = config\n this.getQueueName = getQueueName\n this.dlqManager = dlqManager ?? null\n }\n\n createWorker<TData>(\n topic: string,\n stepName: string,\n handler: (event: Event<TData>) => void | Promise<void>,\n options?: QueueConfig,\n ): SubscriptionHandle {\n const id = uuidv4()\n const queueName = this.getQueueName(topic, stepName)\n\n this.addTopicSubscription(topic, id)\n\n const concurrency = options?.type === 'fifo' ? FIFO_CONCURRENCY : this.config.concurrency\n const attempts = options?.maxRetries != null ? options.maxRetries + 1 : this.config.defaultJobOptions.attempts\n const lockDuration = options?.visibilityTimeout ? options.visibilityTimeout * MILLISECONDS_PER_SECOND : undefined\n\n const worker = new Worker(\n queueName,\n async (job: Job<JobData<TData>>) => {\n const eventData = job.data\n const event = {\n topic: eventData.topic,\n data: eventData.data,\n traceId: eventData.traceId,\n flows: eventData.flows,\n messageGroupId: eventData.messageGroupId,\n } as Event<TData>\n await handler(event)\n },\n {\n connection: this.connection,\n prefix: this.config.prefix,\n concurrency,\n lockDuration,\n removeOnComplete: this.config.defaultJobOptions.removeOnComplete,\n removeOnFail: this.config.defaultJobOptions.removeOnFail,\n },\n )\n\n this.setupWorkerHandlers(worker, topic, stepName, attempts ?? 3)\n\n const handle: SubscriptionHandle = {\n topic,\n id,\n unsubscribe: async () => {\n await this.removeWorker(handle.id)\n },\n }\n\n const workerInfo: WorkerInfo = {\n worker,\n topic,\n stepName,\n handle,\n queueConfig: options,\n }\n\n this.workers.set(id, workerInfo)\n return handle\n }\n\n getSubscribers(topic: string): SubscriberInfo[] {\n const subscriptionIds = this.topicSubscriptions.get(topic)\n if (!subscriptionIds || subscriptionIds.size === 0) {\n return []\n }\n\n return Array.from(subscriptionIds)\n .map((id) => this.workers.get(id))\n .filter((info): info is WorkerInfo => info !== undefined)\n .map((info) => ({ topic: info.topic, stepName: info.stepName, queueConfig: info.queueConfig }))\n }\n\n getWorkerInfo(id: string): WorkerInfo | undefined {\n return this.workers.get(id)\n }\n\n async removeWorker(id: string): Promise<void> {\n const workerInfo = this.workers.get(id)\n if (!workerInfo) {\n return\n }\n\n this.removeTopicSubscription(workerInfo.topic, id)\n await workerInfo.worker.close()\n this.workers.delete(id)\n }\n\n async closeAll(): Promise<void> {\n const promises = Array.from(this.workers.values()).map((info) =>\n info.worker.close().catch((err) => {\n console.error(`[BullMQ] Error closing worker for topic ${info.topic}, step ${info.stepName}:`, err)\n }),\n )\n await Promise.allSettled(promises)\n this.workers.clear()\n this.topicSubscriptions.clear()\n }\n\n getSubscriptionCount(topic: string): number {\n return Array.from(this.workers.values()).filter((w) => w.topic === topic).length\n }\n\n listTopics(): string[] {\n return Array.from(new Set(Array.from(this.workers.values()).map((w) => w.topic)))\n }\n\n private addTopicSubscription(topic: string, id: string): void {\n if (!this.topicSubscriptions.has(topic)) {\n this.topicSubscriptions.set(topic, new Set())\n }\n this.topicSubscriptions.get(topic)?.add(id)\n }\n\n private removeTopicSubscription(topic: string, id: string): void {\n const subscriptions = this.topicSubscriptions.get(topic)\n if (subscriptions) {\n subscriptions.delete(id)\n if (subscriptions.size === 0) {\n this.topicSubscriptions.delete(topic)\n }\n }\n }\n\n private setupWorkerHandlers(worker: Worker, topic: string, stepName: string, attempts: number): void {\n worker.on('error', (err: Error) => {\n const error = new WorkerCreationError(topic, stepName, err)\n console.error(`[BullMQ] Worker error for topic ${topic}, step ${stepName}:`, error)\n })\n worker.on('failed', async (job: Job<JobData<unknown>> | undefined, err: Error) => {\n if (job) {\n const attemptsMade = job.attemptsMade || 0\n if (attemptsMade >= attempts) {\n if (this.dlqManager) {\n const eventData = job.data\n const event = {\n topic: eventData.topic || topic,\n data: eventData.data,\n traceId: eventData.traceId || 'unknown',\n ...(eventData.flows && { flows: eventData.flows }),\n ...(eventData.messageGroupId && { messageGroupId: eventData.messageGroupId }),\n } as Event<unknown>\n\n await this.dlqManager.moveToDLQ(topic, stepName, event, err, attemptsMade, job.id)\n }\n }\n }\n })\n }\n}\n"],"mappings":";;;;;;AA+BA,IAAa,gBAAb,MAA2B;CAQzB,YACE,YACA,QACA,cACA,YACA;iCAZkD,IAAI,KAAK;4CACG,IAAI,KAAK;AAYvE,OAAK,aAAa;AAClB,OAAK,SAAS;AACd,OAAK,eAAe;AACpB,OAAK,aAAa,cAAc;;CAGlC,aACE,OACA,UACA,SACA,SACoB;EACpB,MAAM,KAAKA,IAAQ;EACnB,MAAM,YAAY,KAAK,aAAa,OAAO,SAAS;AAEpD,OAAK,qBAAqB,OAAO,GAAG;EAEpC,MAAM,cAAc,SAAS,SAAS,SAAS,mBAAmB,KAAK,OAAO;EAC9E,MAAM,WAAW,SAAS,cAAc,OAAO,QAAQ,aAAa,IAAI,KAAK,OAAO,kBAAkB;EACtG,MAAM,eAAe,SAAS,oBAAoB,QAAQ,oBAAoB,0BAA0B;EAExG,MAAM,SAAS,IAAI,OACjB,WACA,OAAO,QAA6B;GAClC,MAAM,YAAY,IAAI;AAQtB,SAAM,QAPQ;IACZ,OAAO,UAAU;IACjB,MAAM,UAAU;IAChB,SAAS,UAAU;IACnB,OAAO,UAAU;IACjB,gBAAgB,UAAU;IAC3B,CACmB;KAEtB;GACE,YAAY,KAAK;GACjB,QAAQ,KAAK,OAAO;GACpB;GACA;GACA,kBAAkB,KAAK,OAAO,kBAAkB;GAChD,cAAc,KAAK,OAAO,kBAAkB;GAC7C,CACF;AAED,OAAK,oBAAoB,QAAQ,OAAO,UAAU,YAAY,EAAE;EAEhE,MAAMC,SAA6B;GACjC;GACA;GACA,aAAa,YAAY;AACvB,UAAM,KAAK,aAAa,OAAO,GAAG;;GAErC;EAED,MAAMC,aAAyB;GAC7B;GACA;GACA;GACA;GACA,aAAa;GACd;AAED,OAAK,QAAQ,IAAI,IAAI,WAAW;AAChC,SAAO;;CAGT,eAAe,OAAiC;EAC9C,MAAM,kBAAkB,KAAK,mBAAmB,IAAI,MAAM;AAC1D,MAAI,CAAC,mBAAmB,gBAAgB,SAAS,EAC/C,QAAO,EAAE;AAGX,SAAO,MAAM,KAAK,gBAAgB,CAC/B,KAAK,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,CACjC,QAAQ,SAA6B,SAAS,OAAU,CACxD,KAAK,UAAU;GAAE,OAAO,KAAK;GAAO,UAAU,KAAK;GAAU,aAAa,KAAK;GAAa,EAAE;;CAGnG,cAAc,IAAoC;AAChD,SAAO,KAAK,QAAQ,IAAI,GAAG;;CAG7B,MAAM,aAAa,IAA2B;EAC5C,MAAM,aAAa,KAAK,QAAQ,IAAI,GAAG;AACvC,MAAI,CAAC,WACH;AAGF,OAAK,wBAAwB,WAAW,OAAO,GAAG;AAClD,QAAM,WAAW,OAAO,OAAO;AAC/B,OAAK,QAAQ,OAAO,GAAG;;CAGzB,MAAM,WAA0B;EAC9B,MAAM,WAAW,MAAM,KAAK,KAAK,QAAQ,QAAQ,CAAC,CAAC,KAAK,SACtD,KAAK,OAAO,OAAO,CAAC,OAAO,QAAQ;AACjC,WAAQ,MAAM,2CAA2C,KAAK,MAAM,SAAS,KAAK,SAAS,IAAI,IAAI;IACnG,CACH;AACD,QAAM,QAAQ,WAAW,SAAS;AAClC,OAAK,QAAQ,OAAO;AACpB,OAAK,mBAAmB,OAAO;;CAGjC,qBAAqB,OAAuB;AAC1C,SAAO,MAAM,KAAK,KAAK,QAAQ,QAAQ,CAAC,CAAC,QAAQ,MAAM,EAAE,UAAU,MAAM,CAAC;;CAG5E,aAAuB;AACrB,SAAO,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,KAAK,QAAQ,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC;;CAGnF,AAAQ,qBAAqB,OAAe,IAAkB;AAC5D,MAAI,CAAC,KAAK,mBAAmB,IAAI,MAAM,CACrC,MAAK,mBAAmB,IAAI,uBAAO,IAAI,KAAK,CAAC;AAE/C,OAAK,mBAAmB,IAAI,MAAM,EAAE,IAAI,GAAG;;CAG7C,AAAQ,wBAAwB,OAAe,IAAkB;EAC/D,MAAM,gBAAgB,KAAK,mBAAmB,IAAI,MAAM;AACxD,MAAI,eAAe;AACjB,iBAAc,OAAO,GAAG;AACxB,OAAI,cAAc,SAAS,EACzB,MAAK,mBAAmB,OAAO,MAAM;;;CAK3C,AAAQ,oBAAoB,QAAgB,OAAe,UAAkB,UAAwB;AACnG,SAAO,GAAG,UAAU,QAAe;GACjC,MAAM,QAAQ,IAAI,oBAAoB,OAAO,UAAU,IAAI;AAC3D,WAAQ,MAAM,mCAAmC,MAAM,SAAS,SAAS,IAAI,MAAM;IACnF;AACF,SAAO,GAAG,UAAU,OAAO,KAAwC,QAAe;AAChF,OAAI,KAAK;IACP,MAAM,eAAe,IAAI,gBAAgB;AACzC,QAAI,gBAAgB,UAClB;SAAI,KAAK,YAAY;MACnB,MAAM,YAAY,IAAI;MACtB,MAAM,QAAQ;OACZ,OAAO,UAAU,SAAS;OAC1B,MAAM,UAAU;OAChB,SAAS,UAAU,WAAW;OAC9B,GAAI,UAAU,SAAS,EAAE,OAAO,UAAU,OAAO;OACjD,GAAI,UAAU,kBAAkB,EAAE,gBAAgB,UAAU,gBAAgB;OAC7E;AAED,YAAM,KAAK,WAAW,UAAU,OAAO,UAAU,OAAO,KAAK,cAAc,IAAI,GAAG;;;;IAIxF"}
|
package/package.json
CHANGED
|
@@ -1,25 +1,29 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@motiadev/adapter-bullmq-events",
|
|
3
3
|
"description": "BullMQ event adapter for Motia framework, enabling distributed event handling with advanced features like retries, priorities, delays, and FIFO queues.",
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "dist/index.mjs",
|
|
6
|
+
"module": "dist/index.mjs",
|
|
7
|
+
"types": "dist/index.d.mts",
|
|
8
|
+
"version": "0.14.0-beta.165-246284",
|
|
7
9
|
"dependencies": {
|
|
8
10
|
"bullmq": "^5.63.0",
|
|
9
11
|
"ioredis": "^5.8.2",
|
|
10
12
|
"uuid": "^11.1.0",
|
|
11
|
-
"@motiadev/core": "0.14.0-beta.165-
|
|
13
|
+
"@motiadev/core": "0.14.0-beta.165-246284"
|
|
12
14
|
},
|
|
13
15
|
"devDependencies": {
|
|
14
|
-
"
|
|
16
|
+
"rimraf": "^6.0.1",
|
|
17
|
+
"tsdown": "^0.16.6",
|
|
15
18
|
"typescript": "^5.7.2"
|
|
16
19
|
},
|
|
17
20
|
"peerDependencies": {
|
|
18
21
|
"@motiadev/core": ">=0.8.0"
|
|
19
22
|
},
|
|
20
23
|
"scripts": {
|
|
21
|
-
"build": "
|
|
24
|
+
"build": "tsdown",
|
|
25
|
+
"dev": "tsdown --watch",
|
|
22
26
|
"lint": "biome check .",
|
|
23
|
-
"
|
|
27
|
+
"clean": "rimraf dist"
|
|
24
28
|
}
|
|
25
29
|
}
|
package/tsconfig.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"compilerOptions": {
|
|
3
3
|
"target": "ES2020",
|
|
4
|
-
"module": "
|
|
4
|
+
"module": "ESNext",
|
|
5
5
|
"lib": ["ES2020"],
|
|
6
6
|
"declaration": true,
|
|
7
7
|
"declarationMap": true,
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"skipLibCheck": true,
|
|
13
13
|
"forceConsistentCasingInFileNames": true,
|
|
14
14
|
"resolveJsonModule": true,
|
|
15
|
-
"moduleResolution": "
|
|
15
|
+
"moduleResolution": "bundler"
|
|
16
16
|
},
|
|
17
17
|
"include": ["src/**/*"],
|
|
18
18
|
"exclude": ["node_modules", "dist"]
|
package/tsdown.config.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { defineConfig } from 'tsdown'
|
|
2
|
+
|
|
3
|
+
export default defineConfig({
|
|
4
|
+
entry: {
|
|
5
|
+
index: './src/index.ts',
|
|
6
|
+
},
|
|
7
|
+
format: 'esm',
|
|
8
|
+
platform: 'node',
|
|
9
|
+
external: ['@motiadev/core', 'bullmq', 'ioredis', 'uuid'],
|
|
10
|
+
dts: {
|
|
11
|
+
build: true,
|
|
12
|
+
},
|
|
13
|
+
clean: true,
|
|
14
|
+
outDir: 'dist',
|
|
15
|
+
sourcemap: true,
|
|
16
|
+
unbundle: true,
|
|
17
|
+
})
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import type { Event, EventAdapter, QueueConfig, SubscriptionHandle } from '@motiadev/core';
|
|
2
|
-
import type { Redis } from 'ioredis';
|
|
3
|
-
import { DLQManager } from './dlq-manager';
|
|
4
|
-
import { QueueManager } from './queue-manager';
|
|
5
|
-
import type { BullMQEventAdapterConfig } from './types';
|
|
6
|
-
import { WorkerManager } from './worker-manager';
|
|
7
|
-
export declare class BullMQEventAdapter implements EventAdapter {
|
|
8
|
-
private readonly connectionManager;
|
|
9
|
-
private readonly _queueManager;
|
|
10
|
-
private readonly _workerManager;
|
|
11
|
-
private readonly _dlqManager;
|
|
12
|
-
private readonly _config;
|
|
13
|
-
constructor(config: BullMQEventAdapterConfig);
|
|
14
|
-
get connection(): Redis;
|
|
15
|
-
get prefix(): string;
|
|
16
|
-
get dlqSuffix(): string;
|
|
17
|
-
get queueManager(): QueueManager;
|
|
18
|
-
get workerManager(): WorkerManager;
|
|
19
|
-
get dlqManager(): DLQManager;
|
|
20
|
-
emit<TData>(event: Event<TData>): Promise<void>;
|
|
21
|
-
subscribe<TData>(topic: string, stepName: string, handler: (event: Event<TData>) => void | Promise<void>, options?: QueueConfig): Promise<SubscriptionHandle>;
|
|
22
|
-
unsubscribe(handle: SubscriptionHandle): Promise<void>;
|
|
23
|
-
shutdown(): Promise<void>;
|
|
24
|
-
getSubscriptionCount(topic: string): Promise<number>;
|
|
25
|
-
listTopics(): Promise<string[]>;
|
|
26
|
-
}
|
|
27
|
-
//# sourceMappingURL=bullmq-event-adapter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"bullmq-event-adapter.d.ts","sourceRoot":"","sources":["../src/bullmq-event-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAC1F,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGpC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAA;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAEhD,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IACrD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAc;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAe;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAY;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;gBAE1B,MAAM,EAAE,wBAAwB;IAa5C,IAAI,UAAU,IAAI,KAAK,CAEtB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAEK,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/C,SAAS,CAAC,KAAK,EACnB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACtD,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,kBAAkB,CAAC;IAOxB,WAAW,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAczB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIpD,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGtC"}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BullMQEventAdapter = void 0;
|
|
4
|
-
const config_builder_1 = require("./config-builder");
|
|
5
|
-
const connection_manager_1 = require("./connection-manager");
|
|
6
|
-
const dlq_manager_1 = require("./dlq-manager");
|
|
7
|
-
const queue_manager_1 = require("./queue-manager");
|
|
8
|
-
const worker_manager_1 = require("./worker-manager");
|
|
9
|
-
class BullMQEventAdapter {
|
|
10
|
-
constructor(config) {
|
|
11
|
-
this._config = (0, config_builder_1.buildConfig)(config);
|
|
12
|
-
this.connectionManager = new connection_manager_1.ConnectionManager(config.connection);
|
|
13
|
-
this._queueManager = new queue_manager_1.QueueManager(this.connectionManager.connection, this._config);
|
|
14
|
-
this._dlqManager = new dlq_manager_1.DLQManager(this.connectionManager.connection, this._config);
|
|
15
|
-
this._workerManager = new worker_manager_1.WorkerManager(this.connectionManager.connection, this._config, (topic, stepName) => this._queueManager.getQueueName(topic, stepName), this._dlqManager);
|
|
16
|
-
}
|
|
17
|
-
get connection() {
|
|
18
|
-
return this.connectionManager.connection;
|
|
19
|
-
}
|
|
20
|
-
get prefix() {
|
|
21
|
-
return this._config.prefix;
|
|
22
|
-
}
|
|
23
|
-
get dlqSuffix() {
|
|
24
|
-
return this._config.dlq.suffix;
|
|
25
|
-
}
|
|
26
|
-
get queueManager() {
|
|
27
|
-
return this._queueManager;
|
|
28
|
-
}
|
|
29
|
-
get workerManager() {
|
|
30
|
-
return this._workerManager;
|
|
31
|
-
}
|
|
32
|
-
get dlqManager() {
|
|
33
|
-
return this._dlqManager;
|
|
34
|
-
}
|
|
35
|
-
async emit(event) {
|
|
36
|
-
const subscribers = this._workerManager.getSubscribers(event.topic);
|
|
37
|
-
if (subscribers.length === 0) {
|
|
38
|
-
return;
|
|
39
|
-
}
|
|
40
|
-
await this._queueManager.enqueueToAll(event, subscribers);
|
|
41
|
-
}
|
|
42
|
-
async subscribe(topic, stepName, handler, options) {
|
|
43
|
-
const queueName = this._queueManager.getQueueName(topic, stepName);
|
|
44
|
-
this._queueManager.getQueue(queueName);
|
|
45
|
-
return this._workerManager.createWorker(topic, stepName, handler, options);
|
|
46
|
-
}
|
|
47
|
-
async unsubscribe(handle) {
|
|
48
|
-
const workerInfo = this._workerManager.getWorkerInfo(handle.id);
|
|
49
|
-
if (workerInfo) {
|
|
50
|
-
const queueName = this._queueManager.getQueueName(workerInfo.topic, workerInfo.stepName);
|
|
51
|
-
await this._queueManager.closeQueue(queueName);
|
|
52
|
-
await this._workerManager.removeWorker(handle.id);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
async shutdown() {
|
|
56
|
-
await Promise.allSettled([
|
|
57
|
-
this._workerManager.closeAll(),
|
|
58
|
-
this._queueManager.closeAll(),
|
|
59
|
-
this._dlqManager.closeAll(),
|
|
60
|
-
]);
|
|
61
|
-
try {
|
|
62
|
-
await this.connectionManager.close();
|
|
63
|
-
}
|
|
64
|
-
catch (err) {
|
|
65
|
-
console.error('[BullMQ] Error closing connection:', err);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
async getSubscriptionCount(topic) {
|
|
69
|
-
return this._workerManager.getSubscriptionCount(topic);
|
|
70
|
-
}
|
|
71
|
-
async listTopics() {
|
|
72
|
-
return this._workerManager.listTopics();
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
exports.BullMQEventAdapter = BullMQEventAdapter;
|
package/dist/config-builder.d.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { BullMQEventAdapterConfig } from './types';
|
|
2
|
-
export type MergedConfig = Required<Pick<BullMQEventAdapterConfig, 'defaultJobOptions' | 'prefix' | 'concurrency'>> & {
|
|
3
|
-
dlq: Required<NonNullable<BullMQEventAdapterConfig['dlq']>>;
|
|
4
|
-
};
|
|
5
|
-
export declare function buildConfig(config: BullMQEventAdapterConfig): MergedConfig;
|
|
6
|
-
//# sourceMappingURL=config-builder.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config-builder.d.ts","sourceRoot":"","sources":["../src/config-builder.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAA;AAEvD,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,wBAAwB,EAAE,mBAAmB,GAAG,QAAQ,GAAG,aAAa,CAAC,CAAC,GAAG;IACpH,GAAG,EAAE,QAAQ,CAAC,WAAW,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;CAC5D,CAAA;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,wBAAwB,GAAG,YAAY,CAwB1E"}
|
package/dist/config-builder.js
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.buildConfig = buildConfig;
|
|
4
|
-
const constants_1 = require("./constants");
|
|
5
|
-
function buildConfig(config) {
|
|
6
|
-
return {
|
|
7
|
-
concurrency: config.concurrency ?? constants_1.DEFAULT_CONCURRENCY,
|
|
8
|
-
defaultJobOptions: {
|
|
9
|
-
attempts: constants_1.DEFAULT_ATTEMPTS,
|
|
10
|
-
backoff: {
|
|
11
|
-
type: 'fixed',
|
|
12
|
-
delay: constants_1.DEFAULT_BACKOFF_DELAY,
|
|
13
|
-
},
|
|
14
|
-
removeOnComplete: {
|
|
15
|
-
count: constants_1.DEFAULT_REMOVE_ON_COMPLETE_COUNT,
|
|
16
|
-
},
|
|
17
|
-
removeOnFail: {
|
|
18
|
-
count: constants_1.DEFAULT_REMOVE_ON_FAIL_COUNT,
|
|
19
|
-
},
|
|
20
|
-
...config.defaultJobOptions,
|
|
21
|
-
},
|
|
22
|
-
prefix: config.prefix ?? constants_1.DEFAULT_PREFIX,
|
|
23
|
-
dlq: {
|
|
24
|
-
enabled: config.dlq?.enabled ?? true,
|
|
25
|
-
ttl: config.dlq?.ttl ?? constants_1.DEFAULT_DLQ_TTL,
|
|
26
|
-
suffix: config.dlq?.suffix ?? constants_1.DEFAULT_DLQ_SUFFIX,
|
|
27
|
-
},
|
|
28
|
-
};
|
|
29
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import { type Redis } from 'ioredis';
|
|
2
|
-
import type { BullMQConnectionConfig } from './types';
|
|
3
|
-
export declare class ConnectionManager {
|
|
4
|
-
readonly connection: Redis;
|
|
5
|
-
readonly ownsConnection: boolean;
|
|
6
|
-
constructor(config: BullMQConnectionConfig);
|
|
7
|
-
private setupEventHandlers;
|
|
8
|
-
close(): Promise<void>;
|
|
9
|
-
}
|
|
10
|
-
//# sourceMappingURL=connection-manager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../src/connection-manager.ts"],"names":[],"mappings":"AAAA,OAAgB,EAAE,KAAK,KAAK,EAAE,MAAM,SAAS,CAAA;AAE7C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAA;AAErD,qBAAa,iBAAiB;IAC5B,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAA;IAC1B,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;gBAEpB,MAAM,EAAE,sBAAsB;IAe1C,OAAO,CAAC,kBAAkB;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.ConnectionManager = void 0;
|
|
7
|
-
const ioredis_1 = __importDefault(require("ioredis"));
|
|
8
|
-
const errors_1 = require("./errors");
|
|
9
|
-
class ConnectionManager {
|
|
10
|
-
constructor(config) {
|
|
11
|
-
if (config instanceof ioredis_1.default) {
|
|
12
|
-
this.connection = config;
|
|
13
|
-
this.ownsConnection = false;
|
|
14
|
-
}
|
|
15
|
-
else {
|
|
16
|
-
this.connection = new ioredis_1.default({
|
|
17
|
-
maxRetriesPerRequest: null,
|
|
18
|
-
...config,
|
|
19
|
-
});
|
|
20
|
-
this.ownsConnection = true;
|
|
21
|
-
}
|
|
22
|
-
this.setupEventHandlers();
|
|
23
|
-
}
|
|
24
|
-
setupEventHandlers() {
|
|
25
|
-
this.connection.on('error', (err) => {
|
|
26
|
-
const error = new errors_1.ConnectionError(err.message, err);
|
|
27
|
-
console.error('[BullMQ] Connection error:', error);
|
|
28
|
-
});
|
|
29
|
-
this.connection.on('close', () => {
|
|
30
|
-
console.warn('[BullMQ] Connection closed');
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
async close() {
|
|
34
|
-
if (this.ownsConnection && this.connection.status !== 'end') {
|
|
35
|
-
await this.connection.quit();
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
exports.ConnectionManager = ConnectionManager;
|
package/dist/constants.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export declare const DEFAULT_CONCURRENCY = 5;
|
|
2
|
-
export declare const DEFAULT_ATTEMPTS = 3;
|
|
3
|
-
export declare const DEFAULT_BACKOFF_DELAY = 2000;
|
|
4
|
-
export declare const DEFAULT_REMOVE_ON_COMPLETE_COUNT = 1000;
|
|
5
|
-
export declare const DEFAULT_REMOVE_ON_FAIL_COUNT = 5000;
|
|
6
|
-
export declare const DEFAULT_PREFIX = "motia";
|
|
7
|
-
export declare const FIFO_CONCURRENCY = 1;
|
|
8
|
-
export declare const MILLISECONDS_PER_SECOND = 1000;
|
|
9
|
-
export declare const SECONDS_PER_DAY = 86400;
|
|
10
|
-
export declare const DEFAULT_DLQ_TTL: number;
|
|
11
|
-
export declare const DEFAULT_DLQ_SUFFIX = ".dlq";
|
|
12
|
-
export declare const DLQ_JOB_PREFIX = "dlq-";
|
|
13
|
-
export declare const LOG_PREFIX = "[BullMQ]";
|
|
14
|
-
//# sourceMappingURL=constants.d.ts.map
|
package/dist/constants.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,IAAI,CAAA;AACpC,eAAO,MAAM,gBAAgB,IAAI,CAAA;AACjC,eAAO,MAAM,qBAAqB,OAAO,CAAA;AACzC,eAAO,MAAM,gCAAgC,OAAO,CAAA;AACpD,eAAO,MAAM,4BAA4B,OAAO,CAAA;AAChD,eAAO,MAAM,cAAc,UAAU,CAAA;AACrC,eAAO,MAAM,gBAAgB,IAAI,CAAA;AACjC,eAAO,MAAM,uBAAuB,OAAO,CAAA;AAC3C,eAAO,MAAM,eAAe,QAAQ,CAAA;AACpC,eAAO,MAAM,eAAe,QAAuB,CAAA;AACnD,eAAO,MAAM,kBAAkB,SAAS,CAAA;AACxC,eAAO,MAAM,cAAc,SAAS,CAAA;AAEpC,eAAO,MAAM,UAAU,aAAa,CAAA"}
|
package/dist/constants.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.LOG_PREFIX = exports.DLQ_JOB_PREFIX = exports.DEFAULT_DLQ_SUFFIX = exports.DEFAULT_DLQ_TTL = exports.SECONDS_PER_DAY = exports.MILLISECONDS_PER_SECOND = exports.FIFO_CONCURRENCY = exports.DEFAULT_PREFIX = exports.DEFAULT_REMOVE_ON_FAIL_COUNT = exports.DEFAULT_REMOVE_ON_COMPLETE_COUNT = exports.DEFAULT_BACKOFF_DELAY = exports.DEFAULT_ATTEMPTS = exports.DEFAULT_CONCURRENCY = void 0;
|
|
4
|
-
exports.DEFAULT_CONCURRENCY = 5;
|
|
5
|
-
exports.DEFAULT_ATTEMPTS = 3;
|
|
6
|
-
exports.DEFAULT_BACKOFF_DELAY = 2000;
|
|
7
|
-
exports.DEFAULT_REMOVE_ON_COMPLETE_COUNT = 1000;
|
|
8
|
-
exports.DEFAULT_REMOVE_ON_FAIL_COUNT = 5000;
|
|
9
|
-
exports.DEFAULT_PREFIX = 'motia';
|
|
10
|
-
exports.FIFO_CONCURRENCY = 1;
|
|
11
|
-
exports.MILLISECONDS_PER_SECOND = 1000;
|
|
12
|
-
exports.SECONDS_PER_DAY = 86400;
|
|
13
|
-
exports.DEFAULT_DLQ_TTL = 30 * exports.SECONDS_PER_DAY;
|
|
14
|
-
exports.DEFAULT_DLQ_SUFFIX = '.dlq';
|
|
15
|
-
exports.DLQ_JOB_PREFIX = 'dlq-';
|
|
16
|
-
exports.LOG_PREFIX = '[BullMQ]';
|
package/dist/dlq-manager.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { Event } from '@motiadev/core';
|
|
2
|
-
import { Queue } from 'bullmq';
|
|
3
|
-
import type { Redis } from 'ioredis';
|
|
4
|
-
import type { MergedConfig } from './config-builder';
|
|
5
|
-
export declare class DLQManager {
|
|
6
|
-
private readonly dlqQueues;
|
|
7
|
-
private readonly connection;
|
|
8
|
-
private readonly config;
|
|
9
|
-
constructor(connection: Redis, config: MergedConfig);
|
|
10
|
-
getDLQQueueName(topic: string, stepName: string): string;
|
|
11
|
-
private getOrCreateDLQQueue;
|
|
12
|
-
moveToDLQ<TData>(topic: string, stepName: string, event: Event<TData>, error: Error, attemptsMade: number, originalJobId?: string): Promise<void>;
|
|
13
|
-
closeDLQQueue(queueName: string): Promise<void>;
|
|
14
|
-
closeAll(): Promise<void>;
|
|
15
|
-
getDLQQueue(queueName: string): Queue | undefined;
|
|
16
|
-
getOrCreateDLQ(queueName: string): Queue;
|
|
17
|
-
listDLQQueueNames(): string[];
|
|
18
|
-
getDLQSuffix(): string;
|
|
19
|
-
getPrefix(): string;
|
|
20
|
-
getConnection(): Redis;
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=dlq-manager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dlq-manager.d.ts","sourceRoot":"","sources":["../src/dlq-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAC9B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAYpD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;gBAEzB,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAKnD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKxD,OAAO,CAAC,mBAAmB;IA8BrB,SAAS,CAAC,KAAK,EACnB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EACnB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE,MAAM,EACpB,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC;IA0CV,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;IAIjD,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK;IAIxC,iBAAiB,IAAI,MAAM,EAAE;IAI7B,YAAY,IAAI,MAAM;IAItB,SAAS,IAAI,MAAM;IAInB,aAAa,IAAI,KAAK;CAGvB"}
|
package/dist/dlq-manager.js
DELETED
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DLQManager = void 0;
|
|
4
|
-
const bullmq_1 = require("bullmq");
|
|
5
|
-
const constants_1 = require("./constants");
|
|
6
|
-
const errors_1 = require("./errors");
|
|
7
|
-
class DLQManager {
|
|
8
|
-
constructor(connection, config) {
|
|
9
|
-
this.dlqQueues = new Map();
|
|
10
|
-
this.connection = connection;
|
|
11
|
-
this.config = config;
|
|
12
|
-
}
|
|
13
|
-
getDLQQueueName(topic, stepName) {
|
|
14
|
-
const baseQueueName = `${topic}.${stepName}`;
|
|
15
|
-
return `${baseQueueName}${this.config.dlq.suffix}`;
|
|
16
|
-
}
|
|
17
|
-
getOrCreateDLQQueue(queueName) {
|
|
18
|
-
if (!this.dlqQueues.has(queueName)) {
|
|
19
|
-
const ttlMs = this.config.dlq.ttl * constants_1.MILLISECONDS_PER_SECOND;
|
|
20
|
-
const queue = new bullmq_1.Queue(queueName, {
|
|
21
|
-
connection: this.connection,
|
|
22
|
-
prefix: this.config.prefix,
|
|
23
|
-
defaultJobOptions: {
|
|
24
|
-
removeOnComplete: {
|
|
25
|
-
age: ttlMs,
|
|
26
|
-
},
|
|
27
|
-
removeOnFail: {
|
|
28
|
-
age: ttlMs,
|
|
29
|
-
},
|
|
30
|
-
},
|
|
31
|
-
});
|
|
32
|
-
queue.on('error', (err) => {
|
|
33
|
-
console.error(`${constants_1.LOG_PREFIX} DLQ error for ${queueName}:`, err);
|
|
34
|
-
});
|
|
35
|
-
this.dlqQueues.set(queueName, queue);
|
|
36
|
-
}
|
|
37
|
-
const queue = this.dlqQueues.get(queueName);
|
|
38
|
-
if (!queue) {
|
|
39
|
-
throw new errors_1.QueueCreationError(queueName);
|
|
40
|
-
}
|
|
41
|
-
return queue;
|
|
42
|
-
}
|
|
43
|
-
async moveToDLQ(topic, stepName, event, error, attemptsMade, originalJobId) {
|
|
44
|
-
if (!this.config.dlq.enabled) {
|
|
45
|
-
console.warn(`${constants_1.LOG_PREFIX} DLQ is disabled, skipping move to DLQ for topic ${topic}, step ${stepName}`);
|
|
46
|
-
return;
|
|
47
|
-
}
|
|
48
|
-
try {
|
|
49
|
-
const dlqQueueName = this.getDLQQueueName(topic, stepName);
|
|
50
|
-
const dlqQueue = this.getOrCreateDLQQueue(dlqQueueName);
|
|
51
|
-
const sanitizedEvent = {
|
|
52
|
-
topic: event.topic,
|
|
53
|
-
data: event.data,
|
|
54
|
-
traceId: event.traceId || 'unknown',
|
|
55
|
-
...(event.flows && { flows: event.flows }),
|
|
56
|
-
...(event.messageGroupId && { messageGroupId: event.messageGroupId }),
|
|
57
|
-
};
|
|
58
|
-
const dlqJobData = {
|
|
59
|
-
originalEvent: sanitizedEvent,
|
|
60
|
-
failureReason: error.message || 'Unknown error',
|
|
61
|
-
failureTimestamp: Date.now(),
|
|
62
|
-
attemptsMade,
|
|
63
|
-
...(originalJobId && { originalJobId }),
|
|
64
|
-
};
|
|
65
|
-
const jobOptions = originalJobId ? { jobId: `${constants_1.DLQ_JOB_PREFIX}${originalJobId}` } : {};
|
|
66
|
-
await dlqQueue.add(`${topic}.dlq`, dlqJobData, jobOptions);
|
|
67
|
-
console.warn(`${constants_1.LOG_PREFIX} Moved failed job to DLQ: ${dlqQueueName}`, {
|
|
68
|
-
topic,
|
|
69
|
-
stepName,
|
|
70
|
-
attemptsMade,
|
|
71
|
-
error: error.message,
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
catch (err) {
|
|
75
|
-
const dlqError = err instanceof Error ? err : new Error(String(err));
|
|
76
|
-
console.error(`${constants_1.LOG_PREFIX} Failed to move job to DLQ for topic ${topic}, step ${stepName}:`, dlqError);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
async closeDLQQueue(queueName) {
|
|
80
|
-
const queue = this.dlqQueues.get(queueName);
|
|
81
|
-
if (queue) {
|
|
82
|
-
await queue.close();
|
|
83
|
-
this.dlqQueues.delete(queueName);
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
async closeAll() {
|
|
87
|
-
const promises = Array.from(this.dlqQueues.values()).map((queue) => queue.close().catch((err) => {
|
|
88
|
-
console.error(`${constants_1.LOG_PREFIX} Error closing DLQ queue:`, err);
|
|
89
|
-
}));
|
|
90
|
-
await Promise.allSettled(promises);
|
|
91
|
-
this.dlqQueues.clear();
|
|
92
|
-
}
|
|
93
|
-
getDLQQueue(queueName) {
|
|
94
|
-
return this.dlqQueues.get(queueName);
|
|
95
|
-
}
|
|
96
|
-
getOrCreateDLQ(queueName) {
|
|
97
|
-
return this.getOrCreateDLQQueue(queueName);
|
|
98
|
-
}
|
|
99
|
-
listDLQQueueNames() {
|
|
100
|
-
return Array.from(this.dlqQueues.keys());
|
|
101
|
-
}
|
|
102
|
-
getDLQSuffix() {
|
|
103
|
-
return this.config.dlq.suffix;
|
|
104
|
-
}
|
|
105
|
-
getPrefix() {
|
|
106
|
-
return this.config.prefix;
|
|
107
|
-
}
|
|
108
|
-
getConnection() {
|
|
109
|
-
return this.connection;
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
exports.DLQManager = DLQManager;
|
package/dist/errors.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export declare class BullMQAdapterError extends Error {
|
|
2
|
-
readonly cause?: Error | undefined;
|
|
3
|
-
constructor(message: string, cause?: Error | undefined);
|
|
4
|
-
}
|
|
5
|
-
export declare class QueueCreationError extends BullMQAdapterError {
|
|
6
|
-
constructor(queueName: string, cause?: Error);
|
|
7
|
-
}
|
|
8
|
-
export declare class WorkerCreationError extends BullMQAdapterError {
|
|
9
|
-
constructor(topic: string, stepName: string, cause?: Error);
|
|
10
|
-
}
|
|
11
|
-
export declare class ConnectionError extends BullMQAdapterError {
|
|
12
|
-
constructor(message: string, cause?: Error);
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=errors.d.ts.map
|
package/dist/errors.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qBAAa,kBAAmB,SAAQ,KAAK;aAGzB,KAAK,CAAC,EAAE,KAAK;gBAD7B,OAAO,EAAE,MAAM,EACC,KAAK,CAAC,EAAE,KAAK,YAAA;CAQhC;AAED,qBAAa,kBAAmB,SAAQ,kBAAkB;gBAC5C,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI7C;AAED,qBAAa,mBAAoB,SAAQ,kBAAkB;gBAC7C,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3D;AAED,qBAAa,eAAgB,SAAQ,kBAAkB;gBACzC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C"}
|