@motiadev/adapter-bullmq-events 0.13.0-beta.162-717198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +93 -0
- package/README.md +222 -0
- package/dist/bullmq-event-adapter.d.ts +27 -0
- package/dist/bullmq-event-adapter.d.ts.map +1 -0
- package/dist/bullmq-event-adapter.js +75 -0
- package/dist/config-builder.d.ts +6 -0
- package/dist/config-builder.d.ts.map +1 -0
- package/dist/config-builder.js +29 -0
- package/dist/connection-manager.d.ts +10 -0
- package/dist/connection-manager.d.ts.map +1 -0
- package/dist/connection-manager.js +39 -0
- package/dist/constants.d.ts +14 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +16 -0
- package/dist/dlq-manager.d.ts +22 -0
- package/dist/dlq-manager.d.ts.map +1 -0
- package/dist/dlq-manager.js +112 -0
- package/dist/errors.d.ts +14 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +35 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/queue-manager.d.ts +20 -0
- package/dist/queue-manager.d.ts.map +1 -0
- package/dist/queue-manager.js +85 -0
- package/dist/types.d.ts +23 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/worker-manager.d.ts +38 -0
- package/dist/worker-manager.d.ts.map +1 -0
- package/dist/worker-manager.js +136 -0
- package/package.json +25 -0
- package/src/bullmq-event-adapter.ts +105 -0
- package/src/config-builder.ts +41 -0
- package/src/connection-manager.ts +40 -0
- package/src/constants.ts +14 -0
- package/src/dlq-manager.ts +151 -0
- package/src/errors.ts +33 -0
- package/src/index.ts +6 -0
- package/src/queue-manager.ts +107 -0
- package/src/types.ts +24 -0
- package/src/worker-manager.ts +200 -0
- package/tsconfig.json +19 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Elastic License 2.0
|
|
2
|
+
|
|
3
|
+
URL: https://www.elastic.co/licensing/elastic-license
|
|
4
|
+
|
|
5
|
+
## Acceptance
|
|
6
|
+
|
|
7
|
+
By using the software, you agree to all of the terms and conditions below.
|
|
8
|
+
|
|
9
|
+
## Copyright License
|
|
10
|
+
|
|
11
|
+
The licensor grants you a non-exclusive, royalty-free, worldwide,
|
|
12
|
+
non-sublicensable, non-transferable license to use, copy, distribute, make
|
|
13
|
+
available, and prepare derivative works of the software, in each case subject
|
|
14
|
+
to the limitations and conditions below.
|
|
15
|
+
|
|
16
|
+
## Limitations
|
|
17
|
+
|
|
18
|
+
You may not provide the software to third parties as a hosted or managed
|
|
19
|
+
service, where the service provides users with access to any substantial set of
|
|
20
|
+
the features or functionality of the software.
|
|
21
|
+
|
|
22
|
+
You may not move, change, disable, or circumvent the license key functionality
|
|
23
|
+
in the software, and you may not remove or obscure any functionality in the
|
|
24
|
+
software that is protected by the license key.
|
|
25
|
+
|
|
26
|
+
You may not alter, remove, or obscure any licensing, copyright, or other notices
|
|
27
|
+
of the licensor in the software. Any use of the licensor's trademarks is subject
|
|
28
|
+
to applicable law.
|
|
29
|
+
|
|
30
|
+
## Patents
|
|
31
|
+
|
|
32
|
+
The licensor grants you a license, under any patent claims the licensor can
|
|
33
|
+
license, or becomes able to license, to make, have made, use, sell, offer for
|
|
34
|
+
sale, import and have imported the software, in each case subject to the
|
|
35
|
+
limitations and conditions in this license. This license does not cover any
|
|
36
|
+
patent claims that you cause to be infringed by modifications or additions to
|
|
37
|
+
the software. If you or your company make any written claim that the software
|
|
38
|
+
infringes or contributes to infringement of any patent, your patent license for
|
|
39
|
+
the software granted under these terms ends immediately. If your company makes
|
|
40
|
+
such a claim, your patent license ends immediately for work on behalf of your
|
|
41
|
+
company.
|
|
42
|
+
|
|
43
|
+
## Notices
|
|
44
|
+
|
|
45
|
+
You must ensure that anyone who gets a copy of any part of the software from you
|
|
46
|
+
also gets a copy of these terms.
|
|
47
|
+
|
|
48
|
+
If you modify the software, you must include in any modified copies of the
|
|
49
|
+
software prominent notices stating that you have modified the software.
|
|
50
|
+
|
|
51
|
+
## No Other Rights
|
|
52
|
+
|
|
53
|
+
These terms do not imply any licenses other than those expressly granted in
|
|
54
|
+
these terms.
|
|
55
|
+
|
|
56
|
+
## Termination
|
|
57
|
+
|
|
58
|
+
If you use the software in violation of these terms, such use is not licensed,
|
|
59
|
+
and your licenses will automatically terminate. If the licensor provides you
|
|
60
|
+
with a notice of your violation, and you cease all violation of this license no
|
|
61
|
+
later than 30 days after you receive that notice, your licenses will be
|
|
62
|
+
reinstated retroactively. However, if you violate these terms after such
|
|
63
|
+
reinstatement, any additional violation of these terms will cause your licenses
|
|
64
|
+
to terminate automatically and permanently.
|
|
65
|
+
|
|
66
|
+
## No Liability
|
|
67
|
+
|
|
68
|
+
*As far as the law allows, the software comes as is, without any warranty or
|
|
69
|
+
condition, and the licensor will not be liable to you for any damages arising
|
|
70
|
+
out of these terms or the use or nature of the software, under any kind of
|
|
71
|
+
legal claim.*
|
|
72
|
+
|
|
73
|
+
## Definitions
|
|
74
|
+
|
|
75
|
+
The **licensor** is the entity offering these terms, and the **software** is the
|
|
76
|
+
software the licensor makes available under these terms, including any portion
|
|
77
|
+
of it.
|
|
78
|
+
|
|
79
|
+
**you** refers to the individual or entity agreeing to these terms.
|
|
80
|
+
|
|
81
|
+
**your company** is any legal entity, sole proprietorship, or other kind of
|
|
82
|
+
organization that you work for, plus all organizations that have control over,
|
|
83
|
+
are under the control of, or are under common control with that organization.
|
|
84
|
+
**control** means ownership of substantially all the assets of an entity, or the
|
|
85
|
+
power to direct its management and policies by vote, contract, or otherwise.
|
|
86
|
+
Control can be direct or indirect.
|
|
87
|
+
|
|
88
|
+
**your licenses** are all the licenses granted to you for the software under
|
|
89
|
+
these terms.
|
|
90
|
+
|
|
91
|
+
**use** means anything you do with the software requiring one of your licenses.
|
|
92
|
+
|
|
93
|
+
**trademark** means trademarks, service marks, and similar rights.
|
package/README.md
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# @motiadev/adapter-bullmq-events
|
|
2
|
+
|
|
3
|
+
BullMQ event adapter for Motia framework, enabling distributed event handling with advanced features like retries, priorities, delays, and FIFO queues.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @motiadev/adapter-bullmq-events
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
Configure the BullMQ adapter in your `motia.config.ts`:
|
|
14
|
+
|
|
15
|
+
```typescript
|
|
16
|
+
import { config } from '@motiadev/core'
|
|
17
|
+
import { BullMQEventAdapter } from '@motiadev/adapter-bullmq-events'
|
|
18
|
+
|
|
19
|
+
export default config({
|
|
20
|
+
adapters: {
|
|
21
|
+
events: new BullMQEventAdapter({
|
|
22
|
+
connection: {
|
|
23
|
+
host: process.env.REDIS_HOST || 'localhost',
|
|
24
|
+
port: 6379,
|
|
25
|
+
password: process.env.REDIS_PASSWORD,
|
|
26
|
+
},
|
|
27
|
+
}),
|
|
28
|
+
},
|
|
29
|
+
})
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Using an Existing Redis Connection
|
|
33
|
+
|
|
34
|
+
You can also pass an existing IORedis instance:
|
|
35
|
+
|
|
36
|
+
```typescript
|
|
37
|
+
import IORedis from 'ioredis'
|
|
38
|
+
import { BullMQEventAdapter } from '@motiadev/adapter-bullmq-events'
|
|
39
|
+
|
|
40
|
+
const redis = new IORedis({
|
|
41
|
+
host: 'localhost',
|
|
42
|
+
port: 6379,
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
const adapter = new BullMQEventAdapter({
|
|
46
|
+
connection: redis,
|
|
47
|
+
})
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Configuration Options
|
|
51
|
+
|
|
52
|
+
### BullMQEventAdapterConfig
|
|
53
|
+
|
|
54
|
+
| Option | Type | Default | Description |
|
|
55
|
+
|--------|------|---------|-------------|
|
|
56
|
+
| `connection` | `IORedis.Redis \| IORedis.RedisOptions` | Required | Redis connection instance or configuration |
|
|
57
|
+
| `prefix` | `string` | `'motia'` | Prefix for BullMQ queue keys |
|
|
58
|
+
| `defaultJobOptions` | `object` | See below | Default options for all jobs |
|
|
59
|
+
|
|
60
|
+
### Default Job Options
|
|
61
|
+
|
|
62
|
+
| Option | Type | Default | Description |
|
|
63
|
+
|--------|------|---------|-------------|
|
|
64
|
+
| `attempts` | `number` | `3` | Number of retry attempts |
|
|
65
|
+
| `backoff` | `object` | `{ type: 'exponential', delay: 2000 }` | Backoff strategy for retries |
|
|
66
|
+
| `removeOnComplete` | `boolean \| number \| object` | `{ count: 1000 }` | Remove completed jobs after N jobs or age |
|
|
67
|
+
| `removeOnFail` | `boolean \| number \| object` | `{ count: 5000 }` | Remove failed jobs after N jobs or age |
|
|
68
|
+
|
|
69
|
+
### Backoff Types
|
|
70
|
+
|
|
71
|
+
- `fixed`: Wait a fixed delay between retries
|
|
72
|
+
- `exponential`: Exponentially increase delay between retries (2s, 4s, 8s, etc.)
|
|
73
|
+
|
|
74
|
+
## Features
|
|
75
|
+
|
|
76
|
+
- **Queue/Worker Pattern**: Uses BullMQ's robust job queue system
|
|
77
|
+
- **Automatic Retries**: Configurable retry attempts with backoff strategies
|
|
78
|
+
- **FIFO Queues**: Support for FIFO processing via `concurrency: 1`
|
|
79
|
+
- **Priority Support**: Jobs can have priorities (configured via defaultJobOptions)
|
|
80
|
+
- **Delayed Jobs**: Support for delayed job execution
|
|
81
|
+
- **Visibility Timeout**: Configurable lock duration for job processing
|
|
82
|
+
- **Connection Management**: Accepts existing Redis connections or creates new ones
|
|
83
|
+
- **Graceful Shutdown**: Properly closes all queues and workers on shutdown
|
|
84
|
+
|
|
85
|
+
## Advanced Configuration
|
|
86
|
+
|
|
87
|
+
### Custom Retry Strategy
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
const adapter = new BullMQEventAdapter({
|
|
91
|
+
connection: {
|
|
92
|
+
host: 'localhost',
|
|
93
|
+
port: 6379,
|
|
94
|
+
},
|
|
95
|
+
defaultJobOptions: {
|
|
96
|
+
attempts: 5,
|
|
97
|
+
backoff: {
|
|
98
|
+
type: 'exponential',
|
|
99
|
+
delay: 3000,
|
|
100
|
+
},
|
|
101
|
+
removeOnComplete: {
|
|
102
|
+
count: 500,
|
|
103
|
+
age: 3600,
|
|
104
|
+
},
|
|
105
|
+
removeOnFail: {
|
|
106
|
+
count: 1000,
|
|
107
|
+
age: 86400,
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
})
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### FIFO Queue Support
|
|
114
|
+
|
|
115
|
+
FIFO queues are automatically enabled when `options.type === 'fifo'` is passed to `subscribe()`:
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
await adapter.subscribe(
|
|
119
|
+
'order.processed',
|
|
120
|
+
'processOrder',
|
|
121
|
+
async (event) => {
|
|
122
|
+
// Process order sequentially
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
type: 'fifo',
|
|
126
|
+
maxRetries: 3,
|
|
127
|
+
visibilityTimeout: 30,
|
|
128
|
+
}
|
|
129
|
+
)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Example
|
|
133
|
+
|
|
134
|
+
```typescript
|
|
135
|
+
import { BullMQEventAdapter } from '@motiadev/adapter-bullmq-events'
|
|
136
|
+
|
|
137
|
+
const adapter = new BullMQEventAdapter({
|
|
138
|
+
connection: {
|
|
139
|
+
host: 'redis.example.com',
|
|
140
|
+
port: 6379,
|
|
141
|
+
password: 'your-password',
|
|
142
|
+
db: 0,
|
|
143
|
+
},
|
|
144
|
+
prefix: 'myapp',
|
|
145
|
+
defaultJobOptions: {
|
|
146
|
+
attempts: 3,
|
|
147
|
+
backoff: {
|
|
148
|
+
type: 'exponential',
|
|
149
|
+
delay: 2000,
|
|
150
|
+
},
|
|
151
|
+
},
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
await adapter.emit({
|
|
155
|
+
topic: 'user.created',
|
|
156
|
+
data: { userId: '123', email: 'user@example.com' },
|
|
157
|
+
traceId: 'trace-123',
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
await adapter.subscribe(
|
|
161
|
+
'user.created',
|
|
162
|
+
'sendWelcomeEmail',
|
|
163
|
+
async (event) => {
|
|
164
|
+
console.log('Processing user creation:', event.data)
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
type: 'standard',
|
|
168
|
+
maxRetries: 5,
|
|
169
|
+
visibilityTimeout: 60,
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
## Environment Variables
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
REDIS_HOST=localhost
|
|
178
|
+
REDIS_PORT=6379
|
|
179
|
+
REDIS_PASSWORD=your-password
|
|
180
|
+
REDIS_DB=0
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## Performance Considerations
|
|
184
|
+
|
|
185
|
+
- Use appropriate `concurrency` settings for your workload
|
|
186
|
+
- Configure `removeOnComplete` and `removeOnFail` to prevent Redis memory bloat
|
|
187
|
+
- Monitor queue depths using BullMQ dashboard or Redis commands
|
|
188
|
+
- Use connection pooling when creating multiple adapters
|
|
189
|
+
- Set appropriate `lockDuration` (visibility timeout) based on job processing time
|
|
190
|
+
|
|
191
|
+
## Troubleshooting
|
|
192
|
+
|
|
193
|
+
### Connection Issues
|
|
194
|
+
|
|
195
|
+
If you experience connection problems:
|
|
196
|
+
1. Verify Redis is running and accessible
|
|
197
|
+
2. Check your connection configuration and credentials
|
|
198
|
+
3. Ensure firewall rules allow connections on Redis port (default 6379)
|
|
199
|
+
4. Review Redis logs for errors
|
|
200
|
+
|
|
201
|
+
### Job Processing
|
|
202
|
+
|
|
203
|
+
If jobs are not being processed:
|
|
204
|
+
1. Verify workers are subscribed to the correct topics
|
|
205
|
+
2. Check job failure logs in Redis
|
|
206
|
+
3. Review retry and backoff configurations
|
|
207
|
+
4. Monitor queue metrics using BullMQ dashboard
|
|
208
|
+
|
|
209
|
+
### Memory Issues
|
|
210
|
+
|
|
211
|
+
If Redis memory usage is high:
|
|
212
|
+
1. Configure `removeOnComplete` and `removeOnFail` appropriately
|
|
213
|
+
2. Use job age limits instead of count limits for long-running queues
|
|
214
|
+
3. Monitor Redis memory usage and set up eviction policies
|
|
215
|
+
|
|
216
|
+
## License
|
|
217
|
+
|
|
218
|
+
MIT
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Event, EventAdapter, QueueConfig, SubscriptionHandle } from '@motiadev/core';
|
|
2
|
+
import type { Redis } from 'ioredis';
|
|
3
|
+
import { DLQManager } from './dlq-manager';
|
|
4
|
+
import { QueueManager } from './queue-manager';
|
|
5
|
+
import type { BullMQEventAdapterConfig } from './types';
|
|
6
|
+
import { WorkerManager } from './worker-manager';
|
|
7
|
+
export declare class BullMQEventAdapter implements EventAdapter {
|
|
8
|
+
private readonly connectionManager;
|
|
9
|
+
private readonly _queueManager;
|
|
10
|
+
private readonly _workerManager;
|
|
11
|
+
private readonly _dlqManager;
|
|
12
|
+
private readonly _config;
|
|
13
|
+
constructor(config: BullMQEventAdapterConfig);
|
|
14
|
+
get connection(): Redis;
|
|
15
|
+
get prefix(): string;
|
|
16
|
+
get dlqSuffix(): string;
|
|
17
|
+
get queueManager(): QueueManager;
|
|
18
|
+
get workerManager(): WorkerManager;
|
|
19
|
+
get dlqManager(): DLQManager;
|
|
20
|
+
emit<TData>(event: Event<TData>): Promise<void>;
|
|
21
|
+
subscribe<TData>(topic: string, stepName: string, handler: (event: Event<TData>) => void | Promise<void>, options?: QueueConfig): Promise<SubscriptionHandle>;
|
|
22
|
+
unsubscribe(handle: SubscriptionHandle): Promise<void>;
|
|
23
|
+
shutdown(): Promise<void>;
|
|
24
|
+
getSubscriptionCount(topic: string): Promise<number>;
|
|
25
|
+
listTopics(): Promise<string[]>;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=bullmq-event-adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bullmq-event-adapter.d.ts","sourceRoot":"","sources":["../src/bullmq-event-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAC1F,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGpC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAA;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAEhD,qBAAa,kBAAmB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IACrD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAc;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAe;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAY;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;gBAE1B,MAAM,EAAE,wBAAwB;IAa5C,IAAI,UAAU,IAAI,KAAK,CAEtB;IAED,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAEK,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/C,SAAS,CAAC,KAAK,EACnB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EACtD,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,kBAAkB,CAAC;IAOxB,WAAW,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAczB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIpD,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGtC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BullMQEventAdapter = void 0;
|
|
4
|
+
const config_builder_1 = require("./config-builder");
|
|
5
|
+
const connection_manager_1 = require("./connection-manager");
|
|
6
|
+
const dlq_manager_1 = require("./dlq-manager");
|
|
7
|
+
const queue_manager_1 = require("./queue-manager");
|
|
8
|
+
const worker_manager_1 = require("./worker-manager");
|
|
9
|
+
class BullMQEventAdapter {
|
|
10
|
+
constructor(config) {
|
|
11
|
+
this._config = (0, config_builder_1.buildConfig)(config);
|
|
12
|
+
this.connectionManager = new connection_manager_1.ConnectionManager(config.connection);
|
|
13
|
+
this._queueManager = new queue_manager_1.QueueManager(this.connectionManager.connection, this._config);
|
|
14
|
+
this._dlqManager = new dlq_manager_1.DLQManager(this.connectionManager.connection, this._config);
|
|
15
|
+
this._workerManager = new worker_manager_1.WorkerManager(this.connectionManager.connection, this._config, (topic, stepName) => this._queueManager.getQueueName(topic, stepName), this._dlqManager);
|
|
16
|
+
}
|
|
17
|
+
get connection() {
|
|
18
|
+
return this.connectionManager.connection;
|
|
19
|
+
}
|
|
20
|
+
get prefix() {
|
|
21
|
+
return this._config.prefix;
|
|
22
|
+
}
|
|
23
|
+
get dlqSuffix() {
|
|
24
|
+
return this._config.dlq.suffix;
|
|
25
|
+
}
|
|
26
|
+
get queueManager() {
|
|
27
|
+
return this._queueManager;
|
|
28
|
+
}
|
|
29
|
+
get workerManager() {
|
|
30
|
+
return this._workerManager;
|
|
31
|
+
}
|
|
32
|
+
get dlqManager() {
|
|
33
|
+
return this._dlqManager;
|
|
34
|
+
}
|
|
35
|
+
async emit(event) {
|
|
36
|
+
const subscribers = this._workerManager.getSubscribers(event.topic);
|
|
37
|
+
if (subscribers.length === 0) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
await this._queueManager.enqueueToAll(event, subscribers);
|
|
41
|
+
}
|
|
42
|
+
async subscribe(topic, stepName, handler, options) {
|
|
43
|
+
const queueName = this._queueManager.getQueueName(topic, stepName);
|
|
44
|
+
this._queueManager.getQueue(queueName);
|
|
45
|
+
return this._workerManager.createWorker(topic, stepName, handler, options);
|
|
46
|
+
}
|
|
47
|
+
async unsubscribe(handle) {
|
|
48
|
+
const workerInfo = this._workerManager.getWorkerInfo(handle.id);
|
|
49
|
+
if (workerInfo) {
|
|
50
|
+
const queueName = this._queueManager.getQueueName(workerInfo.topic, workerInfo.stepName);
|
|
51
|
+
await this._queueManager.closeQueue(queueName);
|
|
52
|
+
await this._workerManager.removeWorker(handle.id);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
async shutdown() {
|
|
56
|
+
await Promise.allSettled([
|
|
57
|
+
this._workerManager.closeAll(),
|
|
58
|
+
this._queueManager.closeAll(),
|
|
59
|
+
this._dlqManager.closeAll(),
|
|
60
|
+
]);
|
|
61
|
+
try {
|
|
62
|
+
await this.connectionManager.close();
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
console.error('[BullMQ] Error closing connection:', err);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async getSubscriptionCount(topic) {
|
|
69
|
+
return this._workerManager.getSubscriptionCount(topic);
|
|
70
|
+
}
|
|
71
|
+
async listTopics() {
|
|
72
|
+
return this._workerManager.listTopics();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.BullMQEventAdapter = BullMQEventAdapter;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { BullMQEventAdapterConfig } from './types';
|
|
2
|
+
export type MergedConfig = Required<Pick<BullMQEventAdapterConfig, 'defaultJobOptions' | 'prefix' | 'concurrency'>> & {
|
|
3
|
+
dlq: Required<NonNullable<BullMQEventAdapterConfig['dlq']>>;
|
|
4
|
+
};
|
|
5
|
+
export declare function buildConfig(config: BullMQEventAdapterConfig): MergedConfig;
|
|
6
|
+
//# sourceMappingURL=config-builder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-builder.d.ts","sourceRoot":"","sources":["../src/config-builder.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAA;AAEvD,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,wBAAwB,EAAE,mBAAmB,GAAG,QAAQ,GAAG,aAAa,CAAC,CAAC,GAAG;IACpH,GAAG,EAAE,QAAQ,CAAC,WAAW,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;CAC5D,CAAA;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,wBAAwB,GAAG,YAAY,CAwB1E"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildConfig = buildConfig;
|
|
4
|
+
const constants_1 = require("./constants");
|
|
5
|
+
function buildConfig(config) {
|
|
6
|
+
return {
|
|
7
|
+
concurrency: config.concurrency ?? constants_1.DEFAULT_CONCURRENCY,
|
|
8
|
+
defaultJobOptions: {
|
|
9
|
+
attempts: constants_1.DEFAULT_ATTEMPTS,
|
|
10
|
+
backoff: {
|
|
11
|
+
type: 'fixed',
|
|
12
|
+
delay: constants_1.DEFAULT_BACKOFF_DELAY,
|
|
13
|
+
},
|
|
14
|
+
removeOnComplete: {
|
|
15
|
+
count: constants_1.DEFAULT_REMOVE_ON_COMPLETE_COUNT,
|
|
16
|
+
},
|
|
17
|
+
removeOnFail: {
|
|
18
|
+
count: constants_1.DEFAULT_REMOVE_ON_FAIL_COUNT,
|
|
19
|
+
},
|
|
20
|
+
...config.defaultJobOptions,
|
|
21
|
+
},
|
|
22
|
+
prefix: config.prefix ?? constants_1.DEFAULT_PREFIX,
|
|
23
|
+
dlq: {
|
|
24
|
+
enabled: config.dlq?.enabled ?? true,
|
|
25
|
+
ttl: config.dlq?.ttl ?? constants_1.DEFAULT_DLQ_TTL,
|
|
26
|
+
suffix: config.dlq?.suffix ?? constants_1.DEFAULT_DLQ_SUFFIX,
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type Redis } from 'ioredis';
|
|
2
|
+
import type { BullMQConnectionConfig } from './types';
|
|
3
|
+
export declare class ConnectionManager {
|
|
4
|
+
readonly connection: Redis;
|
|
5
|
+
readonly ownsConnection: boolean;
|
|
6
|
+
constructor(config: BullMQConnectionConfig);
|
|
7
|
+
private setupEventHandlers;
|
|
8
|
+
close(): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=connection-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../src/connection-manager.ts"],"names":[],"mappings":"AAAA,OAAgB,EAAE,KAAK,KAAK,EAAE,MAAM,SAAS,CAAA;AAE7C,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAA;AAErD,qBAAa,iBAAiB;IAC5B,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAA;IAC1B,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAA;gBAEpB,MAAM,EAAE,sBAAsB;IAe1C,OAAO,CAAC,kBAAkB;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ConnectionManager = void 0;
|
|
7
|
+
const ioredis_1 = __importDefault(require("ioredis"));
|
|
8
|
+
const errors_1 = require("./errors");
|
|
9
|
+
class ConnectionManager {
|
|
10
|
+
constructor(config) {
|
|
11
|
+
if (config instanceof ioredis_1.default) {
|
|
12
|
+
this.connection = config;
|
|
13
|
+
this.ownsConnection = false;
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
this.connection = new ioredis_1.default({
|
|
17
|
+
maxRetriesPerRequest: null,
|
|
18
|
+
...config,
|
|
19
|
+
});
|
|
20
|
+
this.ownsConnection = true;
|
|
21
|
+
}
|
|
22
|
+
this.setupEventHandlers();
|
|
23
|
+
}
|
|
24
|
+
setupEventHandlers() {
|
|
25
|
+
this.connection.on('error', (err) => {
|
|
26
|
+
const error = new errors_1.ConnectionError(err.message, err);
|
|
27
|
+
console.error('[BullMQ] Connection error:', error);
|
|
28
|
+
});
|
|
29
|
+
this.connection.on('close', () => {
|
|
30
|
+
console.warn('[BullMQ] Connection closed');
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
async close() {
|
|
34
|
+
if (this.ownsConnection && this.connection.status !== 'end') {
|
|
35
|
+
await this.connection.quit();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
exports.ConnectionManager = ConnectionManager;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const DEFAULT_CONCURRENCY = 5;
|
|
2
|
+
export declare const DEFAULT_ATTEMPTS = 3;
|
|
3
|
+
export declare const DEFAULT_BACKOFF_DELAY = 2000;
|
|
4
|
+
export declare const DEFAULT_REMOVE_ON_COMPLETE_COUNT = 1000;
|
|
5
|
+
export declare const DEFAULT_REMOVE_ON_FAIL_COUNT = 5000;
|
|
6
|
+
export declare const DEFAULT_PREFIX = "motia";
|
|
7
|
+
export declare const FIFO_CONCURRENCY = 1;
|
|
8
|
+
export declare const MILLISECONDS_PER_SECOND = 1000;
|
|
9
|
+
export declare const SECONDS_PER_DAY = 86400;
|
|
10
|
+
export declare const DEFAULT_DLQ_TTL: number;
|
|
11
|
+
export declare const DEFAULT_DLQ_SUFFIX = ".dlq";
|
|
12
|
+
export declare const DLQ_JOB_PREFIX = "dlq-";
|
|
13
|
+
export declare const LOG_PREFIX = "[BullMQ]";
|
|
14
|
+
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,IAAI,CAAA;AACpC,eAAO,MAAM,gBAAgB,IAAI,CAAA;AACjC,eAAO,MAAM,qBAAqB,OAAO,CAAA;AACzC,eAAO,MAAM,gCAAgC,OAAO,CAAA;AACpD,eAAO,MAAM,4BAA4B,OAAO,CAAA;AAChD,eAAO,MAAM,cAAc,UAAU,CAAA;AACrC,eAAO,MAAM,gBAAgB,IAAI,CAAA;AACjC,eAAO,MAAM,uBAAuB,OAAO,CAAA;AAC3C,eAAO,MAAM,eAAe,QAAQ,CAAA;AACpC,eAAO,MAAM,eAAe,QAAuB,CAAA;AACnD,eAAO,MAAM,kBAAkB,SAAS,CAAA;AACxC,eAAO,MAAM,cAAc,SAAS,CAAA;AAEpC,eAAO,MAAM,UAAU,aAAa,CAAA"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LOG_PREFIX = exports.DLQ_JOB_PREFIX = exports.DEFAULT_DLQ_SUFFIX = exports.DEFAULT_DLQ_TTL = exports.SECONDS_PER_DAY = exports.MILLISECONDS_PER_SECOND = exports.FIFO_CONCURRENCY = exports.DEFAULT_PREFIX = exports.DEFAULT_REMOVE_ON_FAIL_COUNT = exports.DEFAULT_REMOVE_ON_COMPLETE_COUNT = exports.DEFAULT_BACKOFF_DELAY = exports.DEFAULT_ATTEMPTS = exports.DEFAULT_CONCURRENCY = void 0;
|
|
4
|
+
exports.DEFAULT_CONCURRENCY = 5;
|
|
5
|
+
exports.DEFAULT_ATTEMPTS = 3;
|
|
6
|
+
exports.DEFAULT_BACKOFF_DELAY = 2000;
|
|
7
|
+
exports.DEFAULT_REMOVE_ON_COMPLETE_COUNT = 1000;
|
|
8
|
+
exports.DEFAULT_REMOVE_ON_FAIL_COUNT = 5000;
|
|
9
|
+
exports.DEFAULT_PREFIX = 'motia';
|
|
10
|
+
exports.FIFO_CONCURRENCY = 1;
|
|
11
|
+
exports.MILLISECONDS_PER_SECOND = 1000;
|
|
12
|
+
exports.SECONDS_PER_DAY = 86400;
|
|
13
|
+
exports.DEFAULT_DLQ_TTL = 30 * exports.SECONDS_PER_DAY;
|
|
14
|
+
exports.DEFAULT_DLQ_SUFFIX = '.dlq';
|
|
15
|
+
exports.DLQ_JOB_PREFIX = 'dlq-';
|
|
16
|
+
exports.LOG_PREFIX = '[BullMQ]';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Event } from '@motiadev/core';
|
|
2
|
+
import { Queue } from 'bullmq';
|
|
3
|
+
import type { Redis } from 'ioredis';
|
|
4
|
+
import type { MergedConfig } from './config-builder';
|
|
5
|
+
export declare class DLQManager {
|
|
6
|
+
private readonly dlqQueues;
|
|
7
|
+
private readonly connection;
|
|
8
|
+
private readonly config;
|
|
9
|
+
constructor(connection: Redis, config: MergedConfig);
|
|
10
|
+
getDLQQueueName(topic: string, stepName: string): string;
|
|
11
|
+
private getOrCreateDLQQueue;
|
|
12
|
+
moveToDLQ<TData>(topic: string, stepName: string, event: Event<TData>, error: Error, attemptsMade: number, originalJobId?: string): Promise<void>;
|
|
13
|
+
closeDLQQueue(queueName: string): Promise<void>;
|
|
14
|
+
closeAll(): Promise<void>;
|
|
15
|
+
getDLQQueue(queueName: string): Queue | undefined;
|
|
16
|
+
getOrCreateDLQ(queueName: string): Queue;
|
|
17
|
+
listDLQQueueNames(): string[];
|
|
18
|
+
getDLQSuffix(): string;
|
|
19
|
+
getPrefix(): string;
|
|
20
|
+
getConnection(): Redis;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=dlq-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dlq-manager.d.ts","sourceRoot":"","sources":["../src/dlq-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAC9B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAYpD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;gBAEzB,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY;IAKnD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKxD,OAAO,CAAC,mBAAmB;IA8BrB,SAAS,CAAC,KAAK,EACnB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EACnB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE,MAAM,EACpB,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC;IA0CV,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS;IAIjD,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,KAAK;IAIxC,iBAAiB,IAAI,MAAM,EAAE;IAI7B,YAAY,IAAI,MAAM;IAItB,SAAS,IAAI,MAAM;IAInB,aAAa,IAAI,KAAK;CAGvB"}
|