@boringnode/queue 0.0.1-alpha → 0.0.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +326 -0
- package/build/index.d.ts +2 -3
- package/build/index.js +81 -95
- package/build/index.js.map +1 -1
- package/build/{job-CcAUWe8j.d.ts → job-Bd_c2lFK.d.ts} +45 -21
- package/build/src/contracts/adapter.d.ts +1 -2
- package/build/src/drivers/redis_adapter.d.ts +8 -5
- package/build/src/drivers/redis_adapter.js +147 -31
- package/build/src/drivers/redis_adapter.js.map +1 -1
- package/build/src/drivers/sync_adapter.d.ts +11 -5
- package/build/src/drivers/sync_adapter.js +11 -3
- package/build/src/drivers/sync_adapter.js.map +1 -1
- package/build/src/types/main.d.ts +1 -2
- package/package.json +6 -3
- package/build/src/contracts/lease_manager.d.ts +0 -8
- package/build/src/contracts/lease_manager.js +0 -1
- package/build/src/contracts/lease_manager.js.map +0 -1
package/README.md
CHANGED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
# @boringnode/queue
|
|
2
|
+
|
|
3
|
+
A simple and efficient queue system for Node.js applications. Built for simplicity and ease of use, `@boringnode/queue` allows you to dispatch background jobs and process them asynchronously with support for multiple queue adapters.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install @boringnode/queue
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
- **Multiple Queue Adapters**: Support for Redis (production) and Sync (testing/development)
|
|
14
|
+
- **Type-Safe Jobs**: Define jobs as TypeScript classes with typed payloads
|
|
15
|
+
- **Delayed Jobs**: Schedule jobs to run after a specific delay
|
|
16
|
+
- **Multiple Queues**: Organize jobs into different queues for better organization
|
|
17
|
+
- **Worker Management**: Process jobs with configurable concurrency
|
|
18
|
+
- **Auto-Discovery**: Automatically discover and register jobs from specified locations
|
|
19
|
+
- **Priority Queues**: Process high-priority jobs first
|
|
20
|
+
- **Retry with Backoff**: Automatic retries with exponential, linear, or fixed backoff strategies
|
|
21
|
+
- **Job Timeout**: Automatically fail or retry jobs that exceed a time limit
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
### 1. Define a Job
|
|
26
|
+
|
|
27
|
+
Create a job by extending the `Job` class:
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
import { Job } from '@boringnode/queue'
|
|
31
|
+
import type { JobOptions } from '@boringnode/queue/types/main'
|
|
32
|
+
|
|
33
|
+
interface SendEmailPayload {
|
|
34
|
+
to: string
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export default class SendEmailJob extends Job<SendEmailPayload> {
|
|
38
|
+
static readonly jobName = 'SendEmailJob'
|
|
39
|
+
|
|
40
|
+
static options: JobOptions = {
|
|
41
|
+
queue: 'email',
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async execute(): Promise<void> {
|
|
45
|
+
console.log(`Sending email to: ${this.payload.to}`)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### 2. Configure the Queue Manager
|
|
51
|
+
|
|
52
|
+
```typescript
|
|
53
|
+
import { QueueManager } from '@boringnode/queue'
|
|
54
|
+
import { redis } from '@boringnode/queue/drivers/redis_adapter'
|
|
55
|
+
import { sync } from '@boringnode/queue/drivers/sync_adapter'
|
|
56
|
+
import { Redis } from 'ioredis'
|
|
57
|
+
|
|
58
|
+
const redisConnection = new Redis({
|
|
59
|
+
host: 'localhost',
|
|
60
|
+
port: 6379,
|
|
61
|
+
keyPrefix: 'boringnode::queue::',
|
|
62
|
+
db: 0,
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
const config = {
|
|
66
|
+
default: 'redis',
|
|
67
|
+
|
|
68
|
+
adapters: {
|
|
69
|
+
sync: sync(),
|
|
70
|
+
redis: redis(redisConnection),
|
|
71
|
+
},
|
|
72
|
+
|
|
73
|
+
worker: {
|
|
74
|
+
concurrency: 5,
|
|
75
|
+
pollingInterval: '10ms',
|
|
76
|
+
},
|
|
77
|
+
|
|
78
|
+
locations: ['./app/jobs/**/*.ts'],
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
await QueueManager.init(config)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 3. Dispatch Jobs
|
|
85
|
+
|
|
86
|
+
```typescript
|
|
87
|
+
import SendEmailJob from './jobs/send_email_job.ts'
|
|
88
|
+
|
|
89
|
+
// Dispatch immediately
|
|
90
|
+
await SendEmailJob.dispatch({ to: 'user@example.com' })
|
|
91
|
+
|
|
92
|
+
// Dispatch with delay
|
|
93
|
+
await SendEmailJob.dispatch({ to: 'user@example.com' }).in('5m')
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### 4. Start a Worker
|
|
97
|
+
|
|
98
|
+
Create a worker to process jobs:
|
|
99
|
+
|
|
100
|
+
```typescript
|
|
101
|
+
import { Worker } from '@boringnode/queue'
|
|
102
|
+
|
|
103
|
+
const worker = new Worker(config)
|
|
104
|
+
await worker.start(['default', 'email', 'reports'])
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Configuration
|
|
108
|
+
|
|
109
|
+
### Queue Manager Options
|
|
110
|
+
|
|
111
|
+
```typescript
|
|
112
|
+
interface QueueManagerConfig {
|
|
113
|
+
// Default adapter to use
|
|
114
|
+
default: string
|
|
115
|
+
|
|
116
|
+
// Available queue adapters
|
|
117
|
+
adapters: {
|
|
118
|
+
[key: string]: QueueAdapter
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Worker configuration
|
|
122
|
+
worker: {
|
|
123
|
+
concurrency: number
|
|
124
|
+
pollingInterval: string
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Job discovery locations
|
|
128
|
+
locations: string[]
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Job Options
|
|
133
|
+
|
|
134
|
+
Configure individual jobs with the `options` property:
|
|
135
|
+
|
|
136
|
+
```typescript
|
|
137
|
+
static options: JobOptions = {
|
|
138
|
+
queue: 'email', // Queue name (default: 'default')
|
|
139
|
+
adapter: 'redis', // Override default adapter
|
|
140
|
+
priority: 1, // Lower number = higher priority (default: 5)
|
|
141
|
+
maxRetries: 3, // Maximum retry attempts
|
|
142
|
+
timeout: '30s', // Job timeout duration
|
|
143
|
+
failOnTimeout: true, // Fail permanently on timeout (default: false, will retry)
|
|
144
|
+
}
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Adapters
|
|
148
|
+
|
|
149
|
+
### Redis Adapter
|
|
150
|
+
|
|
151
|
+
For production use with distributed systems:
|
|
152
|
+
|
|
153
|
+
```typescript
|
|
154
|
+
import { redis } from '@boringnode/queue/drivers/redis_adapter'
|
|
155
|
+
import { Redis } from 'ioredis'
|
|
156
|
+
|
|
157
|
+
const redisConnection = new Redis({
|
|
158
|
+
host: 'localhost',
|
|
159
|
+
port: 6379,
|
|
160
|
+
keyPrefix: 'boringnode::queue::',
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
const adapter = redis(redisConnection)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Sync Adapter
|
|
167
|
+
|
|
168
|
+
For testing and development:
|
|
169
|
+
|
|
170
|
+
```typescript
|
|
171
|
+
import { sync } from '@boringnode/queue/drivers/sync_adapter'
|
|
172
|
+
|
|
173
|
+
const adapter = sync()
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Worker Configuration
|
|
177
|
+
|
|
178
|
+
Workers process jobs from one or more queues:
|
|
179
|
+
|
|
180
|
+
```typescript
|
|
181
|
+
const worker = new Worker(config)
|
|
182
|
+
|
|
183
|
+
// Process specific queues
|
|
184
|
+
await worker.start(['default', 'email', 'reports'])
|
|
185
|
+
|
|
186
|
+
// Worker will:
|
|
187
|
+
// - Process jobs with configured concurrency
|
|
188
|
+
// - Poll queues at the configured interval
|
|
189
|
+
// - Execute jobs in the order they were queued
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Delayed Jobs
|
|
193
|
+
|
|
194
|
+
Schedule jobs to run in the future:
|
|
195
|
+
|
|
196
|
+
```typescript
|
|
197
|
+
// Various time formats
|
|
198
|
+
await SendEmailJob.dispatch(payload).in('30s') // 30 seconds
|
|
199
|
+
await SendEmailJob.dispatch(payload).in('5m') // 5 minutes
|
|
200
|
+
await SendEmailJob.dispatch(payload).in('2h') // 2 hours
|
|
201
|
+
await SendEmailJob.dispatch(payload).in('1d') // 1 day
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
## Priority
|
|
205
|
+
|
|
206
|
+
Jobs with lower priority numbers are processed first:
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
export default class UrgentJob extends Job<Payload> {
|
|
210
|
+
static readonly jobName = 'UrgentJob'
|
|
211
|
+
|
|
212
|
+
static options: JobOptions = {
|
|
213
|
+
priority: 1, // Processed before default priority (5)
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async execute(): Promise<void> {
|
|
217
|
+
// ...
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## Retry and Backoff
|
|
223
|
+
|
|
224
|
+
Configure automatic retries with backoff strategies:
|
|
225
|
+
|
|
226
|
+
```typescript
|
|
227
|
+
import { exponentialBackoff, linearBackoff, fixedBackoff } from '@boringnode/queue'
|
|
228
|
+
|
|
229
|
+
export default class ReliableJob extends Job<Payload> {
|
|
230
|
+
static readonly jobName = 'ReliableJob'
|
|
231
|
+
|
|
232
|
+
static options: JobOptions = {
|
|
233
|
+
maxRetries: 5,
|
|
234
|
+
retry: {
|
|
235
|
+
backoff: () => exponentialBackoff({
|
|
236
|
+
baseDelay: '1s',
|
|
237
|
+
maxDelay: '1m',
|
|
238
|
+
multiplier: 2,
|
|
239
|
+
jitter: true,
|
|
240
|
+
}),
|
|
241
|
+
},
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async execute(): Promise<void> {
|
|
245
|
+
// ...
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Available backoff strategies:
|
|
251
|
+
|
|
252
|
+
- `exponentialBackoff({ baseDelay, maxDelay, multiplier, jitter })` - Exponential increase
|
|
253
|
+
- `linearBackoff({ baseDelay, maxDelay, multiplier })` - Linear increase
|
|
254
|
+
- `fixedBackoff({ baseDelay, jitter })` - Fixed delay between retries
|
|
255
|
+
|
|
256
|
+
## Job Timeout
|
|
257
|
+
|
|
258
|
+
Set a maximum execution time for jobs:
|
|
259
|
+
|
|
260
|
+
```typescript
|
|
261
|
+
export default class LimitedJob extends Job<Payload> {
|
|
262
|
+
static readonly jobName = 'LimitedJob'
|
|
263
|
+
|
|
264
|
+
static options: JobOptions = {
|
|
265
|
+
timeout: '30s', // Maximum execution time
|
|
266
|
+
failOnTimeout: false, // Retry on timeout (default)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async execute(): Promise<void> {
|
|
270
|
+
// Long running operation...
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
You can also set a global timeout in the worker configuration:
|
|
276
|
+
|
|
277
|
+
```typescript
|
|
278
|
+
const config = {
|
|
279
|
+
worker: {
|
|
280
|
+
timeout: '1m', // Default timeout for all jobs
|
|
281
|
+
},
|
|
282
|
+
}
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
## Job Discovery
|
|
286
|
+
|
|
287
|
+
The queue manager automatically discovers and registers jobs from the specified locations:
|
|
288
|
+
|
|
289
|
+
```typescript
|
|
290
|
+
const config = {
|
|
291
|
+
locations: [
|
|
292
|
+
'./app/jobs/**/*.ts',
|
|
293
|
+
'./modules/**/jobs/**/*.ts',
|
|
294
|
+
],
|
|
295
|
+
}
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
Jobs must:
|
|
299
|
+
- Extend the `Job` class
|
|
300
|
+
- Have a static `jobName` property
|
|
301
|
+
- Implement the `execute` method
|
|
302
|
+
- Be exported as default
|
|
303
|
+
|
|
304
|
+
## Benchmarks
|
|
305
|
+
|
|
306
|
+
Performance comparison with BullMQ measuring pure dequeue overhead (jobs are no-ops). Results are averaged over 3 runs:
|
|
307
|
+
|
|
308
|
+
| Jobs | Concurrency | @boringnode/queue | BullMQ | Diff |
|
|
309
|
+
|------|-------------|-------------------|--------|--------------|
|
|
310
|
+
| 100 | 1 | 15ms | 23ms | 34.8% faster |
|
|
311
|
+
| 100 | 5 | 24ms | 18ms | 33.3% slower |
|
|
312
|
+
| 100 | 10 | 16ms | 17ms | ~same |
|
|
313
|
+
| 1000 | 1 | 171ms | 135ms | 26.7% slower |
|
|
314
|
+
| 1000 | 5 | 106ms | 55ms | 92.7% slower |
|
|
315
|
+
| 1000 | 10 | 88ms | 57ms | 54.4% slower |
|
|
316
|
+
| 5000 | 1 | 495ms | 615ms | 19.5% faster |
|
|
317
|
+
| 5000 | 5 | 342ms | 253ms | 35.2% slower |
|
|
318
|
+
| 5000 | 10 | 456ms | 234ms | 94.9% slower |
|
|
319
|
+
|
|
320
|
+
These numbers represent queue overhead only. With real job execution, the difference becomes negligible.
|
|
321
|
+
|
|
322
|
+
Run benchmarks yourself:
|
|
323
|
+
|
|
324
|
+
```bash
|
|
325
|
+
npm run benchmark
|
|
326
|
+
```
|
package/build/index.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { Q as QueueManagerConfig, W as WorkerCycle, A as Adapter, R as RetryConfig } from './job-
|
|
2
|
-
export { J as Job, c as customBackoff, e as exponentialBackoff, f as fixedBackoff, l as linearBackoff } from './job-
|
|
3
|
-
import './src/contracts/lease_manager.js';
|
|
1
|
+
import { Q as QueueManagerConfig, W as WorkerCycle, A as Adapter, R as RetryConfig } from './job-Bd_c2lFK.js';
|
|
2
|
+
export { J as Job, c as customBackoff, e as exponentialBackoff, f as fixedBackoff, l as linearBackoff } from './job-Bd_c2lFK.js';
|
|
4
3
|
|
|
5
4
|
declare class Worker {
|
|
6
5
|
#private;
|
package/build/index.js
CHANGED
|
@@ -34,15 +34,16 @@ import debug from "#src/debug";
|
|
|
34
34
|
import { parse } from "#src/utils";
|
|
35
35
|
import * as errors from "#src/exceptions";
|
|
36
36
|
import { QueueManager } from "#src/queue_manager";
|
|
37
|
+
import { JobPool } from "#src/job_pool";
|
|
37
38
|
import { Locator } from "#src/locator";
|
|
38
39
|
var Worker = class {
|
|
39
40
|
#id;
|
|
40
41
|
#config;
|
|
41
42
|
#adapter;
|
|
42
|
-
#leaseManager;
|
|
43
43
|
#running = false;
|
|
44
44
|
#initialized = false;
|
|
45
45
|
#generator;
|
|
46
|
+
#pool;
|
|
46
47
|
get id() {
|
|
47
48
|
return this.#id;
|
|
48
49
|
}
|
|
@@ -58,11 +59,7 @@ var Worker = class {
|
|
|
58
59
|
debug("initializing worker %s", this.#id);
|
|
59
60
|
await QueueManager.init(this.#config);
|
|
60
61
|
this.#adapter = QueueManager.use();
|
|
61
|
-
this.#
|
|
62
|
-
workerId: this.#id,
|
|
63
|
-
leaseTimeout: parse(this.#config.worker?.leaseTimeout || "5m"),
|
|
64
|
-
renewalInterval: parse(this.#config.worker?.renewalInterval || "5m")
|
|
65
|
-
});
|
|
62
|
+
this.#adapter.setWorkerId(this.#id);
|
|
66
63
|
this.#initialized = true;
|
|
67
64
|
debug("worker %s initialized", this.#id);
|
|
68
65
|
}
|
|
@@ -93,8 +90,9 @@ var Worker = class {
|
|
|
93
90
|
async stop() {
|
|
94
91
|
debug("stopping worker %s", this.#id);
|
|
95
92
|
this.#running = false;
|
|
96
|
-
if (this.#
|
|
97
|
-
|
|
93
|
+
if (this.#pool) {
|
|
94
|
+
debug("worker %s: waiting for %d running jobs to complete", this.#id, this.#pool.size);
|
|
95
|
+
await this.#pool.drain();
|
|
98
96
|
}
|
|
99
97
|
if (this.#adapter) {
|
|
100
98
|
await this.#adapter.destroy();
|
|
@@ -114,56 +112,55 @@ var Worker = class {
|
|
|
114
112
|
return result.value;
|
|
115
113
|
}
|
|
116
114
|
async *process(queues) {
|
|
117
|
-
const
|
|
118
|
-
|
|
115
|
+
const pollingInterval = parse(this.#config.worker?.pollingInterval || "2s");
|
|
116
|
+
this.#pool = new JobPool();
|
|
117
|
+
while (this.#running) {
|
|
119
118
|
try {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
yield { type: "started", queue, job };
|
|
125
|
-
}
|
|
126
|
-
const results = await Promise.allSettled(jobs.map((job) => this.#execute(job, queue)));
|
|
127
|
-
for (const job of jobs) {
|
|
128
|
-
yield { type: "completed", queue, job };
|
|
129
|
-
}
|
|
130
|
-
const hasError = results.some((r) => r.status === "rejected");
|
|
131
|
-
if (hasError) {
|
|
132
|
-
const error = results.find((r) => r.status === "rejected");
|
|
133
|
-
yield { type: "error", error: error.reason, suggestedDelay: parse("5s") };
|
|
134
|
-
}
|
|
135
|
-
continue runningLoop;
|
|
136
|
-
}
|
|
119
|
+
yield* this.#fillPool(queues);
|
|
120
|
+
if (this.#pool.isEmpty()) {
|
|
121
|
+
yield { type: "idle", suggestedDelay: pollingInterval };
|
|
122
|
+
continue;
|
|
137
123
|
}
|
|
138
|
-
const
|
|
139
|
-
yield { type: "
|
|
124
|
+
const completed = await this.#pool.waitForNextCompletion();
|
|
125
|
+
yield { type: "completed", queue: completed.queue, job: completed.job };
|
|
140
126
|
} catch (error) {
|
|
141
127
|
yield { type: "error", error, suggestedDelay: parse("5s") };
|
|
142
128
|
}
|
|
143
129
|
}
|
|
144
130
|
}
|
|
131
|
+
async *#fillPool(queues) {
|
|
132
|
+
const concurrency = this.#config.worker?.concurrency || 1;
|
|
133
|
+
while (this.#pool.hasCapacity(concurrency)) {
|
|
134
|
+
const result = await this.#acquireNextJob(queues);
|
|
135
|
+
if (!result) break;
|
|
136
|
+
const { job, queue } = result;
|
|
137
|
+
const promise = this.#execute(job, queue);
|
|
138
|
+
this.#pool.add(job, queue, promise);
|
|
139
|
+
yield { type: "started", queue, job };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
145
142
|
async #execute(job, queue) {
|
|
146
143
|
const startTime = performance.now();
|
|
147
144
|
debug("worker %s: executing job %s (%s)", this.#id, job.id, job.name);
|
|
148
|
-
|
|
145
|
+
const { instance, options, timeout } = await this.#initJob(job, queue);
|
|
149
146
|
try {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
debug("worker %s: job class %s not found for job %s", this.#id, job.name, job.id);
|
|
153
|
-
throw error;
|
|
154
|
-
}
|
|
155
|
-
const instance = new JobClass(job.payload);
|
|
156
|
-
const options = JobClass.options || {};
|
|
157
|
-
try {
|
|
158
|
-
await instance.execute();
|
|
159
|
-
await job._lease.commit();
|
|
147
|
+
await this.#executeWithTimeout(instance, timeout);
|
|
148
|
+
await this.#adapter.completeJob(job.id, queue);
|
|
160
149
|
const duration = (performance.now() - startTime).toFixed(2);
|
|
161
150
|
debug("worker %s: successfully executed job %s in %dms", this.#id, job.id, duration);
|
|
162
151
|
} catch (e) {
|
|
152
|
+
const isTimeout = e instanceof errors.E_JOB_TIMEOUT;
|
|
153
|
+
if (isTimeout && options.failOnTimeout) {
|
|
154
|
+
debug("worker %s: job %s timed out and failOnTimeout is set", this.#id, job.id);
|
|
155
|
+
await this.#adapter.failJob(job.id, queue, e);
|
|
156
|
+
await instance.failed?.(e);
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
163
159
|
const mergedConfig = QueueManager.getMergedRetryConfig(queue, options.retry);
|
|
164
160
|
if (typeof mergedConfig.maxRetries === "undefined" || mergedConfig.maxRetries <= 0) {
|
|
165
161
|
debug("worker %s: job %s has no retries configured, marking as failed", this.#id, job.id);
|
|
166
|
-
await
|
|
162
|
+
await this.#adapter.failJob(job.id, queue, e);
|
|
163
|
+
await instance.failed?.(e);
|
|
167
164
|
return;
|
|
168
165
|
}
|
|
169
166
|
if (job.attempts >= mergedConfig.maxRetries) {
|
|
@@ -173,76 +170,65 @@ var Worker = class {
|
|
|
173
170
|
job.id,
|
|
174
171
|
mergedConfig.maxRetries
|
|
175
172
|
);
|
|
173
|
+
await this.#adapter.failJob(job.id, queue, e);
|
|
176
174
|
const exception = new errors.E_JOB_MAX_ATTEMPTS_REACHED([job.name]);
|
|
177
|
-
await instance.failed(exception);
|
|
175
|
+
await instance.failed?.(exception);
|
|
178
176
|
return;
|
|
179
177
|
}
|
|
180
178
|
if (mergedConfig.backoff) {
|
|
181
179
|
const strategy = mergedConfig.backoff();
|
|
182
180
|
const nextRetryAt = strategy.getNextRetryAt(job.attempts + 1);
|
|
183
181
|
debug("worker %s: job %s will retry at %s", this.#id, job.id, nextRetryAt.toISOString());
|
|
184
|
-
await this.#
|
|
182
|
+
await this.#adapter.retryJob(job.id, queue, nextRetryAt);
|
|
185
183
|
return;
|
|
186
184
|
}
|
|
187
|
-
await this.#
|
|
185
|
+
await this.#adapter.retryJob(job.id, queue);
|
|
188
186
|
}
|
|
189
187
|
}
|
|
190
|
-
async #
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
debug("worker %s: failed to acquire lease for job %s", this.#id, job.id);
|
|
202
|
-
await this.#adapter.pushOn(queue, job);
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
debug("worker %s: acquired lease for job %s", this.#id, job.id);
|
|
206
|
-
jobs.push({
|
|
207
|
-
...job,
|
|
208
|
-
_lease: {
|
|
209
|
-
commit: () => this.#commitJob(job.id),
|
|
210
|
-
rollback: () => this.#rollbackJob(job, queue)
|
|
211
|
-
}
|
|
212
|
-
});
|
|
213
|
-
} catch (error) {
|
|
214
|
-
console.log(error);
|
|
215
|
-
throw error;
|
|
216
|
-
}
|
|
188
|
+
async #initJob(job, queue) {
|
|
189
|
+
try {
|
|
190
|
+
const JobClass = Locator.getOrThrow(job.name);
|
|
191
|
+
const instance = new JobClass(job.payload);
|
|
192
|
+
const options = JobClass.options || {};
|
|
193
|
+
const timeout = this.#getJobTimeout(options);
|
|
194
|
+
return { instance, options, timeout };
|
|
195
|
+
} catch (error) {
|
|
196
|
+
debug("worker %s: failed to initialize job %s (%s)", this.#id, job.id, job.name);
|
|
197
|
+
await this.#adapter.failJob(job.id, queue, error);
|
|
198
|
+
throw error;
|
|
217
199
|
}
|
|
218
|
-
return jobs;
|
|
219
200
|
}
|
|
220
|
-
#
|
|
221
|
-
|
|
222
|
-
|
|
201
|
+
#getJobTimeout(options) {
|
|
202
|
+
if (options.timeout !== void 0) {
|
|
203
|
+
return parse(options.timeout);
|
|
204
|
+
}
|
|
205
|
+
if (this.#config.worker?.timeout !== void 0) {
|
|
206
|
+
return parse(this.#config.worker.timeout);
|
|
207
|
+
}
|
|
208
|
+
return void 0;
|
|
223
209
|
}
|
|
224
|
-
async #
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
210
|
+
async #executeWithTimeout(instance, timeout) {
|
|
211
|
+
if (!timeout) {
|
|
212
|
+
return instance.execute();
|
|
213
|
+
}
|
|
214
|
+
const signal = AbortSignal.timeout(timeout);
|
|
215
|
+
const abortPromise = new Promise((_, reject) => {
|
|
216
|
+
signal.addEventListener("abort", () => {
|
|
217
|
+
reject(new errors.E_JOB_TIMEOUT([instance.constructor.name, timeout]));
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
await Promise.race([instance.execute(signal), abortPromise]);
|
|
231
221
|
}
|
|
232
|
-
async #
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
const delay = nextRetryAt.getTime() - Date.now();
|
|
241
|
-
if (delay > 0) {
|
|
242
|
-
await this.#adapter.pushLaterOn(queue, updatedJob, delay);
|
|
243
|
-
} else {
|
|
244
|
-
await this.#adapter.pushOn(queue, updatedJob);
|
|
222
|
+
async #acquireNextJob(queues) {
|
|
223
|
+
for (const queue of queues) {
|
|
224
|
+
const job = await this.#adapter.popFrom(queue);
|
|
225
|
+
if (!job) {
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
debug("worker %s: acquired job %s", this.#id, job.id);
|
|
229
|
+
return { job, queue };
|
|
245
230
|
}
|
|
231
|
+
return null;
|
|
246
232
|
}
|
|
247
233
|
async #setupGracefulShutdown() {
|
|
248
234
|
const shutdown = async () => {
|
package/build/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/job.ts","../src/worker.ts","../src/queue_manager.ts","../src/strategies/backoff_strategy.ts"],"sourcesContent":["import { JobDispatcher } from '#src/job_dispatcher'\nimport type { JobOptions } from '#types/main'\n\nexport abstract class Job<Payload = any> {\n readonly #payload: Payload\n\n static options: JobOptions = {}\n\n get payload(): Payload {\n return this.#payload\n }\n\n constructor(payload: Payload) {\n this.#payload = payload\n }\n\n static dispatch<T extends Job>(\n this: new (payload: any) => T,\n payload: T extends Job<infer P> ? P : never\n ): JobDispatcher<T extends Job<infer P> ? P : never> {\n const dispatcher = new JobDispatcher<T extends Job<infer P> ? P : never>(\n (this as any).jobName,\n payload\n )\n\n if ((this as any).options.queue) {\n dispatcher.toQueue((this as any).options.queue)\n }\n\n if ((this as any).options.adapter) {\n dispatcher.with((this as any).options.adapter)\n }\n\n if ((this as any).options.priority !== undefined) {\n dispatcher.priority((this as any).options.priority)\n }\n\n return dispatcher\n }\n\n abstract execute(): Promise<void>\n\n failed?(error: Error): Promise<void>\n}\n","import { randomUUID } from 'node:crypto'\nimport { setTimeout } from 'node:timers/promises'\nimport debug from '#src/debug'\nimport { parse } from '#src/utils'\nimport * as errors from '#src/exceptions'\nimport { QueueManager } from '#src/queue_manager'\nimport type { Adapter } from '#contracts/adapter'\nimport type { LeaseManager } from '#contracts/lease_manager'\nimport type { AcquiredJob, JobData, QueueManagerConfig, WorkerCycle } from '#types/main'\nimport { Locator } from '#src/locator'\nimport type { JobOptions } from '#types/main'\n\nexport class Worker {\n readonly #id: string\n readonly #config: QueueManagerConfig\n #adapter!: Adapter\n #leaseManager!: LeaseManager\n #running = false\n #initialized = false\n #generator?: AsyncGenerator<WorkerCycle, void, unknown>\n\n get id() {\n return this.#id\n }\n\n constructor(config: QueueManagerConfig) {\n this.#config = config\n this.#id = randomUUID()\n\n debug('created worker with id %s and config %O', this.#id, config)\n }\n\n async init() {\n if (this.#initialized) {\n return\n }\n\n debug('initializing worker %s', this.#id)\n\n await QueueManager.init(this.#config)\n\n this.#adapter = QueueManager.use()\n this.#leaseManager = this.#adapter.createLeaseManager({\n workerId: this.#id,\n leaseTimeout: parse(this.#config.worker?.leaseTimeout || '5m'),\n renewalInterval: parse(this.#config.worker?.renewalInterval || '5m'),\n })\n\n this.#initialized = true\n\n debug('worker %s initialized', this.#id)\n }\n\n async start(queues: string[] = ['default']): Promise<void> {\n await this.init()\n\n if (this.#running) {\n debug('worker %s is already running', this.#id)\n return\n }\n\n this.#running = true\n\n debug('starting worker %s on queues: %O', this.#id, queues)\n\n await this.#setupGracefulShutdown()\n\n for await (const cycle of this.process(queues)) {\n if (['started', 'completed'].includes(cycle.type)) {\n continue\n }\n\n if (['idle', 'error'].includes(cycle.type)) {\n // @ts-expect-error - we know suggestedDelay exists for these types\n const delay = parse(cycle.suggestedDelay)\n\n if (cycle.type === 'error') {\n debug('worker %s encountered an error: %O', this.#id, cycle.error)\n } else {\n debug('worker %s is idle, waiting for %dms', this.#id, delay)\n }\n\n await setTimeout(delay)\n }\n }\n }\n\n async stop() {\n debug('stopping worker %s', this.#id)\n\n this.#running = false\n\n if (this.#leaseManager) {\n await this.#leaseManager.destroy()\n }\n\n if (this.#adapter) {\n await this.#adapter.destroy()\n }\n }\n\n async processCycle(queues: string[]): Promise<WorkerCycle | null> {\n await this.init()\n\n this.#running = true\n\n if (!this.#generator) {\n this.#generator = this.process(queues)\n }\n\n const result = await this.#generator.next()\n\n if (result.done) {\n this.#generator = undefined\n return null\n }\n\n return result.value\n }\n\n async *process(queues: string[]): AsyncGenerator<WorkerCycle, void, unknown> {\n const concurrency = this.#config.worker?.concurrency || 1\n\n runningLoop: while (this.#running) {\n try {\n for (const queue of queues) {\n const jobs = await this.#acquireJobs(queue, concurrency)\n\n if (jobs.length > 0) {\n // Yield started events for all jobs\n for (const job of jobs) {\n yield { type: 'started', queue, job }\n }\n\n // Execute all jobs in parallel\n const results = await Promise.allSettled(jobs.map((job) => this.#execute(job, queue)))\n\n // Yield completed events for all jobs\n for (const job of jobs) {\n yield { type: 'completed', queue, job: job }\n }\n\n // Check if any job failed\n const hasError = results.some((r) => r.status === 'rejected')\n if (hasError) {\n const error = results.find((r) => r.status === 'rejected') as PromiseRejectedResult\n yield { type: 'error', error: error.reason, suggestedDelay: parse('5s') }\n }\n\n continue runningLoop\n }\n }\n\n const pollingInterval = parse(this.#config.worker?.pollingInterval || '2s')\n yield { type: 'idle', suggestedDelay: pollingInterval }\n } catch (error) {\n yield { type: 'error', error: error as Error, suggestedDelay: parse('5s') }\n }\n }\n }\n\n async #execute(job: AcquiredJob, queue: string): Promise<void> {\n const startTime = performance.now()\n\n debug('worker %s: executing job %s (%s)', this.#id, job.id, job.name)\n\n let JobClass: any\n\n try {\n JobClass = Locator.getOrThrow(job.name)\n } catch (error) {\n debug('worker %s: job class %s not found for job %s', this.#id, job.name, job.id)\n throw error\n }\n\n const instance = new JobClass(job.payload)\n const options: JobOptions = JobClass.options || {}\n\n try {\n await instance.execute()\n await job._lease.commit()\n\n const duration = (performance.now() - startTime).toFixed(2)\n debug('worker %s: successfully executed job %s in %dms', this.#id, job.id, duration)\n } catch (e) {\n const mergedConfig = QueueManager.getMergedRetryConfig(queue, options.retry)\n\n if (typeof mergedConfig.maxRetries === 'undefined' || mergedConfig.maxRetries <= 0) {\n debug('worker %s: job %s has no retries configured, marking as failed', this.#id, job.id)\n\n await instance.failed(e as Error)\n return\n }\n\n if (job.attempts >= mergedConfig.maxRetries!) {\n debug(\n 'worker %s: job %s has exceeded max retries (%d), marking as failed',\n this.#id,\n job.id,\n mergedConfig.maxRetries\n )\n\n const exception = new errors.E_JOB_MAX_ATTEMPTS_REACHED([job.name])\n await instance.failed(exception)\n\n return\n }\n\n if (mergedConfig.backoff) {\n const strategy = mergedConfig.backoff()\n const nextRetryAt = strategy.getNextRetryAt(job.attempts + 1)\n\n debug('worker %s: job %s will retry at %s', this.#id, job.id, nextRetryAt.toISOString())\n\n await this.#rollbackJobWithBackoff(job, queue, nextRetryAt)\n\n return\n }\n\n await this.#rollbackJob(job, queue)\n }\n }\n\n async #acquireJobs(queue: string, count: number): Promise<AcquiredJob[]> {\n const jobs: AcquiredJob[] = []\n\n // Try to acquire up to `count` jobs\n for (let i = 0; i < count; i++) {\n const job = await this.#adapter.popFrom(queue)\n\n if (!job) {\n break\n }\n\n debug('worker %s: attempting to acquire lease for job %s', this.#id, job.id)\n\n try {\n const acquired = await this.#leaseManager.acquire(job.id)\n\n if (!acquired) {\n debug('worker %s: failed to acquire lease for job %s', this.#id, job.id)\n\n await this.#adapter.pushOn(queue, job)\n continue\n }\n\n debug('worker %s: acquired lease for job %s', this.#id, job.id)\n\n jobs.push({\n ...job,\n _lease: {\n commit: () => this.#commitJob(job.id),\n rollback: () => this.#rollbackJob(job, queue),\n },\n })\n } catch (error) {\n console.log(error)\n throw error\n }\n }\n\n return jobs\n }\n\n #commitJob(jobId: string) {\n debug('worker %s: committing job %s', this.#id, jobId)\n return this.#leaseManager.release(jobId)\n }\n\n async #rollbackJob(job: JobData, queue: string) {\n debug('worker %s: rolling back job %s', this.#id, job.id)\n\n const updatedJob = {\n ...job,\n attempts: (job.attempts || 0) + 1,\n }\n\n await Promise.all([this.#leaseManager.release(job.id), this.#adapter.pushOn(queue, updatedJob)])\n }\n\n async #rollbackJobWithBackoff(job: JobData, queue: string, nextRetryAt: Date) {\n debug('worker %s: rolling back job %s with backoff', this.#id, job.id)\n\n const updatedJob = {\n ...job,\n attempts: (job.attempts || 0) + 1,\n nextRetryAt,\n }\n\n await this.#leaseManager.release(job.id)\n\n const delay = nextRetryAt.getTime() - Date.now()\n\n if (delay > 0) {\n await this.#adapter.pushLaterOn(queue, updatedJob, delay)\n } else {\n await this.#adapter.pushOn(queue, updatedJob)\n }\n }\n\n async #setupGracefulShutdown() {\n const shutdown = async () => {\n debug('received shutdown signal, stopping worker...')\n await this.stop()\n process.exit(0)\n }\n\n process.on('SIGINT', shutdown)\n process.on('SIGTERM', shutdown)\n }\n}\n","import * as errors from '#src/exceptions'\nimport debug from '#src/debug'\nimport { Locator } from '#src/locator'\nimport type { Adapter } from '#contracts/adapter'\nimport type { AdapterFactory, QueueConfig, QueueManagerConfig, RetryConfig } from '#types/main'\n\nclass QueueManagerSingleton {\n #defaultAdapter!: string\n #adapters: Record<string, AdapterFactory> = {}\n #globalRetryConfig?: RetryConfig\n #queueConfigs: Map<string, QueueConfig> = new Map()\n\n async init(config: QueueManagerConfig) {\n debug('initializing queue manager with config: %O', config)\n\n this.#validateConfig(config)\n\n this.#defaultAdapter = config.default\n this.#adapters = config.adapters\n this.#globalRetryConfig = config.retry\n\n if (config.queues) {\n for (const [queue, queueConfig] of Object.entries(config.queues)) {\n this.#queueConfigs.set(queue, queueConfig as QueueConfig)\n }\n }\n\n await Locator.registerFromGlob(config.locations)\n\n return this\n }\n\n use(adapter?: string): Adapter {\n if (!adapter) {\n adapter = this.#defaultAdapter\n }\n\n const adapterInstance = this.#adapters[adapter]\n\n if (!adapterInstance) {\n throw new errors.E_CONFIGURATION_ERROR([`Adapter \"${adapter}\" is not registered`])\n }\n\n debug('using adapter \"%s\"', adapter)\n\n try {\n return adapterInstance()\n } catch (error) {\n // TODO: Improve error handling\n throw new Error()\n // throw new errors.E_ADAPTER_ERROR(`Failed to initialize adapter \"${adapter}\"`, error as Error)\n }\n }\n\n /**\n * Priority: job > queue > global\n */\n getMergedRetryConfig(queue: string, jobRetryConfig?: RetryConfig): RetryConfig {\n const queueConfig = this.#queueConfigs.get(queue)\n const queueRetryConfig = queueConfig?.retry || {}\n\n let maxRetries =\n jobRetryConfig?.maxRetries ||\n queueRetryConfig.maxRetries ||\n this.#globalRetryConfig?.maxRetries ||\n 0\n\n let backoff =\n jobRetryConfig?.backoff || queueRetryConfig.backoff || this.#globalRetryConfig?.backoff\n\n return { maxRetries, backoff }\n }\n\n #validateConfig(config: QueueManagerConfig): void {\n if (!config.adapters || Object.keys(config.adapters).length === 0) {\n throw new errors.E_CONFIGURATION_ERROR(['At least one adapter must be configured'])\n }\n\n if (!config.default) {\n throw new errors.E_CONFIGURATION_ERROR(['Default adapter must be specified'])\n }\n\n if (!config.locations || config.locations.length === 0) {\n throw new errors.E_CONFIGURATION_ERROR(['Job locations must be specified'])\n }\n\n if (!config.adapters[config.default]) {\n throw new errors.E_CONFIGURATION_ERROR([\n `Default adapter \"${config.default}\" not found in adapters configuration`,\n ])\n }\n\n for (const [name, factory] of Object.entries(config.adapters)) {\n if (typeof factory !== 'function') {\n throw new errors.E_CONFIGURATION_ERROR([`Adapter \"${name}\" must be a factory function`])\n }\n }\n }\n\n async destroy() {\n for (const adapterName in this.#adapters) {\n const adapter = this.#adapters[adapterName]()\n await adapter.destroy()\n }\n }\n}\n\nexport const QueueManager = new QueueManagerSingleton()\n","import type { BackoffConfig, Duration } from '#types/main'\nimport * as errors from '#src/exceptions'\nimport { parse } from '#src/utils'\nimport { RuntimeException } from '@poppinss/utils'\nimport { assertUnreachable } from '@poppinss/utils/assert'\n\nexport class BackoffStrategy {\n readonly #config: BackoffConfig\n\n constructor(config: BackoffConfig) {\n this.#config = config\n this.#validateConfig()\n }\n\n calculateDelay(attempt: number): number {\n if (attempt < 1) {\n throw new RuntimeException('Attempt number must be >= 1')\n }\n\n const baseDelayMs = parse(this.#config.baseDelay)\n const maxDelayMs = this.#config.maxDelay ? parse(this.#config.maxDelay) : Infinity\n const multiplier = this.#config.multiplier ?? 2\n\n let delay: number\n\n switch (this.#config.strategy) {\n case 'exponential':\n delay = baseDelayMs * Math.pow(multiplier, attempt - 1)\n break\n case 'linear':\n delay = baseDelayMs * attempt\n break\n case 'fixed':\n delay = baseDelayMs\n break\n default:\n assertUnreachable(this.#config.strategy)\n }\n\n // Apply max delay limit\n delay = Math.min(delay, maxDelayMs)\n\n if (this.#config.jitter) {\n delay = this.#applyJitter(delay)\n }\n\n return Math.floor(delay)\n }\n\n getNextRetryAt(attempt: number): Date {\n const delay = this.calculateDelay(attempt)\n return new Date(Date.now() + delay)\n }\n\n getConfig(): Readonly<BackoffConfig> {\n return Object.freeze({ ...this.#config })\n }\n\n #validateConfig() {\n const baseDelayMs = parse(this.#config.baseDelay)\n\n if (baseDelayMs <= 0) {\n throw new errors.E_INVALID_BASE_DELAY([\n 'Base delay must be a positive integer greater than zero',\n ])\n }\n\n if (this.#config.maxDelay) {\n const maxDelayMs = parse(this.#config.maxDelay)\n\n if (maxDelayMs <= 0) {\n throw new errors.E_INVALID_MAX_DELAY([\n 'Max delay must be a positive integer greater than zero',\n ])\n }\n\n if (maxDelayMs <= baseDelayMs) {\n throw new errors.E_INVALID_MAX_DELAY(['Max delay should be greater than base delay'])\n }\n }\n\n if (this.#config.multiplier !== undefined) {\n if (this.#config.multiplier <= 0) {\n throw new errors.E_INVALID_MULTIPLIER([\n 'Multiplier must be a positive number greater than zero',\n ])\n }\n\n if (this.#config.strategy === 'exponential' && this.#config.multiplier < 1) {\n throw new errors.E_INVALID_MULTIPLIER(['Exponential strategy multiplier should be >= 1'])\n }\n }\n }\n\n #applyJitter(delay: number): number {\n const jitterRange = delay * 0.25\n const jitter = (Math.random() - 0.5) * 2 * jitterRange\n\n return Math.max(0, delay + jitter)\n }\n}\n\nexport function exponentialBackoff(config?: Partial<Omit<BackoffConfig, 'strategy'>>) {\n return () =>\n new BackoffStrategy({\n strategy: 'exponential',\n baseDelay: '1s',\n maxDelay: '5m',\n multiplier: 2,\n jitter: true,\n ...config,\n })\n}\n\nexport function linearBackoff(config?: Partial<Omit<BackoffConfig, 'strategy'>>) {\n return () =>\n new BackoffStrategy({\n strategy: 'linear',\n baseDelay: '5s',\n maxDelay: '2m',\n ...config,\n })\n}\n\nexport function fixedBackoff(delay: Duration = '10s') {\n return () =>\n new BackoffStrategy({\n strategy: 'fixed',\n baseDelay: delay,\n })\n}\n\nexport function customBackoff(config: BackoffConfig) {\n return () => new BackoffStrategy(config)\n}\n"],"mappings":";AAAA,SAAS,qBAAqB;AAGvB,IAAe,MAAf,MAAkC;AAAA,EAC9B;AAAA,EAET,OAAO,UAAsB,CAAC;AAAA,EAE9B,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,YAAY,SAAkB;AAC5B,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,SAEL,SACmD;AACnD,UAAM,aAAa,IAAI;AAAA,MACpB,KAAa;AAAA,MACd;AAAA,IACF;AAEA,QAAK,KAAa,QAAQ,OAAO;AAC/B,iBAAW,QAAS,KAAa,QAAQ,KAAK;AAAA,IAChD;AAEA,QAAK,KAAa,QAAQ,SAAS;AACjC,iBAAW,KAAM,KAAa,QAAQ,OAAO;AAAA,IAC/C;AAEA,QAAK,KAAa,QAAQ,aAAa,QAAW;AAChD,iBAAW,SAAU,KAAa,QAAQ,QAAQ;AAAA,IACpD;AAEA,WAAO;AAAA,EACT;AAKF;;;AC3CA,SAAS,kBAAkB;AAC3B,SAAS,kBAAkB;AAC3B,OAAO,WAAW;AAClB,SAAS,aAAa;AACtB,YAAY,YAAY;AACxB,SAAS,oBAAoB;AAI7B,SAAS,eAAe;AAGjB,IAAM,SAAN,MAAa;AAAA,EACT;AAAA,EACA;AAAA,EACT;AAAA,EACA;AAAA,EACA,WAAW;AAAA,EACX,eAAe;AAAA,EACf;AAAA,EAEA,IAAI,KAAK;AACP,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,YAAY,QAA4B;AACtC,SAAK,UAAU;AACf,SAAK,MAAM,WAAW;AAEtB,UAAM,2CAA2C,KAAK,KAAK,MAAM;AAAA,EACnE;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,KAAK,cAAc;AACrB;AAAA,IACF;AAEA,UAAM,0BAA0B,KAAK,GAAG;AAExC,UAAM,aAAa,KAAK,KAAK,OAAO;AAEpC,SAAK,WAAW,aAAa,IAAI;AACjC,SAAK,gBAAgB,KAAK,SAAS,mBAAmB;AAAA,MACpD,UAAU,KAAK;AAAA,MACf,cAAc,MAAM,KAAK,QAAQ,QAAQ,gBAAgB,IAAI;AAAA,MAC7D,iBAAiB,MAAM,KAAK,QAAQ,QAAQ,mBAAmB,IAAI;AAAA,IACrE,CAAC;AAED,SAAK,eAAe;AAEpB,UAAM,yBAAyB,KAAK,GAAG;AAAA,EACzC;AAAA,EAEA,MAAM,MAAM,SAAmB,CAAC,SAAS,GAAkB;AACzD,UAAM,KAAK,KAAK;AAEhB,QAAI,KAAK,UAAU;AACjB,YAAM,gCAAgC,KAAK,GAAG;AAC9C;AAAA,IACF;AAEA,SAAK,WAAW;AAEhB,UAAM,oCAAoC,KAAK,KAAK,MAAM;AAE1D,UAAM,KAAK,uBAAuB;AAElC,qBAAiB,SAAS,KAAK,QAAQ,MAAM,GAAG;AAC9C,UAAI,CAAC,WAAW,WAAW,EAAE,SAAS,MAAM,IAAI,GAAG;AACjD;AAAA,MACF;AAEA,UAAI,CAAC,QAAQ,OAAO,EAAE,SAAS,MAAM,IAAI,GAAG;AAE1C,cAAM,QAAQ,MAAM,MAAM,cAAc;AAExC,YAAI,MAAM,SAAS,SAAS;AAC1B,gBAAM,sCAAsC,KAAK,KAAK,MAAM,KAAK;AAAA,QACnE,OAAO;AACL,gBAAM,uCAAuC,KAAK,KAAK,KAAK;AAAA,QAC9D;AAEA,cAAM,WAAW,KAAK;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,sBAAsB,KAAK,GAAG;AAEpC,SAAK,WAAW;AAEhB,QAAI,KAAK,eAAe;AACtB,YAAM,KAAK,cAAc,QAAQ;AAAA,IACnC;AAEA,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,QAAQ;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,QAA+C;AAChE,UAAM,KAAK,KAAK;AAEhB,SAAK,WAAW;AAEhB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,KAAK,QAAQ,MAAM;AAAA,IACvC;AAEA,UAAM,SAAS,MAAM,KAAK,WAAW,KAAK;AAE1C,QAAI,OAAO,MAAM;AACf,WAAK,aAAa;AAClB,aAAO;AAAA,IACT;AAEA,WAAO,OAAO;AAAA,EAChB;AAAA,EAEA,OAAO,QAAQ,QAA8D;AAC3E,UAAM,cAAc,KAAK,QAAQ,QAAQ,eAAe;AAExD,gBAAa,QAAO,KAAK,UAAU;AACjC,UAAI;AACF,mBAAW,SAAS,QAAQ;AAC1B,gBAAM,OAAO,MAAM,KAAK,aAAa,OAAO,WAAW;AAEvD,cAAI,KAAK,SAAS,GAAG;AAEnB,uBAAW,OAAO,MAAM;AACtB,oBAAM,EAAE,MAAM,WAAW,OAAO,IAAI;AAAA,YACtC;AAGA,kBAAM,UAAU,MAAM,QAAQ,WAAW,KAAK,IAAI,CAAC,QAAQ,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC;AAGrF,uBAAW,OAAO,MAAM;AACtB,oBAAM,EAAE,MAAM,aAAa,OAAO,IAAS;AAAA,YAC7C;AAGA,kBAAM,WAAW,QAAQ,KAAK,CAAC,MAAM,EAAE,WAAW,UAAU;AAC5D,gBAAI,UAAU;AACZ,oBAAM,QAAQ,QAAQ,KAAK,CAAC,MAAM,EAAE,WAAW,UAAU;AACzD,oBAAM,EAAE,MAAM,SAAS,OAAO,MAAM,QAAQ,gBAAgB,MAAM,IAAI,EAAE;AAAA,YAC1E;AAEA,qBAAS;AAAA,UACX;AAAA,QACF;AAEA,cAAM,kBAAkB,MAAM,KAAK,QAAQ,QAAQ,mBAAmB,IAAI;AAC1E,cAAM,EAAE,MAAM,QAAQ,gBAAgB,gBAAgB;AAAA,MACxD,SAAS,OAAO;AACd,cAAM,EAAE,MAAM,SAAS,OAAuB,gBAAgB,MAAM,IAAI,EAAE;AAAA,MAC5E;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,KAAkB,OAA8B;AAC7D,UAAM,YAAY,YAAY,IAAI;AAElC,UAAM,oCAAoC,KAAK,KAAK,IAAI,IAAI,IAAI,IAAI;AAEpE,QAAI;AAEJ,QAAI;AACF,iBAAW,QAAQ,WAAW,IAAI,IAAI;AAAA,IACxC,SAAS,OAAO;AACd,YAAM,gDAAgD,KAAK,KAAK,IAAI,MAAM,IAAI,EAAE;AAChF,YAAM;AAAA,IACR;AAEA,UAAM,WAAW,IAAI,SAAS,IAAI,OAAO;AACzC,UAAM,UAAsB,SAAS,WAAW,CAAC;AAEjD,QAAI;AACF,YAAM,SAAS,QAAQ;AACvB,YAAM,IAAI,OAAO,OAAO;AAExB,YAAM,YAAY,YAAY,IAAI,IAAI,WAAW,QAAQ,CAAC;AAC1D,YAAM,mDAAmD,KAAK,KAAK,IAAI,IAAI,QAAQ;AAAA,IACrF,SAAS,GAAG;AACV,YAAM,eAAe,aAAa,qBAAqB,OAAO,QAAQ,KAAK;AAE3E,UAAI,OAAO,aAAa,eAAe,eAAe,aAAa,cAAc,GAAG;AAClF,cAAM,kEAAkE,KAAK,KAAK,IAAI,EAAE;AAExF,cAAM,SAAS,OAAO,CAAU;AAChC;AAAA,MACF;AAEA,UAAI,IAAI,YAAY,aAAa,YAAa;AAC5C;AAAA,UACE;AAAA,UACA,KAAK;AAAA,UACL,IAAI;AAAA,UACJ,aAAa;AAAA,QACf;AAEA,cAAM,YAAY,IAAW,kCAA2B,CAAC,IAAI,IAAI,CAAC;AAClE,cAAM,SAAS,OAAO,SAAS;AAE/B;AAAA,MACF;AAEA,UAAI,aAAa,SAAS;AACxB,cAAM,WAAW,aAAa,QAAQ;AACtC,cAAM,cAAc,SAAS,eAAe,IAAI,WAAW,CAAC;AAE5D,cAAM,sCAAsC,KAAK,KAAK,IAAI,IAAI,YAAY,YAAY,CAAC;AAEvF,cAAM,KAAK,wBAAwB,KAAK,OAAO,WAAW;AAE1D;AAAA,MACF;AAEA,YAAM,KAAK,aAAa,KAAK,KAAK;AAAA,IACpC;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,OAAe,OAAuC;AACvE,UAAM,OAAsB,CAAC;AAG7B,aAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,KAAK,SAAS,QAAQ,KAAK;AAE7C,UAAI,CAAC,KAAK;AACR;AAAA,MACF;AAEA,YAAM,qDAAqD,KAAK,KAAK,IAAI,EAAE;AAE3E,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,cAAc,QAAQ,IAAI,EAAE;AAExD,YAAI,CAAC,UAAU;AACb,gBAAM,iDAAiD,KAAK,KAAK,IAAI,EAAE;AAEvE,gBAAM,KAAK,SAAS,OAAO,OAAO,GAAG;AACrC;AAAA,QACF;AAEA,cAAM,wCAAwC,KAAK,KAAK,IAAI,EAAE;AAE9D,aAAK,KAAK;AAAA,UACR,GAAG;AAAA,UACH,QAAQ;AAAA,YACN,QAAQ,MAAM,KAAK,WAAW,IAAI,EAAE;AAAA,YACpC,UAAU,MAAM,KAAK,aAAa,KAAK,KAAK;AAAA,UAC9C;AAAA,QACF,CAAC;AAAA,MACH,SAAS,OAAO;AACd,gBAAQ,IAAI,KAAK;AACjB,cAAM;AAAA,MACR;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,WAAW,OAAe;AACxB,UAAM,gCAAgC,KAAK,KAAK,KAAK;AACrD,WAAO,KAAK,cAAc,QAAQ,KAAK;AAAA,EACzC;AAAA,EAEA,MAAM,aAAa,KAAc,OAAe;AAC9C,UAAM,kCAAkC,KAAK,KAAK,IAAI,EAAE;AAExD,UAAM,aAAa;AAAA,MACjB,GAAG;AAAA,MACH,WAAW,IAAI,YAAY,KAAK;AAAA,IAClC;AAEA,UAAM,QAAQ,IAAI,CAAC,KAAK,cAAc,QAAQ,IAAI,EAAE,GAAG,KAAK,SAAS,OAAO,OAAO,UAAU,CAAC,CAAC;AAAA,EACjG;AAAA,EAEA,MAAM,wBAAwB,KAAc,OAAe,aAAmB;AAC5E,UAAM,+CAA+C,KAAK,KAAK,IAAI,EAAE;AAErE,UAAM,aAAa;AAAA,MACjB,GAAG;AAAA,MACH,WAAW,IAAI,YAAY,KAAK;AAAA,MAChC;AAAA,IACF;AAEA,UAAM,KAAK,cAAc,QAAQ,IAAI,EAAE;AAEvC,UAAM,QAAQ,YAAY,QAAQ,IAAI,KAAK,IAAI;AAE/C,QAAI,QAAQ,GAAG;AACb,YAAM,KAAK,SAAS,YAAY,OAAO,YAAY,KAAK;AAAA,IAC1D,OAAO;AACL,YAAM,KAAK,SAAS,OAAO,OAAO,UAAU;AAAA,IAC9C;AAAA,EACF;AAAA,EAEA,MAAM,yBAAyB;AAC7B,UAAM,WAAW,YAAY;AAC3B,YAAM,8CAA8C;AACpD,YAAM,KAAK,KAAK;AAChB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,YAAQ,GAAG,UAAU,QAAQ;AAC7B,YAAQ,GAAG,WAAW,QAAQ;AAAA,EAChC;AACF;;;ACtTA,YAAYA,aAAY;AACxB,OAAOC,YAAW;AAClB,SAAS,WAAAC,gBAAe;AAIxB,IAAM,wBAAN,MAA4B;AAAA,EAC1B;AAAA,EACA,YAA4C,CAAC;AAAA,EAC7C;AAAA,EACA,gBAA0C,oBAAI,IAAI;AAAA,EAElD,MAAM,KAAK,QAA4B;AACrC,IAAAD,OAAM,8CAA8C,MAAM;AAE1D,SAAK,gBAAgB,MAAM;AAE3B,SAAK,kBAAkB,OAAO;AAC9B,SAAK,YAAY,OAAO;AACxB,SAAK,qBAAqB,OAAO;AAEjC,QAAI,OAAO,QAAQ;AACjB,iBAAW,CAAC,OAAO,WAAW,KAAK,OAAO,QAAQ,OAAO,MAAM,GAAG;AAChE,aAAK,cAAc,IAAI,OAAO,WAA0B;AAAA,MAC1D;AAAA,IACF;AAEA,UAAMC,SAAQ,iBAAiB,OAAO,SAAS;AAE/C,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,SAA2B;AAC7B,QAAI,CAAC,SAAS;AACZ,gBAAU,KAAK;AAAA,IACjB;AAEA,UAAM,kBAAkB,KAAK,UAAU,OAAO;AAE9C,QAAI,CAAC,iBAAiB;AACpB,YAAM,IAAW,8BAAsB,CAAC,YAAY,OAAO,qBAAqB,CAAC;AAAA,IACnF;AAEA,IAAAD,OAAM,sBAAsB,OAAO;AAEnC,QAAI;AACF,aAAO,gBAAgB;AAAA,IACzB,SAAS,OAAO;AAEd,YAAM,IAAI,MAAM;AAAA,IAElB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,qBAAqB,OAAe,gBAA2C;AAC7E,UAAM,cAAc,KAAK,cAAc,IAAI,KAAK;AAChD,UAAM,mBAAmB,aAAa,SAAS,CAAC;AAEhD,QAAI,aACF,gBAAgB,cAChB,iBAAiB,cACjB,KAAK,oBAAoB,cACzB;AAEF,QAAI,UACF,gBAAgB,WAAW,iBAAiB,WAAW,KAAK,oBAAoB;AAElF,WAAO,EAAE,YAAY,QAAQ;AAAA,EAC/B;AAAA,EAEA,gBAAgB,QAAkC;AAChD,QAAI,CAAC,OAAO,YAAY,OAAO,KAAK,OAAO,QAAQ,EAAE,WAAW,GAAG;AACjE,YAAM,IAAW,8BAAsB,CAAC,yCAAyC,CAAC;AAAA,IACpF;AAEA,QAAI,CAAC,OAAO,SAAS;AACnB,YAAM,IAAW,8BAAsB,CAAC,mCAAmC,CAAC;AAAA,IAC9E;AAEA,QAAI,CAAC,OAAO,aAAa,OAAO,UAAU,WAAW,GAAG;AACtD,YAAM,IAAW,8BAAsB,CAAC,iCAAiC,CAAC;AAAA,IAC5E;AAEA,QAAI,CAAC,OAAO,SAAS,OAAO,OAAO,GAAG;AACpC,YAAM,IAAW,8BAAsB;AAAA,QACrC,oBAAoB,OAAO,OAAO;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,eAAW,CAAC,MAAM,OAAO,KAAK,OAAO,QAAQ,OAAO,QAAQ,GAAG;AAC7D,UAAI,OAAO,YAAY,YAAY;AACjC,cAAM,IAAW,8BAAsB,CAAC,YAAY,IAAI,8BAA8B,CAAC;AAAA,MACzF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AACd,eAAW,eAAe,KAAK,WAAW;AACxC,YAAM,UAAU,KAAK,UAAU,WAAW,EAAE;AAC5C,YAAM,QAAQ,QAAQ;AAAA,IACxB;AAAA,EACF;AACF;AAEO,IAAME,gBAAe,IAAI,sBAAsB;;;AC1GtD,YAAYC,aAAY;AACxB,SAAS,SAAAC,cAAa;AACtB,SAAS,wBAAwB;AACjC,SAAS,yBAAyB;AAE3B,IAAM,kBAAN,MAAsB;AAAA,EAClB;AAAA,EAET,YAAY,QAAuB;AACjC,SAAK,UAAU;AACf,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,eAAe,SAAyB;AACtC,QAAI,UAAU,GAAG;AACf,YAAM,IAAI,iBAAiB,6BAA6B;AAAA,IAC1D;AAEA,UAAM,cAAcA,OAAM,KAAK,QAAQ,SAAS;AAChD,UAAM,aAAa,KAAK,QAAQ,WAAWA,OAAM,KAAK,QAAQ,QAAQ,IAAI;AAC1E,UAAM,aAAa,KAAK,QAAQ,cAAc;AAE9C,QAAI;AAEJ,YAAQ,KAAK,QAAQ,UAAU;AAAA,MAC7B,KAAK;AACH,gBAAQ,cAAc,KAAK,IAAI,YAAY,UAAU,CAAC;AACtD;AAAA,MACF,KAAK;AACH,gBAAQ,cAAc;AACtB;AAAA,MACF,KAAK;AACH,gBAAQ;AACR;AAAA,MACF;AACE,0BAAkB,KAAK,QAAQ,QAAQ;AAAA,IAC3C;AAGA,YAAQ,KAAK,IAAI,OAAO,UAAU;AAElC,QAAI,KAAK,QAAQ,QAAQ;AACvB,cAAQ,KAAK,aAAa,KAAK;AAAA,IACjC;AAEA,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,eAAe,SAAuB;AACpC,UAAM,QAAQ,KAAK,eAAe,OAAO;AACzC,WAAO,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK;AAAA,EACpC;AAAA,EAEA,YAAqC;AACnC,WAAO,OAAO,OAAO,EAAE,GAAG,KAAK,QAAQ,CAAC;AAAA,EAC1C;AAAA,EAEA,kBAAkB;AAChB,UAAM,cAAcA,OAAM,KAAK,QAAQ,SAAS;AAEhD,QAAI,eAAe,GAAG;AACpB,YAAM,IAAW,6BAAqB;AAAA,QACpC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,UAAU;AACzB,YAAM,aAAaA,OAAM,KAAK,QAAQ,QAAQ;AAE9C,UAAI,cAAc,GAAG;AACnB,cAAM,IAAW,4BAAoB;AAAA,UACnC;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,cAAc,aAAa;AAC7B,cAAM,IAAW,4BAAoB,CAAC,6CAA6C,CAAC;AAAA,MACtF;AAAA,IACF;AAEA,QAAI,KAAK,QAAQ,eAAe,QAAW;AACzC,UAAI,KAAK,QAAQ,cAAc,GAAG;AAChC,cAAM,IAAW,6BAAqB;AAAA,UACpC;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,KAAK,QAAQ,aAAa,iBAAiB,KAAK,QAAQ,aAAa,GAAG;AAC1E,cAAM,IAAW,6BAAqB,CAAC,gDAAgD,CAAC;AAAA,MAC1F;AAAA,IACF;AAAA,EACF;AAAA,EAEA,aAAa,OAAuB;AAClC,UAAM,cAAc,QAAQ;AAC5B,UAAM,UAAU,KAAK,OAAO,IAAI,OAAO,IAAI;AAE3C,WAAO,KAAK,IAAI,GAAG,QAAQ,MAAM;AAAA,EACnC;AACF;AAEO,SAAS,mBAAmB,QAAmD;AACpF,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,IACX,UAAU;AAAA,IACV,YAAY;AAAA,IACZ,QAAQ;AAAA,IACR,GAAG;AAAA,EACL,CAAC;AACL;AAEO,SAAS,cAAc,QAAmD;AAC/E,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,IACX,UAAU;AAAA,IACV,GAAG;AAAA,EACL,CAAC;AACL;AAEO,SAAS,aAAa,QAAkB,OAAO;AACpD,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,EACb,CAAC;AACL;AAEO,SAAS,cAAc,QAAuB;AACnD,SAAO,MAAM,IAAI,gBAAgB,MAAM;AACzC;","names":["errors","debug","Locator","QueueManager","errors","parse"]}
|
|
1
|
+
{"version":3,"sources":["../src/job.ts","../src/worker.ts","../src/queue_manager.ts","../src/strategies/backoff_strategy.ts"],"sourcesContent":["import { JobDispatcher } from '#src/job_dispatcher'\nimport type { JobOptions } from '#types/main'\n\nexport abstract class Job<Payload = any> {\n readonly #payload: Payload\n\n static options: JobOptions = {}\n\n get payload(): Payload {\n return this.#payload\n }\n\n constructor(payload: Payload) {\n this.#payload = payload\n }\n\n static dispatch<T extends Job>(\n this: new (payload: any) => T,\n payload: T extends Job<infer P> ? P : never\n ): JobDispatcher<T extends Job<infer P> ? P : never> {\n const dispatcher = new JobDispatcher<T extends Job<infer P> ? P : never>(\n (this as any).jobName,\n payload\n )\n\n if ((this as any).options.queue) {\n dispatcher.toQueue((this as any).options.queue)\n }\n\n if ((this as any).options.adapter) {\n dispatcher.with((this as any).options.adapter)\n }\n\n if ((this as any).options.priority !== undefined) {\n dispatcher.priority((this as any).options.priority)\n }\n\n return dispatcher\n }\n\n abstract execute(signal?: AbortSignal): Promise<void>\n\n failed?(error: Error): Promise<void>\n}\n","import { randomUUID } from 'node:crypto'\nimport { setTimeout } from 'node:timers/promises'\nimport debug from '#src/debug'\nimport { parse } from '#src/utils'\nimport * as errors from '#src/exceptions'\nimport { QueueManager } from '#src/queue_manager'\nimport { JobPool } from '#src/job_pool'\nimport type { Adapter, AcquiredJob } from '#contracts/adapter'\nimport type { QueueManagerConfig, WorkerCycle } from '#types/main'\nimport { Locator } from '#src/locator'\nimport type { JobOptions } from '#types/main'\nimport type { Job } from '#src/job'\n\nexport class Worker {\n readonly #id: string\n readonly #config: QueueManagerConfig\n #adapter!: Adapter\n #running = false\n #initialized = false\n #generator?: AsyncGenerator<WorkerCycle, void, unknown>\n #pool?: JobPool\n\n get id() {\n return this.#id\n }\n\n constructor(config: QueueManagerConfig) {\n this.#config = config\n this.#id = randomUUID()\n\n debug('created worker with id %s and config %O', this.#id, config)\n }\n\n async init() {\n if (this.#initialized) {\n return\n }\n\n debug('initializing worker %s', this.#id)\n\n await QueueManager.init(this.#config)\n\n this.#adapter = QueueManager.use()\n this.#adapter.setWorkerId(this.#id)\n\n this.#initialized = true\n\n debug('worker %s initialized', this.#id)\n }\n\n async start(queues: string[] = ['default']): Promise<void> {\n await this.init()\n\n if (this.#running) {\n debug('worker %s is already running', this.#id)\n return\n }\n\n this.#running = true\n\n debug('starting worker %s on queues: %O', this.#id, queues)\n\n await this.#setupGracefulShutdown()\n\n for await (const cycle of this.process(queues)) {\n if (['started', 'completed'].includes(cycle.type)) {\n continue\n }\n\n if (['idle', 'error'].includes(cycle.type)) {\n // @ts-expect-error - we know suggestedDelay exists for these types\n const delay = parse(cycle.suggestedDelay)\n\n if (cycle.type === 'error') {\n debug('worker %s encountered an error: %O', this.#id, cycle.error)\n } else {\n debug('worker %s is idle, waiting for %dms', this.#id, delay)\n }\n\n await setTimeout(delay)\n }\n }\n }\n\n async stop() {\n debug('stopping worker %s', this.#id)\n\n this.#running = false\n\n if (this.#pool) {\n debug('worker %s: waiting for %d running jobs to complete', this.#id, this.#pool.size)\n await this.#pool.drain()\n }\n\n if (this.#adapter) {\n await this.#adapter.destroy()\n }\n }\n\n async processCycle(queues: string[]): Promise<WorkerCycle | null> {\n await this.init()\n\n this.#running = true\n\n if (!this.#generator) {\n this.#generator = this.process(queues)\n }\n\n const result = await this.#generator.next()\n\n if (result.done) {\n this.#generator = undefined\n return null\n }\n\n return result.value\n }\n\n async *process(queues: string[]): AsyncGenerator<WorkerCycle, void, unknown> {\n const pollingInterval = parse(this.#config.worker?.pollingInterval || '2s')\n this.#pool = new JobPool()\n\n while (this.#running) {\n try {\n yield* this.#fillPool(queues)\n\n if (this.#pool.isEmpty()) {\n yield { type: 'idle', suggestedDelay: pollingInterval }\n continue\n }\n\n const completed = await this.#pool.waitForNextCompletion()\n yield { type: 'completed', queue: completed.queue, job: completed.job }\n } catch (error) {\n yield { type: 'error', error: error as Error, suggestedDelay: parse('5s') }\n }\n }\n }\n\n async *#fillPool(queues: string[]): AsyncGenerator<WorkerCycle, void, unknown> {\n const concurrency = this.#config.worker?.concurrency || 1\n\n while (this.#pool!.hasCapacity(concurrency)) {\n const result = await this.#acquireNextJob(queues)\n if (!result) break\n\n const { job, queue } = result\n const promise = this.#execute(job, queue)\n this.#pool!.add(job, queue, promise)\n\n yield { type: 'started', queue, job }\n }\n }\n\n async #execute(job: AcquiredJob, queue: string): Promise<void> {\n const startTime = performance.now()\n\n debug('worker %s: executing job %s (%s)', this.#id, job.id, job.name)\n\n const { instance, options, timeout } = await this.#initJob(job, queue)\n\n try {\n await this.#executeWithTimeout(instance, timeout)\n await this.#adapter.completeJob(job.id, queue)\n\n const duration = (performance.now() - startTime).toFixed(2)\n debug('worker %s: successfully executed job %s in %dms', this.#id, job.id, duration)\n } catch (e) {\n const isTimeout = e instanceof errors.E_JOB_TIMEOUT\n\n if (isTimeout && options.failOnTimeout) {\n debug('worker %s: job %s timed out and failOnTimeout is set', this.#id, job.id)\n await this.#adapter.failJob(job.id, queue, e as Error)\n await instance.failed?.(e as Error)\n return\n }\n\n const mergedConfig = QueueManager.getMergedRetryConfig(queue, options.retry)\n\n if (typeof mergedConfig.maxRetries === 'undefined' || mergedConfig.maxRetries <= 0) {\n debug('worker %s: job %s has no retries configured, marking as failed', this.#id, job.id)\n await this.#adapter.failJob(job.id, queue, e as Error)\n await instance.failed?.(e as Error)\n return\n }\n\n if (job.attempts >= mergedConfig.maxRetries!) {\n debug(\n 'worker %s: job %s has exceeded max retries (%d), marking as failed',\n this.#id,\n job.id,\n mergedConfig.maxRetries\n )\n await this.#adapter.failJob(job.id, queue, e as Error)\n const exception = new errors.E_JOB_MAX_ATTEMPTS_REACHED([job.name])\n await instance.failed?.(exception)\n\n return\n }\n\n if (mergedConfig.backoff) {\n const strategy = mergedConfig.backoff()\n const nextRetryAt = strategy.getNextRetryAt(job.attempts + 1)\n\n debug('worker %s: job %s will retry at %s', this.#id, job.id, nextRetryAt.toISOString())\n\n await this.#adapter.retryJob(job.id, queue, nextRetryAt)\n return\n }\n\n await this.#adapter.retryJob(job.id, queue)\n }\n }\n\n async #initJob(\n job: AcquiredJob,\n queue: string\n ): Promise<{ instance: Job; options: JobOptions; timeout: number | undefined }> {\n try {\n const JobClass = Locator.getOrThrow(job.name)\n const instance = new JobClass(job.payload)\n const options = JobClass.options || {}\n const timeout = this.#getJobTimeout(options)\n\n return { instance, options, timeout }\n } catch (error) {\n debug('worker %s: failed to initialize job %s (%s)', this.#id, job.id, job.name)\n await this.#adapter.failJob(job.id, queue, error as Error)\n throw error\n }\n }\n\n #getJobTimeout(options: JobOptions): number | undefined {\n if (options.timeout !== undefined) {\n return parse(options.timeout)\n }\n\n if (this.#config.worker?.timeout !== undefined) {\n return parse(this.#config.worker.timeout)\n }\n\n return undefined\n }\n\n async #executeWithTimeout(instance: Job, timeout?: number): Promise<void> {\n if (!timeout) {\n return instance.execute()\n }\n\n const signal = AbortSignal.timeout(timeout)\n\n const abortPromise = new Promise<never>((_, reject) => {\n signal.addEventListener('abort', () => {\n reject(new errors.E_JOB_TIMEOUT([instance.constructor.name, timeout]))\n })\n })\n\n await Promise.race([instance.execute(signal), abortPromise])\n }\n\n async #acquireNextJob(queues: string[]): Promise<{ job: AcquiredJob; queue: string } | null> {\n for (const queue of queues) {\n const job = await this.#adapter.popFrom(queue)\n\n if (!job) {\n continue\n }\n\n debug('worker %s: acquired job %s', this.#id, job.id)\n return { job, queue }\n }\n\n return null\n }\n\n async #setupGracefulShutdown() {\n const shutdown = async () => {\n debug('received shutdown signal, stopping worker...')\n await this.stop()\n process.exit(0)\n }\n\n process.on('SIGINT', shutdown)\n process.on('SIGTERM', shutdown)\n }\n}\n","import * as errors from '#src/exceptions'\nimport debug from '#src/debug'\nimport { Locator } from '#src/locator'\nimport type { Adapter } from '#contracts/adapter'\nimport type { AdapterFactory, QueueConfig, QueueManagerConfig, RetryConfig } from '#types/main'\n\nclass QueueManagerSingleton {\n #defaultAdapter!: string\n #adapters: Record<string, AdapterFactory> = {}\n #globalRetryConfig?: RetryConfig\n #queueConfigs: Map<string, QueueConfig> = new Map()\n\n async init(config: QueueManagerConfig) {\n debug('initializing queue manager with config: %O', config)\n\n this.#validateConfig(config)\n\n this.#defaultAdapter = config.default\n this.#adapters = config.adapters\n this.#globalRetryConfig = config.retry\n\n if (config.queues) {\n for (const [queue, queueConfig] of Object.entries(config.queues)) {\n this.#queueConfigs.set(queue, queueConfig as QueueConfig)\n }\n }\n\n await Locator.registerFromGlob(config.locations)\n\n return this\n }\n\n use(adapter?: string): Adapter {\n if (!adapter) {\n adapter = this.#defaultAdapter\n }\n\n const adapterInstance = this.#adapters[adapter]\n\n if (!adapterInstance) {\n throw new errors.E_CONFIGURATION_ERROR([`Adapter \"${adapter}\" is not registered`])\n }\n\n debug('using adapter \"%s\"', adapter)\n\n try {\n return adapterInstance()\n } catch (error) {\n // TODO: Improve error handling\n throw new Error()\n // throw new errors.E_ADAPTER_ERROR(`Failed to initialize adapter \"${adapter}\"`, error as Error)\n }\n }\n\n /**\n * Priority: job > queue > global\n */\n getMergedRetryConfig(queue: string, jobRetryConfig?: RetryConfig): RetryConfig {\n const queueConfig = this.#queueConfigs.get(queue)\n const queueRetryConfig = queueConfig?.retry || {}\n\n let maxRetries =\n jobRetryConfig?.maxRetries ||\n queueRetryConfig.maxRetries ||\n this.#globalRetryConfig?.maxRetries ||\n 0\n\n let backoff =\n jobRetryConfig?.backoff || queueRetryConfig.backoff || this.#globalRetryConfig?.backoff\n\n return { maxRetries, backoff }\n }\n\n #validateConfig(config: QueueManagerConfig): void {\n if (!config.adapters || Object.keys(config.adapters).length === 0) {\n throw new errors.E_CONFIGURATION_ERROR(['At least one adapter must be configured'])\n }\n\n if (!config.default) {\n throw new errors.E_CONFIGURATION_ERROR(['Default adapter must be specified'])\n }\n\n if (!config.locations || config.locations.length === 0) {\n throw new errors.E_CONFIGURATION_ERROR(['Job locations must be specified'])\n }\n\n if (!config.adapters[config.default]) {\n throw new errors.E_CONFIGURATION_ERROR([\n `Default adapter \"${config.default}\" not found in adapters configuration`,\n ])\n }\n\n for (const [name, factory] of Object.entries(config.adapters)) {\n if (typeof factory !== 'function') {\n throw new errors.E_CONFIGURATION_ERROR([`Adapter \"${name}\" must be a factory function`])\n }\n }\n }\n\n async destroy() {\n for (const adapterName in this.#adapters) {\n const adapter = this.#adapters[adapterName]()\n await adapter.destroy()\n }\n }\n}\n\nexport const QueueManager = new QueueManagerSingleton()\n","import type { BackoffConfig, Duration } from '#types/main'\nimport * as errors from '#src/exceptions'\nimport { parse } from '#src/utils'\nimport { RuntimeException } from '@poppinss/utils'\nimport { assertUnreachable } from '@poppinss/utils/assert'\n\nexport class BackoffStrategy {\n readonly #config: BackoffConfig\n\n constructor(config: BackoffConfig) {\n this.#config = config\n this.#validateConfig()\n }\n\n calculateDelay(attempt: number): number {\n if (attempt < 1) {\n throw new RuntimeException('Attempt number must be >= 1')\n }\n\n const baseDelayMs = parse(this.#config.baseDelay)\n const maxDelayMs = this.#config.maxDelay ? parse(this.#config.maxDelay) : Infinity\n const multiplier = this.#config.multiplier ?? 2\n\n let delay: number\n\n switch (this.#config.strategy) {\n case 'exponential':\n delay = baseDelayMs * Math.pow(multiplier, attempt - 1)\n break\n case 'linear':\n delay = baseDelayMs * attempt\n break\n case 'fixed':\n delay = baseDelayMs\n break\n default:\n assertUnreachable(this.#config.strategy)\n }\n\n // Apply max delay limit\n delay = Math.min(delay, maxDelayMs)\n\n if (this.#config.jitter) {\n delay = this.#applyJitter(delay)\n }\n\n return Math.floor(delay)\n }\n\n getNextRetryAt(attempt: number): Date {\n const delay = this.calculateDelay(attempt)\n return new Date(Date.now() + delay)\n }\n\n getConfig(): Readonly<BackoffConfig> {\n return Object.freeze({ ...this.#config })\n }\n\n #validateConfig() {\n const baseDelayMs = parse(this.#config.baseDelay)\n\n if (baseDelayMs <= 0) {\n throw new errors.E_INVALID_BASE_DELAY([\n 'Base delay must be a positive integer greater than zero',\n ])\n }\n\n if (this.#config.maxDelay) {\n const maxDelayMs = parse(this.#config.maxDelay)\n\n if (maxDelayMs <= 0) {\n throw new errors.E_INVALID_MAX_DELAY([\n 'Max delay must be a positive integer greater than zero',\n ])\n }\n\n if (maxDelayMs <= baseDelayMs) {\n throw new errors.E_INVALID_MAX_DELAY(['Max delay should be greater than base delay'])\n }\n }\n\n if (this.#config.multiplier !== undefined) {\n if (this.#config.multiplier <= 0) {\n throw new errors.E_INVALID_MULTIPLIER([\n 'Multiplier must be a positive number greater than zero',\n ])\n }\n\n if (this.#config.strategy === 'exponential' && this.#config.multiplier < 1) {\n throw new errors.E_INVALID_MULTIPLIER(['Exponential strategy multiplier should be >= 1'])\n }\n }\n }\n\n #applyJitter(delay: number): number {\n const jitterRange = delay * 0.25\n const jitter = (Math.random() - 0.5) * 2 * jitterRange\n\n return Math.max(0, delay + jitter)\n }\n}\n\nexport function exponentialBackoff(config?: Partial<Omit<BackoffConfig, 'strategy'>>) {\n return () =>\n new BackoffStrategy({\n strategy: 'exponential',\n baseDelay: '1s',\n maxDelay: '5m',\n multiplier: 2,\n jitter: true,\n ...config,\n })\n}\n\nexport function linearBackoff(config?: Partial<Omit<BackoffConfig, 'strategy'>>) {\n return () =>\n new BackoffStrategy({\n strategy: 'linear',\n baseDelay: '5s',\n maxDelay: '2m',\n ...config,\n })\n}\n\nexport function fixedBackoff(delay: Duration = '10s') {\n return () =>\n new BackoffStrategy({\n strategy: 'fixed',\n baseDelay: delay,\n })\n}\n\nexport function customBackoff(config: BackoffConfig) {\n return () => new BackoffStrategy(config)\n}\n"],"mappings":";AAAA,SAAS,qBAAqB;AAGvB,IAAe,MAAf,MAAkC;AAAA,EAC9B;AAAA,EAET,OAAO,UAAsB,CAAC;AAAA,EAE9B,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,YAAY,SAAkB;AAC5B,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,OAAO,SAEL,SACmD;AACnD,UAAM,aAAa,IAAI;AAAA,MACpB,KAAa;AAAA,MACd;AAAA,IACF;AAEA,QAAK,KAAa,QAAQ,OAAO;AAC/B,iBAAW,QAAS,KAAa,QAAQ,KAAK;AAAA,IAChD;AAEA,QAAK,KAAa,QAAQ,SAAS;AACjC,iBAAW,KAAM,KAAa,QAAQ,OAAO;AAAA,IAC/C;AAEA,QAAK,KAAa,QAAQ,aAAa,QAAW;AAChD,iBAAW,SAAU,KAAa,QAAQ,QAAQ;AAAA,IACpD;AAEA,WAAO;AAAA,EACT;AAKF;;;AC3CA,SAAS,kBAAkB;AAC3B,SAAS,kBAAkB;AAC3B,OAAO,WAAW;AAClB,SAAS,aAAa;AACtB,YAAY,YAAY;AACxB,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AAGxB,SAAS,eAAe;AAIjB,IAAM,SAAN,MAAa;AAAA,EACT;AAAA,EACA;AAAA,EACT;AAAA,EACA,WAAW;AAAA,EACX,eAAe;AAAA,EACf;AAAA,EACA;AAAA,EAEA,IAAI,KAAK;AACP,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,YAAY,QAA4B;AACtC,SAAK,UAAU;AACf,SAAK,MAAM,WAAW;AAEtB,UAAM,2CAA2C,KAAK,KAAK,MAAM;AAAA,EACnE;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,KAAK,cAAc;AACrB;AAAA,IACF;AAEA,UAAM,0BAA0B,KAAK,GAAG;AAExC,UAAM,aAAa,KAAK,KAAK,OAAO;AAEpC,SAAK,WAAW,aAAa,IAAI;AACjC,SAAK,SAAS,YAAY,KAAK,GAAG;AAElC,SAAK,eAAe;AAEpB,UAAM,yBAAyB,KAAK,GAAG;AAAA,EACzC;AAAA,EAEA,MAAM,MAAM,SAAmB,CAAC,SAAS,GAAkB;AACzD,UAAM,KAAK,KAAK;AAEhB,QAAI,KAAK,UAAU;AACjB,YAAM,gCAAgC,KAAK,GAAG;AAC9C;AAAA,IACF;AAEA,SAAK,WAAW;AAEhB,UAAM,oCAAoC,KAAK,KAAK,MAAM;AAE1D,UAAM,KAAK,uBAAuB;AAElC,qBAAiB,SAAS,KAAK,QAAQ,MAAM,GAAG;AAC9C,UAAI,CAAC,WAAW,WAAW,EAAE,SAAS,MAAM,IAAI,GAAG;AACjD;AAAA,MACF;AAEA,UAAI,CAAC,QAAQ,OAAO,EAAE,SAAS,MAAM,IAAI,GAAG;AAE1C,cAAM,QAAQ,MAAM,MAAM,cAAc;AAExC,YAAI,MAAM,SAAS,SAAS;AAC1B,gBAAM,sCAAsC,KAAK,KAAK,MAAM,KAAK;AAAA,QACnE,OAAO;AACL,gBAAM,uCAAuC,KAAK,KAAK,KAAK;AAAA,QAC9D;AAEA,cAAM,WAAW,KAAK;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,sBAAsB,KAAK,GAAG;AAEpC,SAAK,WAAW;AAEhB,QAAI,KAAK,OAAO;AACd,YAAM,sDAAsD,KAAK,KAAK,KAAK,MAAM,IAAI;AACrF,YAAM,KAAK,MAAM,MAAM;AAAA,IACzB;AAEA,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,QAAQ;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,QAA+C;AAChE,UAAM,KAAK,KAAK;AAEhB,SAAK,WAAW;AAEhB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,KAAK,QAAQ,MAAM;AAAA,IACvC;AAEA,UAAM,SAAS,MAAM,KAAK,WAAW,KAAK;AAE1C,QAAI,OAAO,MAAM;AACf,WAAK,aAAa;AAClB,aAAO;AAAA,IACT;AAEA,WAAO,OAAO;AAAA,EAChB;AAAA,EAEA,OAAO,QAAQ,QAA8D;AAC3E,UAAM,kBAAkB,MAAM,KAAK,QAAQ,QAAQ,mBAAmB,IAAI;AAC1E,SAAK,QAAQ,IAAI,QAAQ;AAEzB,WAAO,KAAK,UAAU;AACpB,UAAI;AACF,eAAO,KAAK,UAAU,MAAM;AAE5B,YAAI,KAAK,MAAM,QAAQ,GAAG;AACxB,gBAAM,EAAE,MAAM,QAAQ,gBAAgB,gBAAgB;AACtD;AAAA,QACF;AAEA,cAAM,YAAY,MAAM,KAAK,MAAM,sBAAsB;AACzD,cAAM,EAAE,MAAM,aAAa,OAAO,UAAU,OAAO,KAAK,UAAU,IAAI;AAAA,MACxE,SAAS,OAAO;AACd,cAAM,EAAE,MAAM,SAAS,OAAuB,gBAAgB,MAAM,IAAI,EAAE;AAAA,MAC5E;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAO,UAAU,QAA8D;AAC7E,UAAM,cAAc,KAAK,QAAQ,QAAQ,eAAe;AAExD,WAAO,KAAK,MAAO,YAAY,WAAW,GAAG;AAC3C,YAAM,SAAS,MAAM,KAAK,gBAAgB,MAAM;AAChD,UAAI,CAAC,OAAQ;AAEb,YAAM,EAAE,KAAK,MAAM,IAAI;AACvB,YAAM,UAAU,KAAK,SAAS,KAAK,KAAK;AACxC,WAAK,MAAO,IAAI,KAAK,OAAO,OAAO;AAEnC,YAAM,EAAE,MAAM,WAAW,OAAO,IAAI;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,KAAkB,OAA8B;AAC7D,UAAM,YAAY,YAAY,IAAI;AAElC,UAAM,oCAAoC,KAAK,KAAK,IAAI,IAAI,IAAI,IAAI;AAEpE,UAAM,EAAE,UAAU,SAAS,QAAQ,IAAI,MAAM,KAAK,SAAS,KAAK,KAAK;AAErE,QAAI;AACF,YAAM,KAAK,oBAAoB,UAAU,OAAO;AAChD,YAAM,KAAK,SAAS,YAAY,IAAI,IAAI,KAAK;AAE7C,YAAM,YAAY,YAAY,IAAI,IAAI,WAAW,QAAQ,CAAC;AAC1D,YAAM,mDAAmD,KAAK,KAAK,IAAI,IAAI,QAAQ;AAAA,IACrF,SAAS,GAAG;AACV,YAAM,YAAY,aAAoB;AAEtC,UAAI,aAAa,QAAQ,eAAe;AACtC,cAAM,wDAAwD,KAAK,KAAK,IAAI,EAAE;AAC9E,cAAM,KAAK,SAAS,QAAQ,IAAI,IAAI,OAAO,CAAU;AACrD,cAAM,SAAS,SAAS,CAAU;AAClC;AAAA,MACF;AAEA,YAAM,eAAe,aAAa,qBAAqB,OAAO,QAAQ,KAAK;AAE3E,UAAI,OAAO,aAAa,eAAe,eAAe,aAAa,cAAc,GAAG;AAClF,cAAM,kEAAkE,KAAK,KAAK,IAAI,EAAE;AACxF,cAAM,KAAK,SAAS,QAAQ,IAAI,IAAI,OAAO,CAAU;AACrD,cAAM,SAAS,SAAS,CAAU;AAClC;AAAA,MACF;AAEA,UAAI,IAAI,YAAY,aAAa,YAAa;AAC5C;AAAA,UACE;AAAA,UACA,KAAK;AAAA,UACL,IAAI;AAAA,UACJ,aAAa;AAAA,QACf;AACA,cAAM,KAAK,SAAS,QAAQ,IAAI,IAAI,OAAO,CAAU;AACrD,cAAM,YAAY,IAAW,kCAA2B,CAAC,IAAI,IAAI,CAAC;AAClE,cAAM,SAAS,SAAS,SAAS;AAEjC;AAAA,MACF;AAEA,UAAI,aAAa,SAAS;AACxB,cAAM,WAAW,aAAa,QAAQ;AACtC,cAAM,cAAc,SAAS,eAAe,IAAI,WAAW,CAAC;AAE5D,cAAM,sCAAsC,KAAK,KAAK,IAAI,IAAI,YAAY,YAAY,CAAC;AAEvF,cAAM,KAAK,SAAS,SAAS,IAAI,IAAI,OAAO,WAAW;AACvD;AAAA,MACF;AAEA,YAAM,KAAK,SAAS,SAAS,IAAI,IAAI,KAAK;AAAA,IAC5C;AAAA,EACF;AAAA,EAEA,MAAM,SACJ,KACA,OAC8E;AAC9E,QAAI;AACF,YAAM,WAAW,QAAQ,WAAW,IAAI,IAAI;AAC5C,YAAM,WAAW,IAAI,SAAS,IAAI,OAAO;AACzC,YAAM,UAAU,SAAS,WAAW,CAAC;AACrC,YAAM,UAAU,KAAK,eAAe,OAAO;AAE3C,aAAO,EAAE,UAAU,SAAS,QAAQ;AAAA,IACtC,SAAS,OAAO;AACd,YAAM,+CAA+C,KAAK,KAAK,IAAI,IAAI,IAAI,IAAI;AAC/E,YAAM,KAAK,SAAS,QAAQ,IAAI,IAAI,OAAO,KAAc;AACzD,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,eAAe,SAAyC;AACtD,QAAI,QAAQ,YAAY,QAAW;AACjC,aAAO,MAAM,QAAQ,OAAO;AAAA,IAC9B;AAEA,QAAI,KAAK,QAAQ,QAAQ,YAAY,QAAW;AAC9C,aAAO,MAAM,KAAK,QAAQ,OAAO,OAAO;AAAA,IAC1C;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAAoB,UAAe,SAAiC;AACxE,QAAI,CAAC,SAAS;AACZ,aAAO,SAAS,QAAQ;AAAA,IAC1B;AAEA,UAAM,SAAS,YAAY,QAAQ,OAAO;AAE1C,UAAM,eAAe,IAAI,QAAe,CAAC,GAAG,WAAW;AACrD,aAAO,iBAAiB,SAAS,MAAM;AACrC,eAAO,IAAW,qBAAc,CAAC,SAAS,YAAY,MAAM,OAAO,CAAC,CAAC;AAAA,MACvE,CAAC;AAAA,IACH,CAAC;AAED,UAAM,QAAQ,KAAK,CAAC,SAAS,QAAQ,MAAM,GAAG,YAAY,CAAC;AAAA,EAC7D;AAAA,EAEA,MAAM,gBAAgB,QAAuE;AAC3F,eAAW,SAAS,QAAQ;AAC1B,YAAM,MAAM,MAAM,KAAK,SAAS,QAAQ,KAAK;AAE7C,UAAI,CAAC,KAAK;AACR;AAAA,MACF;AAEA,YAAM,8BAA8B,KAAK,KAAK,IAAI,EAAE;AACpD,aAAO,EAAE,KAAK,MAAM;AAAA,IACtB;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,yBAAyB;AAC7B,UAAM,WAAW,YAAY;AAC3B,YAAM,8CAA8C;AACpD,YAAM,KAAK,KAAK;AAChB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,YAAQ,GAAG,UAAU,QAAQ;AAC7B,YAAQ,GAAG,WAAW,QAAQ;AAAA,EAChC;AACF;;;AC7RA,YAAYA,aAAY;AACxB,OAAOC,YAAW;AAClB,SAAS,WAAAC,gBAAe;AAIxB,IAAM,wBAAN,MAA4B;AAAA,EAC1B;AAAA,EACA,YAA4C,CAAC;AAAA,EAC7C;AAAA,EACA,gBAA0C,oBAAI,IAAI;AAAA,EAElD,MAAM,KAAK,QAA4B;AACrC,IAAAD,OAAM,8CAA8C,MAAM;AAE1D,SAAK,gBAAgB,MAAM;AAE3B,SAAK,kBAAkB,OAAO;AAC9B,SAAK,YAAY,OAAO;AACxB,SAAK,qBAAqB,OAAO;AAEjC,QAAI,OAAO,QAAQ;AACjB,iBAAW,CAAC,OAAO,WAAW,KAAK,OAAO,QAAQ,OAAO,MAAM,GAAG;AAChE,aAAK,cAAc,IAAI,OAAO,WAA0B;AAAA,MAC1D;AAAA,IACF;AAEA,UAAMC,SAAQ,iBAAiB,OAAO,SAAS;AAE/C,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,SAA2B;AAC7B,QAAI,CAAC,SAAS;AACZ,gBAAU,KAAK;AAAA,IACjB;AAEA,UAAM,kBAAkB,KAAK,UAAU,OAAO;AAE9C,QAAI,CAAC,iBAAiB;AACpB,YAAM,IAAW,8BAAsB,CAAC,YAAY,OAAO,qBAAqB,CAAC;AAAA,IACnF;AAEA,IAAAD,OAAM,sBAAsB,OAAO;AAEnC,QAAI;AACF,aAAO,gBAAgB;AAAA,IACzB,SAAS,OAAO;AAEd,YAAM,IAAI,MAAM;AAAA,IAElB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,qBAAqB,OAAe,gBAA2C;AAC7E,UAAM,cAAc,KAAK,cAAc,IAAI,KAAK;AAChD,UAAM,mBAAmB,aAAa,SAAS,CAAC;AAEhD,QAAI,aACF,gBAAgB,cAChB,iBAAiB,cACjB,KAAK,oBAAoB,cACzB;AAEF,QAAI,UACF,gBAAgB,WAAW,iBAAiB,WAAW,KAAK,oBAAoB;AAElF,WAAO,EAAE,YAAY,QAAQ;AAAA,EAC/B;AAAA,EAEA,gBAAgB,QAAkC;AAChD,QAAI,CAAC,OAAO,YAAY,OAAO,KAAK,OAAO,QAAQ,EAAE,WAAW,GAAG;AACjE,YAAM,IAAW,8BAAsB,CAAC,yCAAyC,CAAC;AAAA,IACpF;AAEA,QAAI,CAAC,OAAO,SAAS;AACnB,YAAM,IAAW,8BAAsB,CAAC,mCAAmC,CAAC;AAAA,IAC9E;AAEA,QAAI,CAAC,OAAO,aAAa,OAAO,UAAU,WAAW,GAAG;AACtD,YAAM,IAAW,8BAAsB,CAAC,iCAAiC,CAAC;AAAA,IAC5E;AAEA,QAAI,CAAC,OAAO,SAAS,OAAO,OAAO,GAAG;AACpC,YAAM,IAAW,8BAAsB;AAAA,QACrC,oBAAoB,OAAO,OAAO;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,eAAW,CAAC,MAAM,OAAO,KAAK,OAAO,QAAQ,OAAO,QAAQ,GAAG;AAC7D,UAAI,OAAO,YAAY,YAAY;AACjC,cAAM,IAAW,8BAAsB,CAAC,YAAY,IAAI,8BAA8B,CAAC;AAAA,MACzF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,UAAU;AACd,eAAW,eAAe,KAAK,WAAW;AACxC,YAAM,UAAU,KAAK,UAAU,WAAW,EAAE;AAC5C,YAAM,QAAQ,QAAQ;AAAA,IACxB;AAAA,EACF;AACF;AAEO,IAAME,gBAAe,IAAI,sBAAsB;;;AC1GtD,YAAYC,aAAY;AACxB,SAAS,SAAAC,cAAa;AACtB,SAAS,wBAAwB;AACjC,SAAS,yBAAyB;AAE3B,IAAM,kBAAN,MAAsB;AAAA,EAClB;AAAA,EAET,YAAY,QAAuB;AACjC,SAAK,UAAU;AACf,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,eAAe,SAAyB;AACtC,QAAI,UAAU,GAAG;AACf,YAAM,IAAI,iBAAiB,6BAA6B;AAAA,IAC1D;AAEA,UAAM,cAAcA,OAAM,KAAK,QAAQ,SAAS;AAChD,UAAM,aAAa,KAAK,QAAQ,WAAWA,OAAM,KAAK,QAAQ,QAAQ,IAAI;AAC1E,UAAM,aAAa,KAAK,QAAQ,cAAc;AAE9C,QAAI;AAEJ,YAAQ,KAAK,QAAQ,UAAU;AAAA,MAC7B,KAAK;AACH,gBAAQ,cAAc,KAAK,IAAI,YAAY,UAAU,CAAC;AACtD;AAAA,MACF,KAAK;AACH,gBAAQ,cAAc;AACtB;AAAA,MACF,KAAK;AACH,gBAAQ;AACR;AAAA,MACF;AACE,0BAAkB,KAAK,QAAQ,QAAQ;AAAA,IAC3C;AAGA,YAAQ,KAAK,IAAI,OAAO,UAAU;AAElC,QAAI,KAAK,QAAQ,QAAQ;AACvB,cAAQ,KAAK,aAAa,KAAK;AAAA,IACjC;AAEA,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,eAAe,SAAuB;AACpC,UAAM,QAAQ,KAAK,eAAe,OAAO;AACzC,WAAO,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK;AAAA,EACpC;AAAA,EAEA,YAAqC;AACnC,WAAO,OAAO,OAAO,EAAE,GAAG,KAAK,QAAQ,CAAC;AAAA,EAC1C;AAAA,EAEA,kBAAkB;AAChB,UAAM,cAAcA,OAAM,KAAK,QAAQ,SAAS;AAEhD,QAAI,eAAe,GAAG;AACpB,YAAM,IAAW,6BAAqB;AAAA,QACpC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,QAAQ,UAAU;AACzB,YAAM,aAAaA,OAAM,KAAK,QAAQ,QAAQ;AAE9C,UAAI,cAAc,GAAG;AACnB,cAAM,IAAW,4BAAoB;AAAA,UACnC;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,cAAc,aAAa;AAC7B,cAAM,IAAW,4BAAoB,CAAC,6CAA6C,CAAC;AAAA,MACtF;AAAA,IACF;AAEA,QAAI,KAAK,QAAQ,eAAe,QAAW;AACzC,UAAI,KAAK,QAAQ,cAAc,GAAG;AAChC,cAAM,IAAW,6BAAqB;AAAA,UACpC;AAAA,QACF,CAAC;AAAA,MACH;AAEA,UAAI,KAAK,QAAQ,aAAa,iBAAiB,KAAK,QAAQ,aAAa,GAAG;AAC1E,cAAM,IAAW,6BAAqB,CAAC,gDAAgD,CAAC;AAAA,MAC1F;AAAA,IACF;AAAA,EACF;AAAA,EAEA,aAAa,OAAuB;AAClC,UAAM,cAAc,QAAQ;AAC5B,UAAM,UAAU,KAAK,OAAO,IAAI,OAAO,IAAI;AAE3C,WAAO,KAAK,IAAI,GAAG,QAAQ,MAAM;AAAA,EACnC;AACF;AAEO,SAAS,mBAAmB,QAAmD;AACpF,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,IACX,UAAU;AAAA,IACV,YAAY;AAAA,IACZ,QAAQ;AAAA,IACR,GAAG;AAAA,EACL,CAAC;AACL;AAEO,SAAS,cAAc,QAAmD;AAC/E,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,IACX,UAAU;AAAA,IACV,GAAG;AAAA,EACL,CAAC;AACL;AAEO,SAAS,aAAa,QAAkB,OAAO;AACpD,SAAO,MACL,IAAI,gBAAgB;AAAA,IAClB,UAAU;AAAA,IACV,WAAW;AAAA,EACb,CAAC;AACL;AAEO,SAAS,cAAc,QAAuB;AACnD,SAAO,MAAM,IAAI,gBAAgB,MAAM;AACzC;","names":["errors","debug","Locator","QueueManager","errors","parse"]}
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { LeaseManager } from './src/contracts/lease_manager.js';
|
|
2
|
-
|
|
3
1
|
declare class BackoffStrategy$1 {
|
|
4
2
|
#private;
|
|
5
3
|
constructor(config: BackoffConfig);
|
|
@@ -13,12 +11,6 @@ declare function fixedBackoff(delay?: Duration): () => BackoffStrategy$1;
|
|
|
13
11
|
declare function customBackoff(config: BackoffConfig): () => BackoffStrategy$1;
|
|
14
12
|
|
|
15
13
|
type Duration = number | string;
|
|
16
|
-
interface AcquiredJob extends JobData {
|
|
17
|
-
_lease: {
|
|
18
|
-
commit: () => Promise<void>;
|
|
19
|
-
rollback: () => Promise<void>;
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
14
|
interface JobData {
|
|
23
15
|
id: string;
|
|
24
16
|
name: string;
|
|
@@ -33,8 +25,12 @@ interface JobOptions {
|
|
|
33
25
|
maxRetries?: number;
|
|
34
26
|
priority?: number;
|
|
35
27
|
retry?: RetryConfig;
|
|
28
|
+
timeout?: Duration;
|
|
29
|
+
failOnTimeout?: boolean;
|
|
36
30
|
}
|
|
37
|
-
type JobClass<T extends Job = Job> = new (payload: any) => T
|
|
31
|
+
type JobClass<T extends Job = Job> = (new (payload: any) => T) & {
|
|
32
|
+
options?: JobOptions;
|
|
33
|
+
};
|
|
38
34
|
interface RetryConfig {
|
|
39
35
|
maxRetries?: number;
|
|
40
36
|
backoff?: () => BackoffStrategy$1;
|
|
@@ -47,11 +43,6 @@ interface BackoffConfig {
|
|
|
47
43
|
multiplier?: number;
|
|
48
44
|
jitter?: boolean;
|
|
49
45
|
}
|
|
50
|
-
interface LeaseConfig {
|
|
51
|
-
workerId: string;
|
|
52
|
-
leaseTimeout: Duration;
|
|
53
|
-
renewalInterval: Duration;
|
|
54
|
-
}
|
|
55
46
|
interface QueueConfig {
|
|
56
47
|
adapter?: string;
|
|
57
48
|
retry?: any;
|
|
@@ -61,6 +52,7 @@ interface WorkerConfig {
|
|
|
61
52
|
pollingInterval?: Duration;
|
|
62
53
|
leaseTimeout?: Duration;
|
|
63
54
|
renewalInterval?: Duration;
|
|
55
|
+
timeout?: Duration;
|
|
64
56
|
}
|
|
65
57
|
type WorkerCycle = {
|
|
66
58
|
type: 'started';
|
|
@@ -88,16 +80,48 @@ interface QueueManagerConfig {
|
|
|
88
80
|
locations: string[];
|
|
89
81
|
}
|
|
90
82
|
|
|
83
|
+
interface AcquiredJob extends JobData {
|
|
84
|
+
acquiredAt: number;
|
|
85
|
+
}
|
|
91
86
|
interface Adapter {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
87
|
+
/**
|
|
88
|
+
* Set the worker ID for this adapter instance.
|
|
89
|
+
* Required before calling pop methods when consuming jobs.
|
|
90
|
+
*/
|
|
91
|
+
setWorkerId(workerId: string): void;
|
|
92
|
+
/**
|
|
93
|
+
* Pop the next available job from the default queue.
|
|
94
|
+
* The driver handles locking internally.
|
|
95
|
+
*/
|
|
96
|
+
pop(): Promise<AcquiredJob | null>;
|
|
97
|
+
/**
|
|
98
|
+
* Pop the next available job from a specific queue.
|
|
99
|
+
* The driver handles locking internally.
|
|
100
|
+
*/
|
|
101
|
+
popFrom(queue: string): Promise<AcquiredJob | null>;
|
|
102
|
+
/**
|
|
103
|
+
* Blocking pop that waits for a job to be available.
|
|
104
|
+
* Supported by Redis adapter.
|
|
105
|
+
*/
|
|
106
|
+
popAndWait?(queue: string, timeout: number): Promise<AcquiredJob | null>;
|
|
107
|
+
/**
|
|
108
|
+
* Mark a job as completed and remove it from active set.
|
|
109
|
+
*/
|
|
110
|
+
completeJob(jobId: string, queue: string): Promise<void>;
|
|
111
|
+
/**
|
|
112
|
+
* Mark a job as failed permanently.
|
|
113
|
+
*/
|
|
114
|
+
failJob(jobId: string, queue: string, error?: Error): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Retry a job - move back to pending queue with incremented attempts.
|
|
117
|
+
*/
|
|
118
|
+
retryJob(jobId: string, queue: string, retryAt?: Date): Promise<void>;
|
|
95
119
|
push(jobData: JobData): Promise<void>;
|
|
96
120
|
pushOn(queue: string, jobData: JobData): Promise<void>;
|
|
97
121
|
pushLater(jobData: JobData, delay: number): Promise<void>;
|
|
98
122
|
pushLaterOn(queue: string, jobData: JobData, delay: number): Promise<void>;
|
|
99
|
-
|
|
100
|
-
|
|
123
|
+
size(): Promise<number>;
|
|
124
|
+
sizeOf(queue: string): Promise<number>;
|
|
101
125
|
destroy(): Promise<void>;
|
|
102
126
|
}
|
|
103
127
|
|
|
@@ -118,8 +142,8 @@ declare abstract class Job<Payload = any> {
|
|
|
118
142
|
get payload(): Payload;
|
|
119
143
|
constructor(payload: Payload);
|
|
120
144
|
static dispatch<T extends Job>(this: new (payload: any) => T, payload: T extends Job<infer P> ? P : never): JobDispatcher<T extends Job<infer P> ? P : never>;
|
|
121
|
-
abstract execute(): Promise<void>;
|
|
145
|
+
abstract execute(signal?: AbortSignal): Promise<void>;
|
|
122
146
|
failed?(error: Error): Promise<void>;
|
|
123
147
|
}
|
|
124
148
|
|
|
125
|
-
export { type Adapter as A, type BackoffStrategy as B, type Duration as D, Job as J, type
|
|
149
|
+
export { type Adapter as A, type BackoffStrategy as B, type Duration as D, Job as J, type QueueManagerConfig as Q, type RetryConfig as R, type WorkerCycle as W, type AcquiredJob as a, type JobData as b, customBackoff as c, type JobOptions as d, exponentialBackoff as e, fixedBackoff as f, type JobClass as g, type BackoffConfig as h, type QueueConfig as i, type WorkerConfig as j, type AdapterFactory as k, linearBackoff as l };
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export { A as Adapter } from '../../job-CcAUWe8j.js';
|
|
1
|
+
export { a as AcquiredJob, A as Adapter } from '../../job-Bd_c2lFK.js';
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import { Redis, RedisOptions } from 'ioredis';
|
|
2
|
-
import { A as Adapter,
|
|
3
|
-
import { LeaseManager } from '../contracts/lease_manager.js';
|
|
2
|
+
import { A as Adapter, a as AcquiredJob, b as JobData } from '../../job-Bd_c2lFK.js';
|
|
4
3
|
|
|
5
4
|
type RedisConfig = Redis | RedisOptions;
|
|
6
5
|
declare function redis(config?: RedisConfig): () => RedisAdapter;
|
|
7
6
|
declare class RedisAdapter implements Adapter {
|
|
8
7
|
#private;
|
|
9
8
|
constructor(connection: Redis);
|
|
10
|
-
|
|
9
|
+
setWorkerId(workerId: string): void;
|
|
11
10
|
destroy(): Promise<void>;
|
|
12
|
-
pop(): Promise<
|
|
13
|
-
popFrom(queue: string): Promise<
|
|
11
|
+
pop(): Promise<AcquiredJob | null>;
|
|
12
|
+
popFrom(queue: string): Promise<AcquiredJob | null>;
|
|
13
|
+
popAndWait(queue: string, timeout: number): Promise<AcquiredJob | null>;
|
|
14
|
+
completeJob(jobId: string, queue: string): Promise<void>;
|
|
15
|
+
failJob(jobId: string, queue: string, _error?: Error): Promise<void>;
|
|
16
|
+
retryJob(jobId: string, queue: string, retryAt?: Date): Promise<void>;
|
|
14
17
|
push(jobData: JobData): Promise<void>;
|
|
15
18
|
pushLater(jobData: JobData, delay: number): Promise<void>;
|
|
16
19
|
pushLaterOn(queue: string, jobData: JobData, delay: number): Promise<void>;
|
|
@@ -1,29 +1,97 @@
|
|
|
1
1
|
// src/drivers/redis_adapter.ts
|
|
2
|
-
import { VerrouLeaseManager } from "#lease_managers/verrou";
|
|
3
2
|
import { Redis } from "ioredis";
|
|
4
3
|
var redisKey = "jobs";
|
|
5
|
-
var
|
|
6
|
-
local
|
|
7
|
-
local
|
|
8
|
-
local
|
|
4
|
+
var ACQUIRE_JOB_SCRIPT = `
|
|
5
|
+
local pending_key = KEYS[1]
|
|
6
|
+
local active_key = KEYS[2]
|
|
7
|
+
local delayed_key = KEYS[3]
|
|
8
|
+
local worker_id = ARGV[1]
|
|
9
|
+
local now = ARGV[2]
|
|
9
10
|
|
|
10
|
-
--
|
|
11
|
+
-- First, process delayed jobs
|
|
11
12
|
local ready_jobs = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)
|
|
12
|
-
|
|
13
13
|
if #ready_jobs > 0 then
|
|
14
|
-
-- Move jobs to priority queue and remove from delayed queue atomically
|
|
15
14
|
for i = 1, #ready_jobs do
|
|
16
15
|
local job_data = ready_jobs[i]
|
|
17
16
|
local job = cjson.decode(job_data)
|
|
18
17
|
local priority = job.priority or 5
|
|
19
|
-
|
|
18
|
+
local timestamp = tonumber(now)
|
|
19
|
+
local score = priority * 10000000000000 + timestamp
|
|
20
|
+
redis.call('ZADD', pending_key, score, job_data)
|
|
20
21
|
redis.call('ZREM', delayed_key, job_data)
|
|
21
22
|
end
|
|
23
|
+
end
|
|
22
24
|
|
|
23
|
-
|
|
25
|
+
-- Pop highest priority job (lowest score)
|
|
26
|
+
local result = redis.call('ZPOPMIN', pending_key)
|
|
27
|
+
if not result or #result == 0 then
|
|
28
|
+
return nil
|
|
24
29
|
end
|
|
25
30
|
|
|
26
|
-
|
|
31
|
+
local job_data = result[1]
|
|
32
|
+
local job = cjson.decode(job_data)
|
|
33
|
+
|
|
34
|
+
-- Store in active hash: jobId -> {workerId, acquiredAt, data}
|
|
35
|
+
local active_data = cjson.encode({
|
|
36
|
+
workerId = worker_id,
|
|
37
|
+
acquiredAt = tonumber(now),
|
|
38
|
+
data = job
|
|
39
|
+
})
|
|
40
|
+
redis.call('HSET', active_key, job.id, active_data)
|
|
41
|
+
|
|
42
|
+
-- Return job with acquiredAt
|
|
43
|
+
job.acquiredAt = tonumber(now)
|
|
44
|
+
return cjson.encode(job)
|
|
45
|
+
`;
|
|
46
|
+
var COMPLETE_JOB_SCRIPT = `
|
|
47
|
+
local active_key = KEYS[1]
|
|
48
|
+
local job_id = ARGV[1]
|
|
49
|
+
|
|
50
|
+
redis.call('HDEL', active_key, job_id)
|
|
51
|
+
return 1
|
|
52
|
+
`;
|
|
53
|
+
var FAIL_JOB_SCRIPT = `
|
|
54
|
+
local active_key = KEYS[1]
|
|
55
|
+
local job_id = ARGV[1]
|
|
56
|
+
|
|
57
|
+
redis.call('HDEL', active_key, job_id)
|
|
58
|
+
return 1
|
|
59
|
+
`;
|
|
60
|
+
var RETRY_JOB_SCRIPT = `
|
|
61
|
+
local active_key = KEYS[1]
|
|
62
|
+
local pending_key = KEYS[2]
|
|
63
|
+
local delayed_key = KEYS[3]
|
|
64
|
+
local job_id = ARGV[1]
|
|
65
|
+
local retry_at = tonumber(ARGV[2])
|
|
66
|
+
local now = tonumber(ARGV[3])
|
|
67
|
+
|
|
68
|
+
-- Get job from active hash
|
|
69
|
+
local active_data = redis.call('HGET', active_key, job_id)
|
|
70
|
+
if not active_data then
|
|
71
|
+
return 0
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
local active = cjson.decode(active_data)
|
|
75
|
+
local job = active.data
|
|
76
|
+
|
|
77
|
+
-- Remove from active
|
|
78
|
+
redis.call('HDEL', active_key, job_id)
|
|
79
|
+
|
|
80
|
+
-- Increment attempts
|
|
81
|
+
job.attempts = (job.attempts or 0) + 1
|
|
82
|
+
|
|
83
|
+
local job_data = cjson.encode(job)
|
|
84
|
+
|
|
85
|
+
-- Add back to pending or delayed
|
|
86
|
+
if retry_at and retry_at > now then
|
|
87
|
+
redis.call('ZADD', delayed_key, retry_at, job_data)
|
|
88
|
+
else
|
|
89
|
+
local priority = job.priority or 5
|
|
90
|
+
local score = priority * 10000000000000 + now
|
|
91
|
+
redis.call('ZADD', pending_key, score, job_data)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
return 1
|
|
27
95
|
`;
|
|
28
96
|
function redis(config) {
|
|
29
97
|
return () => {
|
|
@@ -43,11 +111,12 @@ function redis(config) {
|
|
|
43
111
|
}
|
|
44
112
|
var RedisAdapter = class {
|
|
45
113
|
#connection;
|
|
114
|
+
#workerId = "";
|
|
46
115
|
constructor(connection) {
|
|
47
116
|
this.#connection = connection;
|
|
48
117
|
}
|
|
49
|
-
|
|
50
|
-
|
|
118
|
+
setWorkerId(workerId) {
|
|
119
|
+
this.#workerId = workerId;
|
|
51
120
|
}
|
|
52
121
|
async destroy() {
|
|
53
122
|
await this.#connection.quit();
|
|
@@ -56,12 +125,72 @@ var RedisAdapter = class {
|
|
|
56
125
|
return this.popFrom("default");
|
|
57
126
|
}
|
|
58
127
|
async popFrom(queue) {
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
128
|
+
const now = Date.now();
|
|
129
|
+
const pendingKey = `${redisKey}::${queue}`;
|
|
130
|
+
const activeKey = `${redisKey}::${queue}::active`;
|
|
131
|
+
const delayedKey = `${redisKey}::delayed::${queue}`;
|
|
132
|
+
const result = await this.#connection.eval(
|
|
133
|
+
ACQUIRE_JOB_SCRIPT,
|
|
134
|
+
3,
|
|
135
|
+
pendingKey,
|
|
136
|
+
activeKey,
|
|
137
|
+
delayedKey,
|
|
138
|
+
this.#workerId,
|
|
139
|
+
now.toString()
|
|
140
|
+
);
|
|
141
|
+
if (!result) {
|
|
142
|
+
return null;
|
|
63
143
|
}
|
|
64
|
-
return
|
|
144
|
+
return JSON.parse(result);
|
|
145
|
+
}
|
|
146
|
+
async popAndWait(queue, timeout) {
|
|
147
|
+
const immediate = await this.popFrom(queue);
|
|
148
|
+
if (immediate) {
|
|
149
|
+
return immediate;
|
|
150
|
+
}
|
|
151
|
+
const pendingKey = `${redisKey}::${queue}`;
|
|
152
|
+
const activeKey = `${redisKey}::${queue}::active`;
|
|
153
|
+
const now = Date.now();
|
|
154
|
+
const result = await this.#connection.bzpopmin(pendingKey, timeout / 1e3);
|
|
155
|
+
if (!result) {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
const [, jobData] = result;
|
|
159
|
+
const job = JSON.parse(jobData);
|
|
160
|
+
const activeData = JSON.stringify({
|
|
161
|
+
workerId: this.#workerId,
|
|
162
|
+
acquiredAt: now,
|
|
163
|
+
data: job
|
|
164
|
+
});
|
|
165
|
+
await this.#connection.hset(activeKey, job.id, activeData);
|
|
166
|
+
return {
|
|
167
|
+
...job,
|
|
168
|
+
acquiredAt: now
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
async completeJob(jobId, queue) {
|
|
172
|
+
const activeKey = `${redisKey}::${queue}::active`;
|
|
173
|
+
await this.#connection.eval(COMPLETE_JOB_SCRIPT, 1, activeKey, jobId);
|
|
174
|
+
}
|
|
175
|
+
async failJob(jobId, queue, _error) {
|
|
176
|
+
const activeKey = `${redisKey}::${queue}::active`;
|
|
177
|
+
await this.#connection.eval(FAIL_JOB_SCRIPT, 1, activeKey, jobId);
|
|
178
|
+
}
|
|
179
|
+
async retryJob(jobId, queue, retryAt) {
|
|
180
|
+
const now = Date.now();
|
|
181
|
+
const activeKey = `${redisKey}::${queue}::active`;
|
|
182
|
+
const pendingKey = `${redisKey}::${queue}`;
|
|
183
|
+
const delayedKey = `${redisKey}::delayed::${queue}`;
|
|
184
|
+
await this.#connection.eval(
|
|
185
|
+
RETRY_JOB_SCRIPT,
|
|
186
|
+
3,
|
|
187
|
+
activeKey,
|
|
188
|
+
pendingKey,
|
|
189
|
+
delayedKey,
|
|
190
|
+
jobId,
|
|
191
|
+
retryAt ? retryAt.getTime().toString() : "0",
|
|
192
|
+
now.toString()
|
|
193
|
+
);
|
|
65
194
|
}
|
|
66
195
|
push(jobData) {
|
|
67
196
|
return this.pushOn("default", jobData);
|
|
@@ -86,19 +215,6 @@ var RedisAdapter = class {
|
|
|
86
215
|
sizeOf(queue) {
|
|
87
216
|
return this.#connection.zcard(`${redisKey}::${queue}`);
|
|
88
217
|
}
|
|
89
|
-
async #processDelayedJobs(queue) {
|
|
90
|
-
const now = Date.now();
|
|
91
|
-
const delayedKey = `${redisKey}::delayed::${queue}`;
|
|
92
|
-
const queueKey = `${redisKey}::${queue}`;
|
|
93
|
-
return await this.#connection.eval(
|
|
94
|
-
PROCESS_DELAYED_JOBS_SCRIPT,
|
|
95
|
-
2,
|
|
96
|
-
// number of keys
|
|
97
|
-
delayedKey,
|
|
98
|
-
queueKey,
|
|
99
|
-
now.toString()
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
218
|
};
|
|
103
219
|
export {
|
|
104
220
|
RedisAdapter,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/drivers/redis_adapter.ts"],"sourcesContent":["import { VerrouLeaseManager } from '#lease_managers/verrou'\nimport { Redis, type RedisOptions } from 'ioredis'\nimport type { Adapter } from '#contracts/adapter'\nimport type { JobData, LeaseConfig } from '#types/main'\nimport type { LeaseManager } from '#contracts/lease_manager'\n\nconst redisKey = 'jobs'\ntype RedisConfig = Redis | RedisOptions\n\n// Lua script for atomic delayed job processing\nconst PROCESS_DELAYED_JOBS_SCRIPT = `\n local delayed_key = KEYS[1]\n local queue_key = KEYS[2]\n local now = ARGV[1]\n\n -- Get ready jobs (score <= now)\n local ready_jobs = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)\n\n if #ready_jobs > 0 then\n -- Move jobs to priority queue and remove from delayed queue atomically\n for i = 1, #ready_jobs do\n local job_data = ready_jobs[i]\n local job = cjson.decode(job_data)\n local priority = job.priority or 5\n redis.call('ZADD', queue_key, priority, job_data)\n redis.call('ZREM', delayed_key, job_data)\n end\n\n return #ready_jobs\n end\n\n return 0\n`\n\nexport function redis(config?: RedisConfig) {\n return () => {\n if (config instanceof Redis) {\n return new RedisAdapter(config)\n }\n\n // Create new Redis instance from options\n const options: RedisOptions = {\n host: 'localhost',\n port: 6379,\n keyPrefix: 'boringnode::queue::',\n db: 0,\n ...config,\n }\n\n const connection = new Redis(options)\n return new RedisAdapter(connection)\n }\n}\n\nexport class RedisAdapter implements Adapter {\n readonly #connection: Redis\n\n constructor(connection: Redis) {\n this.#connection = connection\n }\n\n createLeaseManager(config: LeaseConfig): LeaseManager {\n return new VerrouLeaseManager(config, this.#connection)\n }\n\n async destroy(): Promise<void> {\n await this.#connection.quit()\n }\n\n pop(): Promise<JobData | null> {\n return this.popFrom('default')\n }\n\n async popFrom(queue: string): Promise<JobData | null> {\n // First, move any ready delayed jobs to the regular queue\n await this.#processDelayedJobs(queue)\n\n // Pop from priority queue (sorted set) - highest priority (lowest score) first\n const queueContent = await this.#connection.zpopmin(`${redisKey}::${queue}`)\n\n if (queueContent && queueContent.length > 0) {\n return JSON.parse(queueContent[0])\n }\n\n return null\n }\n\n push(jobData: JobData): Promise<void> {\n return this.pushOn('default', jobData)\n }\n\n pushLater(jobData: JobData, delay: number): Promise<void> {\n return this.pushLaterOn('default', jobData, delay)\n }\n\n async pushLaterOn(queue: string, jobData: JobData, delay: number): Promise<void> {\n const executeAt = Date.now() + delay\n const delayedKey = `${redisKey}::delayed::${queue}`\n\n await this.#connection.zadd(delayedKey, executeAt, JSON.stringify(jobData))\n }\n\n async pushOn(queue: string, jobData: JobData): Promise<void> {\n const priority = jobData.priority ?? 5\n\n // Use priority as primary score, add timestamp for FIFO order within same priority\n // Date.now() precision is sufficient but perfect FIFO within the same millisecond is not guaranteed\n const timestamp = Date.now()\n const score = priority * 1e13 + timestamp\n\n await this.#connection.zadd(`${redisKey}::${queue}`, score, JSON.stringify(jobData))\n }\n\n size(): Promise<number> {\n return this.sizeOf('default')\n }\n\n sizeOf(queue: string): Promise<number> {\n return this.#connection.zcard(`${redisKey}::${queue}`)\n }\n\n async #processDelayedJobs(queue: string): Promise<number> {\n const now = Date.now()\n const delayedKey = `${redisKey}::delayed::${queue}`\n const queueKey = `${redisKey}::${queue}`\n\n // Use Lua script for atomic operation - much faster than pipeline\n return (await this.#connection.eval(\n PROCESS_DELAYED_JOBS_SCRIPT,\n 2, // number of keys\n delayedKey,\n queueKey,\n now.toString()\n )) as number\n }\n}\n"],"mappings":";AAAA,SAAS,0BAA0B;AACnC,SAAS,aAAgC;AAKzC,IAAM,WAAW;AAIjB,IAAM,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwB7B,SAAS,MAAM,QAAsB;AAC1C,SAAO,MAAM;AACX,QAAI,kBAAkB,OAAO;AAC3B,aAAO,IAAI,aAAa,MAAM;AAAA,IAChC;AAGA,UAAM,UAAwB;AAAA,MAC5B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,IAAI;AAAA,MACJ,GAAG;AAAA,IACL;AAEA,UAAM,aAAa,IAAI,MAAM,OAAO;AACpC,WAAO,IAAI,aAAa,UAAU;AAAA,EACpC;AACF;AAEO,IAAM,eAAN,MAAsC;AAAA,EAClC;AAAA,EAET,YAAY,YAAmB;AAC7B,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,mBAAmB,QAAmC;AACpD,WAAO,IAAI,mBAAmB,QAAQ,KAAK,WAAW;AAAA,EACxD;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,KAAK,YAAY,KAAK;AAAA,EAC9B;AAAA,EAEA,MAA+B;AAC7B,WAAO,KAAK,QAAQ,SAAS;AAAA,EAC/B;AAAA,EAEA,MAAM,QAAQ,OAAwC;AAEpD,UAAM,KAAK,oBAAoB,KAAK;AAGpC,UAAM,eAAe,MAAM,KAAK,YAAY,QAAQ,GAAG,QAAQ,KAAK,KAAK,EAAE;AAE3E,QAAI,gBAAgB,aAAa,SAAS,GAAG;AAC3C,aAAO,KAAK,MAAM,aAAa,CAAC,CAAC;AAAA,IACnC;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,SAAiC;AACpC,WAAO,KAAK,OAAO,WAAW,OAAO;AAAA,EACvC;AAAA,EAEA,UAAU,SAAkB,OAA8B;AACxD,WAAO,KAAK,YAAY,WAAW,SAAS,KAAK;AAAA,EACnD;AAAA,EAEA,MAAM,YAAY,OAAe,SAAkB,OAA8B;AAC/E,UAAM,YAAY,KAAK,IAAI,IAAI;AAC/B,UAAM,aAAa,GAAG,QAAQ,cAAc,KAAK;AAEjD,UAAM,KAAK,YAAY,KAAK,YAAY,WAAW,KAAK,UAAU,OAAO,CAAC;AAAA,EAC5E;AAAA,EAEA,MAAM,OAAO,OAAe,SAAiC;AAC3D,UAAM,WAAW,QAAQ,YAAY;AAIrC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,QAAQ,WAAW,OAAO;AAEhC,UAAM,KAAK,YAAY,KAAK,GAAG,QAAQ,KAAK,KAAK,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC;AAAA,EACrF;AAAA,EAEA,OAAwB;AACtB,WAAO,KAAK,OAAO,SAAS;AAAA,EAC9B;AAAA,EAEA,OAAO,OAAgC;AACrC,WAAO,KAAK,YAAY,MAAM,GAAG,QAAQ,KAAK,KAAK,EAAE;AAAA,EACvD;AAAA,EAEA,MAAM,oBAAoB,OAAgC;AACxD,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,aAAa,GAAG,QAAQ,cAAc,KAAK;AACjD,UAAM,WAAW,GAAG,QAAQ,KAAK,KAAK;AAGtC,WAAQ,MAAM,KAAK,YAAY;AAAA,MAC7B;AAAA,MACA;AAAA;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI,SAAS;AAAA,IACf;AAAA,EACF;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/drivers/redis_adapter.ts"],"sourcesContent":["import { Redis, type RedisOptions } from 'ioredis'\nimport type { Adapter, AcquiredJob } from '#contracts/adapter'\nimport type { JobData } from '#types/main'\n\nconst redisKey = 'jobs'\ntype RedisConfig = Redis | RedisOptions\n\n/**\n * Lua script for atomic job acquisition.\n * 1. Check and process delayed jobs\n * 2. Pop from pending queue\n * 3. Add to active hash with worker info\n * 4. Return job data\n */\nconst ACQUIRE_JOB_SCRIPT = `\n local pending_key = KEYS[1]\n local active_key = KEYS[2]\n local delayed_key = KEYS[3]\n local worker_id = ARGV[1]\n local now = ARGV[2]\n\n -- First, process delayed jobs\n local ready_jobs = redis.call('ZRANGEBYSCORE', delayed_key, 0, now)\n if #ready_jobs > 0 then\n for i = 1, #ready_jobs do\n local job_data = ready_jobs[i]\n local job = cjson.decode(job_data)\n local priority = job.priority or 5\n local timestamp = tonumber(now)\n local score = priority * 10000000000000 + timestamp\n redis.call('ZADD', pending_key, score, job_data)\n redis.call('ZREM', delayed_key, job_data)\n end\n end\n\n -- Pop highest priority job (lowest score)\n local result = redis.call('ZPOPMIN', pending_key)\n if not result or #result == 0 then\n return nil\n end\n\n local job_data = result[1]\n local job = cjson.decode(job_data)\n\n -- Store in active hash: jobId -> {workerId, acquiredAt, data}\n local active_data = cjson.encode({\n workerId = worker_id,\n acquiredAt = tonumber(now),\n data = job\n })\n redis.call('HSET', active_key, job.id, active_data)\n\n -- Return job with acquiredAt\n job.acquiredAt = tonumber(now)\n return cjson.encode(job)\n`\n\n/**\n * Lua script for completing a job.\n * Removes the job from active hash.\n */\nconst COMPLETE_JOB_SCRIPT = `\n local active_key = KEYS[1]\n local job_id = ARGV[1]\n\n redis.call('HDEL', active_key, job_id)\n return 1\n`\n\n/**\n * Lua script for failing a job permanently.\n * Removes from active hash.\n */\nconst FAIL_JOB_SCRIPT = `\n local active_key = KEYS[1]\n local job_id = ARGV[1]\n\n redis.call('HDEL', active_key, job_id)\n return 1\n`\n\n/**\n * Lua script for retrying a job.\n * 1. Get job from active hash\n * 2. Remove from active hash\n * 3. Increment attempts\n * 4. Add back to pending (or delayed if retryAt is set)\n */\nconst RETRY_JOB_SCRIPT = `\n local active_key = KEYS[1]\n local pending_key = KEYS[2]\n local delayed_key = KEYS[3]\n local job_id = ARGV[1]\n local retry_at = tonumber(ARGV[2])\n local now = tonumber(ARGV[3])\n\n -- Get job from active hash\n local active_data = redis.call('HGET', active_key, job_id)\n if not active_data then\n return 0\n end\n\n local active = cjson.decode(active_data)\n local job = active.data\n\n -- Remove from active\n redis.call('HDEL', active_key, job_id)\n\n -- Increment attempts\n job.attempts = (job.attempts or 0) + 1\n\n local job_data = cjson.encode(job)\n\n -- Add back to pending or delayed\n if retry_at and retry_at > now then\n redis.call('ZADD', delayed_key, retry_at, job_data)\n else\n local priority = job.priority or 5\n local score = priority * 10000000000000 + now\n redis.call('ZADD', pending_key, score, job_data)\n end\n\n return 1\n`\n\nexport function redis(config?: RedisConfig) {\n return () => {\n if (config instanceof Redis) {\n return new RedisAdapter(config)\n }\n\n const options: RedisOptions = {\n host: 'localhost',\n port: 6379,\n keyPrefix: 'boringnode::queue::',\n db: 0,\n ...config,\n }\n\n const connection = new Redis(options)\n return new RedisAdapter(connection)\n }\n}\n\nexport class RedisAdapter implements Adapter {\n readonly #connection: Redis\n #workerId: string = ''\n\n constructor(connection: Redis) {\n this.#connection = connection\n }\n\n setWorkerId(workerId: string): void {\n this.#workerId = workerId\n }\n\n async destroy(): Promise<void> {\n await this.#connection.quit()\n }\n\n pop(): Promise<AcquiredJob | null> {\n return this.popFrom('default')\n }\n\n async popFrom(queue: string): Promise<AcquiredJob | null> {\n const now = Date.now()\n const pendingKey = `${redisKey}::${queue}`\n const activeKey = `${redisKey}::${queue}::active`\n const delayedKey = `${redisKey}::delayed::${queue}`\n\n const result = await this.#connection.eval(\n ACQUIRE_JOB_SCRIPT,\n 3,\n pendingKey,\n activeKey,\n delayedKey,\n this.#workerId,\n now.toString()\n )\n\n if (!result) {\n return null\n }\n\n return JSON.parse(result as string)\n }\n\n async popAndWait(queue: string, timeout: number): Promise<AcquiredJob | null> {\n // First try immediate pop\n const immediate = await this.popFrom(queue)\n if (immediate) {\n return immediate\n }\n\n // Wait for new job using BZPOPMIN on pending queue\n const pendingKey = `${redisKey}::${queue}`\n const activeKey = `${redisKey}::${queue}::active`\n const now = Date.now()\n\n // BZPOPMIN returns [key, member, score] or null\n const result = await this.#connection.bzpopmin(pendingKey, timeout / 1000)\n\n if (!result) {\n return null\n }\n\n const [, jobData] = result\n const job = JSON.parse(jobData)\n\n // Store in active hash\n const activeData = JSON.stringify({\n workerId: this.#workerId,\n acquiredAt: now,\n data: job,\n })\n await this.#connection.hset(activeKey, job.id, activeData)\n\n return {\n ...job,\n acquiredAt: now,\n }\n }\n\n async completeJob(jobId: string, queue: string): Promise<void> {\n const activeKey = `${redisKey}::${queue}::active`\n\n await this.#connection.eval(COMPLETE_JOB_SCRIPT, 1, activeKey, jobId)\n }\n\n async failJob(jobId: string, queue: string, _error?: Error): Promise<void> {\n const activeKey = `${redisKey}::${queue}::active`\n\n await this.#connection.eval(FAIL_JOB_SCRIPT, 1, activeKey, jobId)\n }\n\n async retryJob(jobId: string, queue: string, retryAt?: Date): Promise<void> {\n const now = Date.now()\n const activeKey = `${redisKey}::${queue}::active`\n const pendingKey = `${redisKey}::${queue}`\n const delayedKey = `${redisKey}::delayed::${queue}`\n\n await this.#connection.eval(\n RETRY_JOB_SCRIPT,\n 3,\n activeKey,\n pendingKey,\n delayedKey,\n jobId,\n retryAt ? retryAt.getTime().toString() : '0',\n now.toString()\n )\n }\n\n push(jobData: JobData): Promise<void> {\n return this.pushOn('default', jobData)\n }\n\n pushLater(jobData: JobData, delay: number): Promise<void> {\n return this.pushLaterOn('default', jobData, delay)\n }\n\n async pushLaterOn(queue: string, jobData: JobData, delay: number): Promise<void> {\n const executeAt = Date.now() + delay\n const delayedKey = `${redisKey}::delayed::${queue}`\n\n await this.#connection.zadd(delayedKey, executeAt, JSON.stringify(jobData))\n }\n\n async pushOn(queue: string, jobData: JobData): Promise<void> {\n const priority = jobData.priority ?? 5\n\n // Use priority as primary score, add timestamp for FIFO order within same priority\n // Date.now() precision is sufficient but perfect FIFO within the same millisecond is not guaranteed\n const timestamp = Date.now()\n const score = priority * 1e13 + timestamp\n\n await this.#connection.zadd(`${redisKey}::${queue}`, score, JSON.stringify(jobData))\n }\n\n size(): Promise<number> {\n return this.sizeOf('default')\n }\n\n sizeOf(queue: string): Promise<number> {\n return this.#connection.zcard(`${redisKey}::${queue}`)\n }\n}\n"],"mappings":";AAAA,SAAS,aAAgC;AAIzC,IAAM,WAAW;AAUjB,IAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+C3B,IAAM,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAY5B,IAAM,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAexB,IAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqClB,SAAS,MAAM,QAAsB;AAC1C,SAAO,MAAM;AACX,QAAI,kBAAkB,OAAO;AAC3B,aAAO,IAAI,aAAa,MAAM;AAAA,IAChC;AAEA,UAAM,UAAwB;AAAA,MAC5B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,IAAI;AAAA,MACJ,GAAG;AAAA,IACL;AAEA,UAAM,aAAa,IAAI,MAAM,OAAO;AACpC,WAAO,IAAI,aAAa,UAAU;AAAA,EACpC;AACF;AAEO,IAAM,eAAN,MAAsC;AAAA,EAClC;AAAA,EACT,YAAoB;AAAA,EAEpB,YAAY,YAAmB;AAC7B,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,YAAY,UAAwB;AAClC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,KAAK,YAAY,KAAK;AAAA,EAC9B;AAAA,EAEA,MAAmC;AACjC,WAAO,KAAK,QAAQ,SAAS;AAAA,EAC/B;AAAA,EAEA,MAAM,QAAQ,OAA4C;AACxD,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,aAAa,GAAG,QAAQ,KAAK,KAAK;AACxC,UAAM,YAAY,GAAG,QAAQ,KAAK,KAAK;AACvC,UAAM,aAAa,GAAG,QAAQ,cAAc,KAAK;AAEjD,UAAM,SAAS,MAAM,KAAK,YAAY;AAAA,MACpC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK;AAAA,MACL,IAAI,SAAS;AAAA,IACf;AAEA,QAAI,CAAC,QAAQ;AACX,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,MAAM,MAAgB;AAAA,EACpC;AAAA,EAEA,MAAM,WAAW,OAAe,SAA8C;AAE5E,UAAM,YAAY,MAAM,KAAK,QAAQ,KAAK;AAC1C,QAAI,WAAW;AACb,aAAO;AAAA,IACT;AAGA,UAAM,aAAa,GAAG,QAAQ,KAAK,KAAK;AACxC,UAAM,YAAY,GAAG,QAAQ,KAAK,KAAK;AACvC,UAAM,MAAM,KAAK,IAAI;AAGrB,UAAM,SAAS,MAAM,KAAK,YAAY,SAAS,YAAY,UAAU,GAAI;AAEzE,QAAI,CAAC,QAAQ;AACX,aAAO;AAAA,IACT;AAEA,UAAM,CAAC,EAAE,OAAO,IAAI;AACpB,UAAM,MAAM,KAAK,MAAM,OAAO;AAG9B,UAAM,aAAa,KAAK,UAAU;AAAA,MAChC,UAAU,KAAK;AAAA,MACf,YAAY;AAAA,MACZ,MAAM;AAAA,IACR,CAAC;AACD,UAAM,KAAK,YAAY,KAAK,WAAW,IAAI,IAAI,UAAU;AAEzD,WAAO;AAAA,MACL,GAAG;AAAA,MACH,YAAY;AAAA,IACd;AAAA,EACF;AAAA,EAEA,MAAM,YAAY,OAAe,OAA8B;AAC7D,UAAM,YAAY,GAAG,QAAQ,KAAK,KAAK;AAEvC,UAAM,KAAK,YAAY,KAAK,qBAAqB,GAAG,WAAW,KAAK;AAAA,EACtE;AAAA,EAEA,MAAM,QAAQ,OAAe,OAAe,QAA+B;AACzE,UAAM,YAAY,GAAG,QAAQ,KAAK,KAAK;AAEvC,UAAM,KAAK,YAAY,KAAK,iBAAiB,GAAG,WAAW,KAAK;AAAA,EAClE;AAAA,EAEA,MAAM,SAAS,OAAe,OAAe,SAA+B;AAC1E,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,YAAY,GAAG,QAAQ,KAAK,KAAK;AACvC,UAAM,aAAa,GAAG,QAAQ,KAAK,KAAK;AACxC,UAAM,aAAa,GAAG,QAAQ,cAAc,KAAK;AAEjD,UAAM,KAAK,YAAY;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,QAAQ,QAAQ,EAAE,SAAS,IAAI;AAAA,MACzC,IAAI,SAAS;AAAA,IACf;AAAA,EACF;AAAA,EAEA,KAAK,SAAiC;AACpC,WAAO,KAAK,OAAO,WAAW,OAAO;AAAA,EACvC;AAAA,EAEA,UAAU,SAAkB,OAA8B;AACxD,WAAO,KAAK,YAAY,WAAW,SAAS,KAAK;AAAA,EACnD;AAAA,EAEA,MAAM,YAAY,OAAe,SAAkB,OAA8B;AAC/E,UAAM,YAAY,KAAK,IAAI,IAAI;AAC/B,UAAM,aAAa,GAAG,QAAQ,cAAc,KAAK;AAEjD,UAAM,KAAK,YAAY,KAAK,YAAY,WAAW,KAAK,UAAU,OAAO,CAAC;AAAA,EAC5E;AAAA,EAEA,MAAM,OAAO,OAAe,SAAiC;AAC3D,UAAM,WAAW,QAAQ,YAAY;AAIrC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,QAAQ,WAAW,OAAO;AAEhC,UAAM,KAAK,YAAY,KAAK,GAAG,QAAQ,KAAK,KAAK,IAAI,OAAO,KAAK,UAAU,OAAO,CAAC;AAAA,EACrF;AAAA,EAEA,OAAwB;AACtB,WAAO,KAAK,OAAO,SAAS;AAAA,EAC9B;AAAA,EAEA,OAAO,OAAgC;AACrC,WAAO,KAAK,YAAY,MAAM,GAAG,QAAQ,KAAK,KAAK,EAAE;AAAA,EACvD;AACF;","names":[]}
|
|
@@ -1,18 +1,24 @@
|
|
|
1
|
-
import { A as Adapter,
|
|
2
|
-
import { LeaseManager } from '../contracts/lease_manager.js';
|
|
1
|
+
import { A as Adapter, b as JobData, a as AcquiredJob } from '../../job-Bd_c2lFK.js';
|
|
3
2
|
|
|
4
3
|
declare function sync(): () => SyncAdapter;
|
|
4
|
+
/**
|
|
5
|
+
* Sync adapter executes jobs immediately when pushed.
|
|
6
|
+
* Pop/complete/fail/retry are not supported as jobs are executed synchronously.
|
|
7
|
+
*/
|
|
5
8
|
declare class SyncAdapter implements Adapter {
|
|
6
9
|
#private;
|
|
7
|
-
|
|
10
|
+
setWorkerId(_workerId: string): void;
|
|
8
11
|
push(jobData: JobData): Promise<void>;
|
|
9
12
|
pushOn(_queue: string, jobData: JobData): Promise<void>;
|
|
10
13
|
pushLater(jobData: JobData, delay: number): Promise<void>;
|
|
11
14
|
pushLaterOn(_queue: string, jobData: JobData, delay: number): Promise<void>;
|
|
12
15
|
size(): Promise<number>;
|
|
13
16
|
sizeOf(_queue: string): Promise<number>;
|
|
14
|
-
pop(): Promise<
|
|
15
|
-
popFrom(_queue: string): Promise<
|
|
17
|
+
pop(): Promise<AcquiredJob | null>;
|
|
18
|
+
popFrom(_queue: string): Promise<AcquiredJob | null>;
|
|
19
|
+
completeJob(_jobId: string, _queue: string): Promise<void>;
|
|
20
|
+
failJob(_jobId: string, _queue: string, _error?: Error): Promise<void>;
|
|
21
|
+
retryJob(_jobId: string, _queue: string, _retryAt?: Date): Promise<void>;
|
|
16
22
|
destroy(): Promise<void>;
|
|
17
23
|
}
|
|
18
24
|
|
|
@@ -4,8 +4,7 @@ function sync() {
|
|
|
4
4
|
return () => new SyncAdapter();
|
|
5
5
|
}
|
|
6
6
|
var SyncAdapter = class {
|
|
7
|
-
|
|
8
|
-
throw new Error("Method not implemented.");
|
|
7
|
+
setWorkerId(_workerId) {
|
|
9
8
|
}
|
|
10
9
|
push(jobData) {
|
|
11
10
|
return this.pushOn("default", jobData);
|
|
@@ -32,7 +31,16 @@ var SyncAdapter = class {
|
|
|
32
31
|
return this.popFrom("default");
|
|
33
32
|
}
|
|
34
33
|
popFrom(_queue) {
|
|
35
|
-
throw new Error("
|
|
34
|
+
throw new Error("SyncAdapter does not support pop - jobs are executed immediately on push");
|
|
35
|
+
}
|
|
36
|
+
completeJob(_jobId, _queue) {
|
|
37
|
+
return Promise.resolve();
|
|
38
|
+
}
|
|
39
|
+
failJob(_jobId, _queue, _error) {
|
|
40
|
+
return Promise.resolve();
|
|
41
|
+
}
|
|
42
|
+
retryJob(_jobId, _queue, _retryAt) {
|
|
43
|
+
return Promise.resolve();
|
|
36
44
|
}
|
|
37
45
|
destroy() {
|
|
38
46
|
return Promise.resolve();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/drivers/sync_adapter.ts"],"sourcesContent":["import { Locator } from '#src/locator'\nimport type { Adapter } from '#contracts/adapter'\nimport type { JobData
|
|
1
|
+
{"version":3,"sources":["../../../src/drivers/sync_adapter.ts"],"sourcesContent":["import { Locator } from '#src/locator'\nimport type { Adapter, AcquiredJob } from '#contracts/adapter'\nimport type { JobData } from '#types/main'\n\nexport function sync() {\n return () => new SyncAdapter()\n}\n\n/**\n * Sync adapter executes jobs immediately when pushed.\n * Pop/complete/fail/retry are not supported as jobs are executed synchronously.\n */\nexport class SyncAdapter implements Adapter {\n setWorkerId(_workerId: string): void {}\n\n push(jobData: JobData): Promise<void> {\n return this.pushOn('default', jobData)\n }\n\n pushOn(_queue: string, jobData: JobData): Promise<void> {\n return this.#execute(jobData.name, jobData.payload)\n }\n\n pushLater(jobData: JobData, delay: number): Promise<void> {\n return this.pushLaterOn('default', jobData, delay)\n }\n\n pushLaterOn(_queue: string, jobData: JobData, delay: number): Promise<void> {\n setTimeout(() => {\n void this.#execute(jobData.name, jobData.payload)\n }, delay)\n\n return Promise.resolve()\n }\n\n size(): Promise<number> {\n return this.sizeOf('default')\n }\n\n sizeOf(_queue: string): Promise<number> {\n return Promise.resolve(0)\n }\n\n pop(): Promise<AcquiredJob | null> {\n return this.popFrom('default')\n }\n\n popFrom(_queue: string): Promise<AcquiredJob | null> {\n throw new Error('SyncAdapter does not support pop - jobs are executed immediately on push')\n }\n\n completeJob(_jobId: string, _queue: string): Promise<void> {\n return Promise.resolve()\n }\n\n failJob(_jobId: string, _queue: string, _error?: Error): Promise<void> {\n return Promise.resolve()\n }\n\n retryJob(_jobId: string, _queue: string, _retryAt?: Date): Promise<void> {\n return Promise.resolve()\n }\n\n destroy(): Promise<void> {\n return Promise.resolve()\n }\n\n async #execute(jobName: string, payload: any): Promise<any> {\n const JobClass = Locator.get(jobName)\n\n if (!JobClass) {\n throw new Error(`Job class ${jobName} not found.`)\n }\n\n const jobInstance = new JobClass(payload)\n await jobInstance.execute()\n }\n}\n"],"mappings":";AAAA,SAAS,eAAe;AAIjB,SAAS,OAAO;AACrB,SAAO,MAAM,IAAI,YAAY;AAC/B;AAMO,IAAM,cAAN,MAAqC;AAAA,EAC1C,YAAY,WAAyB;AAAA,EAAC;AAAA,EAEtC,KAAK,SAAiC;AACpC,WAAO,KAAK,OAAO,WAAW,OAAO;AAAA,EACvC;AAAA,EAEA,OAAO,QAAgB,SAAiC;AACtD,WAAO,KAAK,SAAS,QAAQ,MAAM,QAAQ,OAAO;AAAA,EACpD;AAAA,EAEA,UAAU,SAAkB,OAA8B;AACxD,WAAO,KAAK,YAAY,WAAW,SAAS,KAAK;AAAA,EACnD;AAAA,EAEA,YAAY,QAAgB,SAAkB,OAA8B;AAC1E,eAAW,MAAM;AACf,WAAK,KAAK,SAAS,QAAQ,MAAM,QAAQ,OAAO;AAAA,IAClD,GAAG,KAAK;AAER,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEA,OAAwB;AACtB,WAAO,KAAK,OAAO,SAAS;AAAA,EAC9B;AAAA,EAEA,OAAO,QAAiC;AACtC,WAAO,QAAQ,QAAQ,CAAC;AAAA,EAC1B;AAAA,EAEA,MAAmC;AACjC,WAAO,KAAK,QAAQ,SAAS;AAAA,EAC/B;AAAA,EAEA,QAAQ,QAA6C;AACnD,UAAM,IAAI,MAAM,0EAA0E;AAAA,EAC5F;AAAA,EAEA,YAAY,QAAgB,QAA+B;AACzD,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEA,QAAQ,QAAgB,QAAgB,QAA+B;AACrE,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEA,SAAS,QAAgB,QAAgB,UAAgC;AACvE,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEA,UAAyB;AACvB,WAAO,QAAQ,QAAQ;AAAA,EACzB;AAAA,EAEA,MAAM,SAAS,SAAiB,SAA4B;AAC1D,UAAM,WAAW,QAAQ,IAAI,OAAO;AAEpC,QAAI,CAAC,UAAU;AACb,YAAM,IAAI,MAAM,aAAa,OAAO,aAAa;AAAA,IACnD;AAEA,UAAM,cAAc,IAAI,SAAS,OAAO;AACxC,UAAM,YAAY,QAAQ;AAAA,EAC5B;AACF;","names":[]}
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
export {
|
|
2
|
-
import '../contracts/lease_manager.js';
|
|
1
|
+
export { k as AdapterFactory, h as BackoffConfig, B as BackoffStrategy, D as Duration, g as JobClass, b as JobData, d as JobOptions, i as QueueConfig, Q as QueueManagerConfig, R as RetryConfig, j as WorkerConfig, W as WorkerCycle } from '../../job-Bd_c2lFK.js';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@boringnode/queue",
|
|
3
3
|
"description": "A simple and efficient queue system for Node.js applications",
|
|
4
|
-
"version": "0.0.1-alpha",
|
|
4
|
+
"version": "0.0.1-alpha.0",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"files": [
|
|
@@ -22,6 +22,9 @@
|
|
|
22
22
|
"#types/*": "./src/types/*.ts"
|
|
23
23
|
},
|
|
24
24
|
"scripts": {
|
|
25
|
+
"benchmark": "node benchmark/run.ts",
|
|
26
|
+
"benchmark:quick": "node benchmark/run.ts --quick",
|
|
27
|
+
"benchmark:full": "node benchmark/run.ts --full",
|
|
25
28
|
"build": "yarn clean && tsup-node",
|
|
26
29
|
"clean": "del-cli build",
|
|
27
30
|
"format": "prettier --write .",
|
|
@@ -33,8 +36,7 @@
|
|
|
33
36
|
},
|
|
34
37
|
"dependencies": {
|
|
35
38
|
"@lukeed/ms": "^2.0.2",
|
|
36
|
-
"@poppinss/utils": "^6.10.1"
|
|
37
|
-
"@verrou/core": "^0.5.1"
|
|
39
|
+
"@poppinss/utils": "^6.10.1"
|
|
38
40
|
},
|
|
39
41
|
"devDependencies": {
|
|
40
42
|
"@adonisjs/eslint-config": "^2.1.2",
|
|
@@ -45,6 +47,7 @@
|
|
|
45
47
|
"@japa/file-system": "^2.3.2",
|
|
46
48
|
"@japa/runner": "^4.4.0",
|
|
47
49
|
"@types/node": "^24.3.1",
|
|
50
|
+
"bullmq": "^5.65.1",
|
|
48
51
|
"c8": "^10.1.3",
|
|
49
52
|
"del-cli": "^7.0.0",
|
|
50
53
|
"eslint": "^9.35.0",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
//# sourceMappingURL=lease_manager.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|