@gravito/stream 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +27 -1
  2. package/dist/BatchConsumer.d.ts +81 -0
  3. package/dist/Consumer.d.ts +215 -0
  4. package/dist/DashboardProvider.d.ts +20 -0
  5. package/dist/Job.d.ts +183 -0
  6. package/dist/OrbitStream.d.ts +151 -0
  7. package/dist/QueueManager.d.ts +319 -0
  8. package/dist/Queueable.d.ts +91 -0
  9. package/dist/Scheduler.d.ts +214 -0
  10. package/dist/StreamEventBackend.d.ts +114 -0
  11. package/dist/SystemEventJob.d.ts +33 -0
  12. package/dist/Worker.d.ts +139 -0
  13. package/dist/benchmarks/PerformanceReporter.d.ts +99 -0
  14. package/dist/consumer/ConcurrencyGate.d.ts +55 -0
  15. package/dist/consumer/ConsumerStrategy.d.ts +41 -0
  16. package/dist/consumer/GroupSequencer.d.ts +57 -0
  17. package/dist/consumer/HeartbeatManager.d.ts +65 -0
  18. package/dist/consumer/JobExecutor.d.ts +61 -0
  19. package/dist/consumer/JobSourceGenerator.d.ts +31 -0
  20. package/dist/consumer/PollingStrategy.d.ts +42 -0
  21. package/dist/consumer/ReactiveStrategy.d.ts +41 -0
  22. package/dist/consumer/StreamingConsumer.d.ts +88 -0
  23. package/dist/consumer/index.d.ts +13 -0
  24. package/dist/consumer/types.d.ts +102 -0
  25. package/dist/drivers/BinaryJobFrame.d.ts +78 -0
  26. package/dist/drivers/BullMQDriver.d.ts +186 -0
  27. package/dist/drivers/DatabaseDriver.d.ts +131 -0
  28. package/dist/drivers/GrpcDriver.d.ts +16 -0
  29. package/dist/drivers/KafkaDriver.d.ts +148 -0
  30. package/dist/drivers/MemoryDriver.d.ts +108 -0
  31. package/dist/drivers/QueueDriver.d.ts +250 -0
  32. package/dist/drivers/RabbitMQDriver.d.ts +102 -0
  33. package/dist/drivers/RedisDriver.d.ts +294 -0
  34. package/dist/drivers/SQSDriver.d.ts +111 -0
  35. package/dist/drivers/kafka/BackpressureController.d.ts +60 -0
  36. package/dist/drivers/kafka/BatchProcessor.d.ts +50 -0
  37. package/dist/drivers/kafka/ConsumerLifecycleManager.d.ts +80 -0
  38. package/dist/drivers/kafka/ErrorCategorizer.d.ts +39 -0
  39. package/dist/drivers/kafka/ErrorRecoveryManager.d.ts +100 -0
  40. package/dist/drivers/kafka/HeartbeatManager.d.ts +57 -0
  41. package/dist/drivers/kafka/KafkaDriver.d.ts +138 -0
  42. package/dist/drivers/kafka/KafkaMetrics.d.ts +88 -0
  43. package/dist/drivers/kafka/KafkaNotifier.d.ts +54 -0
  44. package/dist/drivers/kafka/MessageBuffer.d.ts +71 -0
  45. package/dist/drivers/kafka/OffsetTracker.d.ts +63 -0
  46. package/dist/drivers/kafka/PerformanceMonitor.d.ts +88 -0
  47. package/dist/drivers/kafka/RateLimiter.d.ts +52 -0
  48. package/dist/drivers/kafka/RebalanceHandler.d.ts +104 -0
  49. package/dist/drivers/kafka/RingBuffer.d.ts +63 -0
  50. package/dist/drivers/kafka/index.d.ts +22 -0
  51. package/dist/drivers/kafka/types.d.ts +553 -0
  52. package/dist/drivers/prepareJobForTransport.d.ts +10 -0
  53. package/dist/index.cjs +5644 -5508
  54. package/dist/index.cjs.map +71 -0
  55. package/dist/index.d.ts +60 -4378
  56. package/dist/index.js +5609 -5453
  57. package/dist/index.js.map +71 -0
  58. package/dist/locks/DistributedLock.d.ts +175 -0
  59. package/dist/persistence/BufferedPersistence.d.ts +130 -0
  60. package/dist/persistence/BunBufferedPersistence.d.ts +173 -0
  61. package/dist/persistence/MySQLPersistence.d.ts +134 -0
  62. package/dist/persistence/SQLitePersistence.d.ts +133 -0
  63. package/dist/serializers/BinarySerializer.d.ts +42 -0
  64. package/dist/serializers/CachedSerializer.d.ts +38 -0
  65. package/dist/serializers/CborNativeSerializer.d.ts +56 -0
  66. package/dist/serializers/ClassNameSerializer.d.ts +58 -0
  67. package/dist/serializers/JobSerializer.d.ts +33 -0
  68. package/dist/serializers/JsonSerializer.d.ts +28 -0
  69. package/dist/serializers/JsonlSerializer.d.ts +90 -0
  70. package/dist/serializers/MessagePackSerializer.d.ts +29 -0
  71. package/dist/types.d.ts +653 -0
  72. package/dist/workers/BinaryWorkerProtocol.d.ts +77 -0
  73. package/dist/workers/BunWorker.d.ts +179 -0
  74. package/dist/workers/SandboxedWorker.d.ts +132 -0
  75. package/dist/workers/WorkerFactory.d.ts +128 -0
  76. package/dist/workers/WorkerPool.d.ts +186 -0
  77. package/dist/workers/bun-job-executor.d.ts +14 -0
  78. package/dist/workers/index.d.ts +13 -0
  79. package/dist/workers/job-executor.d.ts +9 -0
  80. package/package.json +6 -4
  81. package/dist/index.d.cts +0 -4387
package/dist/index.d.ts CHANGED
@@ -1,4387 +1,69 @@
1
- import { EventEmitter } from 'node:events';
2
- import { GravitoOrbit, PlanetCore, EventBackend, EventTask } from '@gravito/core';
3
- import { ConnectionContract } from '@gravito/atlas';
4
-
5
1
  /**
6
- * Defines the contract for queueable items (jobs) with fluent configuration.
2
+ * @gravito/stream
7
3
  *
8
- * This interface enables method chaining for configuring job properties such as
9
- * target queue, connection, delay, and priority before dispatching. It ensures
10
- * consistent API surface across different job implementations.
4
+ * Lightweight, high-performance queue system inspired by Laravel while keeping Gravito's core values.
5
+ * Supports multiple storage drivers, embedded/standalone consumer modes, and multiple job serialization strategies.
11
6
  *
12
- * @public
13
7
  * @example
14
8
  * ```typescript
15
- * class MyJob implements Queueable {
16
- * queueName?: string
17
- * // ... implementation
18
- * onQueue(queue: string): this {
19
- * this.queueName = queue
20
- * return this
21
- * }
22
- * // ...
23
- * }
24
- * ```
25
- */
26
- interface Queueable {
27
- /**
28
- * The specific queue name where the job should be processed.
29
- *
30
- * If not set, the default queue for the connection will be used.
31
- */
32
- queueName?: string;
33
- /**
34
- * The connection name (e.g., 'redis', 'sqs') to use for this job.
35
- *
36
- * If not set, the default connection configured in QueueManager will be used.
37
- */
38
- connectionName?: string;
39
- /**
40
- * The number of seconds to delay the job execution.
41
- */
42
- delaySeconds?: number;
43
- /**
44
- * The priority level of the job.
45
- */
46
- priority?: string | number;
47
- /**
48
- * Sets the target queue for the job.
49
- *
50
- * @param queue - The name of the queue to push the job to.
51
- * @returns The instance for method chaining.
52
- *
53
- * @example
54
- * ```typescript
55
- * job.onQueue('notifications');
56
- * ```
57
- */
58
- onQueue(queue: string): this;
59
- /**
60
- * Sets the target connection for the job.
61
- *
62
- * @param connection - The name of the connection to use.
63
- * @returns The instance for method chaining.
64
- *
65
- * @example
66
- * ```typescript
67
- * job.onConnection('sqs');
68
- * ```
69
- */
70
- onConnection(connection: string): this;
71
- /**
72
- * Sets the priority of the job.
73
- *
74
- * @param priority - The priority level (e.g., 'high', 'low', 10).
75
- * @returns The instance for method chaining.
76
- *
77
- * @example
78
- * ```typescript
79
- * job.withPriority('critical');
80
- * ```
81
- */
82
- withPriority(priority: string | number): this;
83
- /**
84
- * Sets a delay before the job is available for processing.
85
- *
86
- * @param delay - The delay in seconds.
87
- * @returns The instance for method chaining.
88
- *
89
- * @example
90
- * ```typescript
91
- * job.delay(300); // 5 minutes
92
- * ```
93
- */
94
- delay(delay: number): this;
95
- }
96
-
97
- /**
98
- * Abstract base class for all background jobs.
99
- *
100
- * This class serves as the foundation for creating queueable tasks. It implements the `Queueable`
101
- * interface for fluent configuration and provides the core structure for defining execution logic (`handle`)
102
- * and failure handling (`failed`).
103
- *
104
- * Subclasses must implement the `handle` method.
9
+ * import { OrbitStream, Job } from '@gravito/stream'
105
10
  *
106
- * @public
107
- * @example
108
- * ```typescript
109
- * export class SendEmailJob extends Job {
110
- * constructor(private email: string, private subject: string) {
111
- * super();
112
- * }
113
- *
114
- * async handle(): Promise<void> {
115
- * await emailService.send(this.email, this.subject);
11
+ * // Create a Job
12
+ * class SendEmail extends Job {
13
+ * async handle() {
14
+ * // handle logic
116
15
  * }
117
16
  * }
118
17
  *
119
- * // Usage
120
- * await queue.push(new SendEmailJob('user@example.com', 'Welcome'))
121
- * .onQueue('emails')
122
- * .delay(60);
123
- * ```
124
- */
125
- declare abstract class Job implements Queueable {
126
- /**
127
- * Unique identifier for the job instance.
128
- *
129
- * Assigned automatically when the job is pushed to the queue.
130
- */
131
- id?: string;
132
- /**
133
- * The name of the queue where this job will be processed.
134
- */
135
- queueName?: string;
136
- /**
137
- * The name of the connection used to transport this job.
138
- */
139
- connectionName?: string;
140
- /**
141
- * Delay in seconds before the job becomes available for processing.
142
- */
143
- delaySeconds?: number;
144
- /**
145
- * The current attempt number (starts at 1).
146
- */
147
- attempts?: number;
148
- /**
149
- * The maximum number of retry attempts allowed.
150
- *
151
- * Can be overridden by the worker configuration or per-job using `maxAttempts`.
152
- */
153
- maxAttempts?: number;
154
- /**
155
- * Group ID for sequential processing.
156
- *
157
- * Jobs with the same `groupId` will be processed in strict order (FIFO)
158
- * if the consumer supports it.
159
- */
160
- groupId?: string;
161
- /**
162
- * Priority level of the job.
163
- */
164
- priority?: string | number;
165
- /**
166
- * Initial delay in seconds before the first retry attempt.
167
- *
168
- * Used for exponential backoff calculation.
169
- */
170
- retryAfterSeconds?: number;
171
- /**
172
- * Multiplier applied to the retry delay for each subsequent attempt.
173
- *
174
- * Used for exponential backoff calculation.
175
- */
176
- retryMultiplier?: number;
177
- /**
178
- * Sets the target queue for the job.
179
- *
180
- * @param queue - The name of the target queue.
181
- * @returns The job instance for chaining.
182
- *
183
- * @example
184
- * ```typescript
185
- * job.onQueue('billing');
186
- * ```
187
- */
188
- onQueue(queue: string): this;
189
- /**
190
- * Sets the target connection for the job.
191
- *
192
- * @param connection - The name of the connection (e.g., 'redis').
193
- * @returns The job instance for chaining.
194
- *
195
- * @example
196
- * ```typescript
197
- * job.onConnection('sqs-primary');
198
- * ```
199
- */
200
- onConnection(connection: string): this;
201
- /**
202
- * Sets the priority of the job.
203
- *
204
- * @param priority - The priority level (e.g., 'high', 10).
205
- * @returns The job instance for chaining.
206
- *
207
- * @example
208
- * ```typescript
209
- * job.withPriority('high');
210
- * ```
211
- */
212
- withPriority(priority: string | number): this;
213
- /**
214
- * Delays the job execution.
215
- *
216
- * @param delay - Delay in seconds.
217
- * @returns The job instance for chaining.
218
- *
219
- * @example
220
- * ```typescript
221
- * job.delay(60); // Run after 1 minute
222
- * ```
223
- */
224
- delay(delay: number): this;
225
- /**
226
- * Configures the exponential backoff strategy for retries.
227
- *
228
- * @param seconds - Initial delay in seconds before the first retry.
229
- * @param multiplier - Factor by which the delay increases for each subsequent attempt (default: 2).
230
- * @returns The job instance for chaining.
231
- *
232
- * @example
233
- * ```typescript
234
- * // Wait 5s, then 10s, then 20s...
235
- * job.backoff(5, 2);
236
- * ```
237
- */
238
- backoff(seconds: number, multiplier?: number): this;
239
- /**
240
- * Calculates the delay for the next retry attempt based on the backoff strategy.
241
- *
242
- * Uses the formula: `initialDelay * multiplier^(attempt - 1)`, capped at 1 hour.
243
- *
244
- * @param attempt - The current attempt number (1-based).
245
- * @returns The calculated delay in milliseconds.
246
- *
247
- * @example
248
- * ```typescript
249
- * const nextDelay = job.getRetryDelay(2);
250
- * ```
251
- */
252
- getRetryDelay(attempt: number): number;
253
- /**
254
- * Contains the main business logic of the job.
255
- *
256
- * This method is called by the worker to process the job.
257
- * Implementations should be idempotent if possible.
258
- *
259
- * @throws {Error} If the job fails and should be retried.
260
- */
261
- abstract handle(): Promise<void>;
262
- /**
263
- * Optional handler for when the job has permanently failed.
264
- *
265
- * Called when the job has exhausted all retry attempts.
266
- * Useful for cleaning up resources, sending alerts, or logging.
267
- *
268
- * @param _error - The error that caused the final failure.
269
- *
270
- * @example
271
- * ```typescript
272
- * async failed(error: Error) {
273
- * await notifyAdmin(`Job failed: ${error.message}`);
274
- * }
275
- * ```
276
- */
277
- failed(_error: Error): Promise<void>;
278
- }
279
-
280
- /**
281
- * Represents a job that has been serialized for storage in a queue.
282
- *
283
- * This interface defines the data structure used to persist jobs in the underlying
284
- * storage mechanism (e.g., Redis, Database, SQS). It encapsulates all metadata
285
- * required for processing, retries, and lifecycle management.
286
- *
287
- * @public
288
- * @example
289
- * ```typescript
290
- * const job: SerializedJob = {
291
- * id: 'job-123',
292
- * type: 'json',
293
- * data: '{"userId": 1}',
294
- * createdAt: Date.now()
295
- * };
296
- * ```
297
- */
298
- interface SerializedJob {
299
- /**
300
- * Unique identifier for the job.
301
- */
302
- id: string;
303
- /**
304
- * The serialization format used for the job data.
305
- *
306
- * - 'json': Simple JSON objects.
307
- * - 'class': Serialized class instances (requires class registration).
308
- * - 'msgpack': Binary MessagePack format.
309
- */
310
- type: 'json' | 'class' | 'msgpack';
311
- /**
312
- * The actual serialized job payload.
313
- *
314
- * Contains the business data needed to execute the job.
315
- */
316
- data: string;
317
- /**
318
- * The fully qualified class name of the job.
319
- *
320
- * Only required when `type` is 'class' to reconstruct the original object instance.
321
- */
322
- className?: string;
323
- /**
324
- * The timestamp (in milliseconds) when the job was originally created.
325
- */
326
- createdAt: number;
327
- /**
328
- * Optional delay in seconds before the job becomes eligible for processing.
329
- *
330
- * Used for scheduling future tasks.
331
- */
332
- delaySeconds?: number;
333
- /**
334
- * The number of times this job has been attempted so far.
335
- */
336
- attempts?: number;
337
- /**
338
- * The maximum number of retry attempts allowed before marking the job as failed.
339
- */
340
- maxAttempts?: number;
341
- /**
342
- * Group ID for sequential processing.
343
- *
344
- * Jobs sharing the same `groupId` are guaranteed to be processed in order (FIFO),
345
- * provided the consumer supports this feature.
346
- */
347
- groupId?: string;
348
- /**
349
- * The initial delay in seconds before the first retry attempt after a failure.
350
- */
351
- retryAfterSeconds?: number;
352
- /**
353
- * The multiplier applied to the delay for exponential backoff strategies.
354
- */
355
- retryMultiplier?: number;
356
- /**
357
- * The error message from the last failed attempt, if any.
358
- */
359
- error?: string;
360
- /**
361
- * The timestamp (in milliseconds) when the job was permanently marked as failed.
362
- */
363
- failedAt?: number;
364
- /**
365
- * The priority level of the job.
366
- *
367
- * Higher values or specific strings (e.g., 'high') indicate higher priority.
368
- */
369
- priority?: string | number;
370
- }
371
- /**
372
- * Statistics snapshot for a specific queue.
373
- *
374
- * Provides insight into the current state of a queue, including pending,
375
- * delayed, reserved, and failed job counts.
376
- *
377
- * @public
378
- * @example
379
- * ```typescript
380
- * const stats: QueueStats = {
381
- * queue: 'default',
382
- * size: 42,
383
- * delayed: 5,
384
- * failed: 1
385
- * };
386
- * ```
387
- */
388
- interface QueueStats {
389
- /** Name of the queue */
390
- queue: string;
391
- /** Number of pending jobs waiting to be processed */
392
- size: number;
393
- /** Number of jobs scheduled for future execution */
394
- delayed?: number;
395
- /** Number of jobs currently being processed by workers */
396
- reserved?: number;
397
- /** Number of jobs in the Dead Letter Queue (DLQ) */
398
- failed?: number;
399
- /** Driver-specific custom metrics */
400
- metrics?: Record<string, number>;
401
- }
402
- /**
403
- * Snapshot of statistics across all connections and queues.
404
- *
405
- * Used by monitoring dashboards to provide a high-level overview
406
- * of the entire background processing system.
407
- *
408
- * @public
409
- */
410
- interface GlobalStats {
411
- /** Map of connection names to their respective queue statistics */
412
- connections: Record<string, QueueStats[]>;
413
- /** Total number of pending jobs across all connections */
414
- totalSize: number;
415
- /** Total number of failed jobs across all connections */
416
- totalFailed: number;
417
- /** Timestamp of the snapshot */
418
- timestamp: number;
419
- }
420
- /**
421
- * Advanced topic configuration options for distributed queue systems.
422
- *
423
- * Used primarily by drivers like Kafka to configure topic properties.
424
- *
425
- * @public
426
- * @example
427
- * ```typescript
428
- * const options: TopicOptions = {
429
- * partitions: 3,
430
- * replicationFactor: 2
431
- * };
432
- * ```
433
- */
434
- interface TopicOptions {
435
- /** Number of partitions for the topic */
436
- partitions?: number;
437
- /** Replication factor for fault tolerance */
438
- replicationFactor?: number;
439
- /** Additional driver-specific configuration key-values */
440
- config?: Record<string, string>;
441
- }
442
- /**
443
- * Configuration for the Database driver.
444
- *
445
- * Configures the queue system to use a SQL database for job storage.
446
- *
447
- * @public
448
- * @example
449
- * ```typescript
450
- * const config: DatabaseDriverConfig = {
451
- * driver: 'database',
452
- * dbService: myDbService,
453
- * table: 'my_jobs'
454
- * };
455
- * ```
456
- */
457
- interface DatabaseDriverConfig$1 {
458
- /** Driver type identifier */
459
- driver: 'database';
460
- /** Database service implementation for executing queries */
461
- dbService: any;
462
- /** Optional custom table name for storing jobs */
463
- table?: string;
464
- }
465
- /**
466
- * Configuration for the Redis driver.
467
- *
468
- * Configures the queue system to use Redis for high-performance job storage.
469
- *
470
- * @public
471
- * @example
472
- * ```typescript
473
- * const config: RedisDriverConfig = {
474
- * driver: 'redis',
475
- * client: redisClient,
476
- * prefix: 'my-app:queue:'
477
- * };
478
- * ```
479
- */
480
- interface RedisDriverConfig$1 {
481
- /** Driver type identifier */
482
- driver: 'redis';
483
- /** Redis client instance (ioredis or node-redis compatible) */
484
- client: any;
485
- /** Optional key prefix for namespacing */
486
- prefix?: string;
487
- }
488
- /**
489
- * Configuration for the Kafka driver.
490
- *
491
- * Configures the queue system to use Apache Kafka.
492
- *
493
- * @example
494
- * ```typescript
495
- * const config: KafkaDriverConfig = {
496
- * driver: 'kafka',
497
- * client: kafkaClient,
498
- * consumerGroupId: 'my-group'
499
- * };
500
- * ```
501
- */
502
- interface KafkaDriverConfig$1 {
503
- /** Driver type identifier */
504
- driver: 'kafka';
505
- /** Kafka client instance */
506
- client: any;
507
- /** Consumer group ID for coordinating workers */
508
- consumerGroupId?: string;
509
- }
510
- /**
511
- * Configuration for the SQS driver.
512
- *
513
- * Configures the queue system to use Amazon Simple Queue Service.
514
- *
515
- * @public
516
- * @example
517
- * ```typescript
518
- * const config: SQSDriverConfig = {
519
- * driver: 'sqs',
520
- * client: sqsClient,
521
- * queueUrlPrefix: 'https://sqs.us-east-1.amazonaws.com/123/'
522
- * };
523
- * ```
524
- */
525
- interface SQSDriverConfig$1 {
526
- /** Driver type identifier */
527
- driver: 'sqs';
528
- /** Amazon SQS client instance */
529
- client: any;
530
- /** Optional prefix for resolving queue names to URLs */
531
- queueUrlPrefix?: string;
532
- /** The duration (in seconds) that received messages are hidden from other consumers */
533
- visibilityTimeout?: number;
534
- /** The duration (in seconds) to wait for a message (Long Polling) */
535
- waitTimeSeconds?: number;
536
- }
537
- /**
538
- * Configuration for the RabbitMQ driver.
539
- *
540
- * Configures the queue system to use RabbitMQ (AMQP).
541
- *
542
- * @example
543
- * ```typescript
544
- * const config: RabbitMQDriverConfig = {
545
- * driver: 'rabbitmq',
546
- * client: amqpConnection,
547
- * exchange: 'jobs',
548
- * exchangeType: 'direct'
549
- * };
550
- * ```
551
- */
552
- interface RabbitMQDriverConfig$1 {
553
- /** Driver type identifier */
554
- driver: 'rabbitmq';
555
- /** AMQP client instance */
556
- client: any;
557
- /** Exchange name to publish to */
558
- exchange?: string;
559
- /** Type of exchange (direct, topic, fanout, headers) */
560
- exchangeType?: string;
561
- }
562
- /**
563
- * Configuration for the gRPC driver.
564
- *
565
- * Configures the queue system to use a remote gRPC service.
566
- *
567
- * @public
568
- * @example
569
- * ```typescript
570
- * const config: GrpcDriverConfig = {
571
- * driver: 'grpc',
572
- * url: 'localhost:50051',
573
- * protoUser: 'myuser',
574
- * protoPassword: 'mypassword',
575
- * serviceName: 'QueueService',
576
- * packageName: 'stream'
577
- * };
578
- * ```
579
- */
580
- interface GrpcDriverConfig {
581
- /** Driver type identifier */
582
- driver: 'grpc';
583
- /** The gRPC server URL (host:port) */
584
- url: string;
585
- /** Path to the .proto file (optional, defaults to built-in) */
586
- protoPath?: string;
587
- /** The package name defined in the .proto file (default: 'stream') */
588
- packageName?: string;
589
- /** The service name defined in the .proto file (default: 'QueueService') */
590
- serviceName?: string;
591
- /** Optional credentials/metadata for connection */
592
- credentials?: {
593
- rootCerts?: Buffer;
594
- privateKey?: Buffer;
595
- certChain?: Buffer;
596
- };
597
- }
598
- /**
599
- * Configuration for the Bull Queue driver.
600
- *
601
- * Configures the queue system to use Bull Queue (backed by Redis).
602
- *
603
- * @public
604
- * @example
605
- * ```typescript
606
- * import { Queue } from 'bullmq'
607
- * import Redis from 'ioredis'
608
- *
609
- * const redis = new Redis()
610
- * const queue = new Queue('gravito-events', { connection: redis })
611
- * const config: BullMQDriverConfig = {
612
- * driver: 'bullmq',
613
- * queue: queue
614
- * };
615
- * ```
616
- */
617
- interface BullMQDriverConfig$1 {
618
- /** Driver type identifier */
619
- driver: 'bullmq';
620
- /** Bull Queue instance */
621
- queue: any;
622
- /** Optional Bull Worker instance */
623
- worker?: any;
624
- /** Key prefix for queue namespacing */
625
- prefix?: string;
626
- /** Enable debug logging */
627
- debug?: boolean;
628
- }
629
- /**
630
- * Union type for all supported queue connection configurations.
631
- *
632
- * @public
633
- */
634
- type QueueConnectionConfig = {
635
- driver: 'memory';
636
- } | DatabaseDriverConfig$1 | RedisDriverConfig$1 | KafkaDriverConfig$1 | SQSDriverConfig$1 | RabbitMQDriverConfig$1 | GrpcDriverConfig | BullMQDriverConfig$1 | {
637
- driver: 'nats';
638
- [key: string]: unknown;
639
- } | {
640
- driver: string;
641
- [key: string]: unknown;
642
- };
643
- /**
644
- * Global configuration for the QueueManager.
645
- *
646
- * Defines available connections, serialization settings, and system-wide behaviors.
647
- *
648
- * @example
649
- * ```typescript
650
- * const config: QueueConfig = {
651
- * default: 'redis',
652
- * connections: {
653
- * redis: { driver: 'redis', client: redis }
654
- * },
655
- * debug: true
656
- * };
657
- * ```
658
- */
659
- interface QueueConfig {
660
- /**
661
- * The name of the default connection to use when none is specified.
662
- */
663
- default?: string;
664
- /**
665
- * Map of connection names to their configurations.
666
- */
667
- connections?: Record<string, QueueConnectionConfig>;
668
- /**
669
- * The default serialization format to use for jobs.
670
- */
671
- defaultSerializer?: 'json' | 'class' | 'msgpack';
672
- /**
673
- * Whether to cache serialized job data.
674
- *
675
- * If true, re-queuing the same Job instance will reuse the cached serialized string,
676
- * improving performance for frequently pushed jobs.
677
- *
678
- * @default false
679
- */
680
- useSerializationCache?: boolean;
681
- /**
682
- * Enable verbose debug logging.
683
- *
684
- * Useful for troubleshooting queue operations and consumer behavior.
685
- *
686
- * @default false
687
- */
688
- debug?: boolean;
689
- /**
690
- * Configuration for the persistence layer (SQL Archive).
691
- */
692
- persistence?: {
693
- /**
694
- * The persistence adapter instance used to store archived jobs.
695
- */
696
- adapter: PersistenceAdapter;
697
- /**
698
- * Whether to automatically archive jobs upon successful completion.
699
- */
700
- archiveCompleted?: boolean;
701
- /**
702
- * Whether to automatically archive jobs upon permanent failure.
703
- */
704
- archiveFailed?: boolean;
705
- /**
706
- * Whether to archive jobs immediately when they are enqueued (Audit Mode).
707
- *
708
- * @default false
709
- */
710
- archiveEnqueued?: boolean;
711
- /**
712
- * Buffer size for batched writes to the archive.
713
- *
714
- * If set, wraps the adapter in a `BufferedPersistence` decorator to improve throughput.
715
- */
716
- bufferSize?: number;
717
- /**
718
- * Maximum time (in milliseconds) to wait before flushing the write buffer.
719
- */
720
- flushInterval?: number;
721
- };
722
- }
723
- /**
724
- * Interface for persistence adapters.
725
- *
726
- * Defines the contract for storing long-term history of jobs in a permanent storage
727
- * (typically a SQL database).
728
- *
729
- * @example
730
- * ```typescript
731
- * class MyPersistence implements PersistenceAdapter {
732
- * async archive(queue, job, status) { ... }
733
- * // ...
734
- * }
735
- * ```
736
- */
737
- interface PersistenceAdapter {
738
- /**
739
- * Archive a single job.
740
- *
741
- * @param queue - The name of the queue.
742
- * @param job - The serialized job data.
743
- * @param status - The final status of the job ('completed', 'failed', etc.).
744
- * @returns A promise that resolves when the job is archived.
745
- */
746
- archive(queue: string, job: SerializedJob, status: 'completed' | 'failed' | 'waiting' | string): Promise<void>;
747
- /**
748
- * Find a specific job in the archive.
749
- *
750
- * @param queue - The name of the queue.
751
- * @param id - The job ID.
752
- * @returns The serialized job if found, or null.
753
- */
754
- find(queue: string, id: string): Promise<SerializedJob | null>;
755
- /**
756
- * List jobs from the archive based on criteria.
757
- *
758
- * @param queue - The name of the queue.
759
- * @param options - Filtering and pagination options.
760
- * @returns A list of matching serialized jobs.
761
- */
762
- list(queue: string, options?: {
763
- limit?: number;
764
- offset?: number;
765
- status?: 'completed' | 'failed' | 'waiting' | string | string[];
766
- jobId?: string;
767
- startTime?: Date;
768
- endTime?: Date;
769
- }): Promise<SerializedJob[]>;
770
- /**
771
- * Archive multiple jobs in a single batch operation.
772
- *
773
- * @param jobs - Array of job data to archive.
774
- * @returns A promise that resolves when all jobs are archived.
775
- */
776
- archiveMany?(jobs: Array<{
777
- queue: string;
778
- job: SerializedJob;
779
- status: 'completed' | 'failed' | 'waiting' | string;
780
- }>): Promise<void>;
781
- /**
782
- * Remove old data from the archive.
783
- *
784
- * @param days - Retention period in days; older records will be deleted.
785
- * @returns The number of records deleted.
786
- */
787
- cleanup(days: number): Promise<number>;
788
- /**
789
- * Flush any buffered data to storage.
790
- *
791
- * @returns A promise that resolves when the flush is complete.
792
- */
793
- flush?(): Promise<void>;
794
- /**
795
- * Count jobs in the archive matching specific criteria.
796
- *
797
- * @param queue - The name of the queue.
798
- * @param options - Filtering options.
799
- * @returns The total count of matching jobs.
800
- */
801
- count(queue: string, options?: {
802
- status?: 'completed' | 'failed' | 'waiting' | string | string[];
803
- jobId?: string;
804
- startTime?: Date;
805
- endTime?: Date;
806
- }): Promise<number>;
807
- /**
808
- * Archive a system log message.
809
- *
810
- * @param log - The log entry to archive.
811
- * @returns A promise that resolves when the log is archived.
812
- */
813
- archiveLog(log: {
814
- level: string;
815
- message: string;
816
- workerId: string;
817
- queue?: string;
818
- timestamp: Date;
819
- }): Promise<void>;
820
- /**
821
- * Archive multiple log messages in a batch.
822
- *
823
- * @param logs - Array of log entries to archive.
824
- * @returns A promise that resolves when logs are archived.
825
- */
826
- archiveLogMany?(logs: Array<{
827
- level: string;
828
- message: string;
829
- workerId: string;
830
- queue?: string;
831
- timestamp: Date;
832
- }>): Promise<void>;
833
- /**
834
- * List system logs from the archive.
835
- *
836
- * @param options - Filtering and pagination options.
837
- * @returns A list of matching log entries.
838
- */
839
- listLogs(options?: {
840
- limit?: number;
841
- offset?: number;
842
- level?: string;
843
- workerId?: string;
844
- queue?: string;
845
- search?: string;
846
- startTime?: Date;
847
- endTime?: Date;
848
- }): Promise<any[]>;
849
- /**
850
- * Count system logs in the archive.
851
- *
852
- * @param options - Filtering options.
853
- * @returns The total count of matching logs.
854
- */
855
- countLogs(options?: {
856
- level?: string;
857
- workerId?: string;
858
- queue?: string;
859
- search?: string;
860
- startTime?: Date;
861
- endTime?: Date;
862
- }): Promise<number>;
863
- }
864
- /**
865
- * Options used when pushing a job to the queue.
866
- *
867
- * Allows customizing delivery behavior, such as delays, priority, and ordering.
868
- *
869
- * @example
870
- * ```typescript
871
- * const options: JobPushOptions = {
872
- * priority: 'high',
873
- * groupId: 'user-123'
874
- * };
875
- * ```
876
- */
877
- interface JobPushOptions {
878
- /**
879
- * Group ID for FIFO ordering.
880
- *
881
- * If provided, jobs with the same `groupId` are guaranteed to be processed strictly
882
- * sequentially. This is useful for event streams where order matters (e.g., per-user events).
883
- */
884
- groupId?: string;
885
- /**
886
- * Job priority level.
887
- *
888
- * Higher priority jobs are processed before lower priority ones, depending on driver support.
889
- * Common values: 'critical', 'high', 'default', 'low'.
890
- */
891
- priority?: string | number;
892
- }
893
-
894
- /**
895
- * Queue driver interface.
896
- *
897
- * Defines the contract that all storage backends (Redis, Database, SQS, etc.) must implement
898
- * to be compatible with the QueueManager. It covers the basic operations for a job queue:
899
- * push, pop, size, and clear.
900
- *
901
- * Advanced capabilities like rate limiting, reliable delivery (acknowledgements), and
902
- * batch operations are optional but recommended for high-performance drivers.
903
- *
904
- * @public
905
- * @example
906
- * ```typescript
907
- * class MyCustomDriver implements QueueDriver {
908
- * async push(queue: string, job: SerializedJob) {
909
- * // ... write to storage
910
- * }
911
- * async pop(queue: string) {
912
- * // ... read from storage
913
- * return job;
914
- * }
915
- * async size(queue: string) { return 0; }
916
- * async clear(queue: string) {}
917
- * }
918
- * ```
919
- */
920
- interface QueueDriver {
921
- /**
922
- * Pushes a job onto the specified queue.
923
- *
924
- * @param queue - The name of the queue.
925
- * @param job - The serialized job data.
926
- * @param options - Optional parameters like priority or group ID.
927
- * @returns A promise that resolves when the job is successfully stored.
928
- */
929
- push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
930
- /**
931
- * Retrieves and removes the next job from the queue (FIFO).
932
- *
933
- * @param queue - The name of the queue.
934
- * @returns The serialized job if available, or `null` if the queue is empty.
935
- */
936
- pop(queue: string): Promise<SerializedJob | null>;
937
- /**
938
- * Blocking version of `pop`. Waits for a job to arrive if the queue is empty.
939
- *
940
- * @param queues - A single queue name or an array of queues to listen to.
941
- * @param timeout - The maximum time to wait in seconds (0 means indefinite).
942
- * @returns The serialized job if one arrives within the timeout, or `null`.
943
- */
944
- popBlocking?(queues: string | string[], timeout: number): Promise<SerializedJob | null>;
945
- /**
946
- * Marks a job as completed.
947
- *
948
- * Used primarily by drivers that support advanced flow control (like FIFO groups)
949
- * or explicit acknowledgement (like SQS/RabbitMQ).
950
- *
951
- * @param queue - The name of the queue.
952
- * @param job - The job that was completed.
953
- */
954
- complete?(queue: string, job: SerializedJob): Promise<void>;
955
- /**
956
- * Returns the number of jobs currently waiting in the queue.
957
- *
958
- * @param queue - The name of the queue.
959
- * @returns The count of pending jobs.
960
- */
961
- size(queue: string): Promise<number>;
962
- /**
963
- * Removes all jobs from the specified queue.
964
- *
965
- * @param queue - The name of the queue to purge.
966
- */
967
- clear(queue: string): Promise<void>;
968
- /**
969
- * Moves a job to the Dead Letter Queue (DLQ) after repeated failures.
970
- *
971
- * @param queue - The original queue name.
972
- * @param job - The job data (usually including error information).
973
- */
974
- fail?(queue: string, job: SerializedJob): Promise<void>;
975
- /**
976
- * Retrieves detailed statistics for a queue.
977
- *
978
- * @param queue - The name of the queue.
979
- * @returns An object containing counts for pending, delayed, failed, etc.
980
- */
981
- stats?(queue: string): Promise<QueueStats>;
982
- /**
983
- * Pushes multiple jobs to the queue in a single batch operation.
984
- *
985
- * @param queue - The name of the queue.
986
- * @param jobs - An array of serialized jobs.
987
- */
988
- pushMany?(queue: string, jobs: SerializedJob[]): Promise<void>;
989
- /**
990
- * Retrieves multiple jobs from the queue in a single batch operation.
991
- *
992
- * @param queue - The name of the queue.
993
- * @param count - The maximum number of jobs to retrieve.
994
- * @returns An array of serialized jobs.
995
- */
996
- popMany?(queue: string, count: number): Promise<SerializedJob[]>;
997
- /**
998
- * Acknowledges a specific message by its ID.
999
- *
1000
- * Used for drivers that require explicit acknowledgement (e.g., SQS, Kafka).
1001
- *
1002
- * @param messageId - The ID of the message to acknowledge.
1003
- */
1004
- acknowledge?(messageId: string): Promise<void>;
1005
- /**
1006
- * Subscribes to a queue for real-time job processing (Push model).
1007
- *
1008
- * Alternative to polling (`pop`). The driver pushes jobs to the callback as they arrive.
1009
- *
1010
- * @param queue - The name of the queue.
1011
- * @param callback - The function to call when a job is received.
1012
- */
1013
- subscribe?(queue: string, callback: (job: SerializedJob) => Promise<void>): Promise<void>;
1014
- /**
1015
- * Creates a new topic or queue (for drivers like Kafka/RabbitMQ).
1016
- *
1017
- * @param topic - The name of the topic/queue.
1018
- * @param options - Configuration options (partitions, replication, etc.).
1019
- */
1020
- createTopic?(topic: string, options?: TopicOptions): Promise<void>;
1021
- /**
1022
- * Deletes a topic or queue.
1023
- *
1024
- * @param topic - The name of the topic/queue to delete.
1025
- */
1026
- deleteTopic?(topic: string): Promise<void>;
1027
- /**
1028
- * Sends a heartbeat signal for a worker instance.
1029
- *
1030
- * Used for monitoring worker health and presence.
1031
- *
1032
- * @param workerInfo - Metadata about the worker (ID, status, resources).
1033
- * @param prefix - Optional key prefix for storage.
1034
- */
1035
- reportHeartbeat?(workerInfo: {
1036
- id: string;
1037
- status: string;
1038
- hostname: string;
1039
- pid: number;
1040
- uptime: number;
1041
- last_ping: string;
1042
- queues: string[];
1043
- metrics?: Record<string, any>;
1044
- [key: string]: any;
1045
- }, prefix?: string): Promise<void>;
1046
- /**
1047
- * Publishes a log entry to the monitoring system.
1048
- *
1049
- * @param logPayload - The log data (level, message, context).
1050
- * @param prefix - Optional key prefix.
1051
- */
1052
- publishLog?(logPayload: {
1053
- level: string;
1054
- message: string;
1055
- workerId: string;
1056
- jobId?: string;
1057
- timestamp: string;
1058
- [key: string]: any;
1059
- }, prefix?: string): Promise<void>;
1060
- /**
1061
- * Checks if a specific queue has exceeded its rate limit.
1062
- *
1063
- * @param queue - The name of the queue.
1064
- * @param config - The rate limit rules (max jobs per duration).
1065
- * @returns `true` if the job is allowed to proceed, `false` if limited.
1066
- */
1067
- checkRateLimit?(queue: string, config: {
1068
- max: number;
1069
- duration: number;
1070
- }): Promise<boolean>;
1071
- /**
1072
- * Retries failed jobs from the Dead Letter Queue.
1073
- *
1074
- * Moves jobs from the DLQ back to the active queue.
1075
- *
1076
- * @param queue - The name of the queue.
1077
- * @param count - The number of jobs to retry (optional).
1078
- * @returns The number of jobs actually moved.
1079
- */
1080
- retryFailed?(queue: string, count?: number): Promise<number>;
1081
- /**
1082
- * Retrieves a list of failed jobs from the Dead Letter Queue.
1083
- *
1084
- * @param queue - The name of the queue.
1085
- * @param start - Pagination start index.
1086
- * @param end - Pagination end index.
1087
- * @returns An array of failed jobs.
1088
- */
1089
- getFailed?(queue: string, start?: number, end?: number): Promise<SerializedJob[]>;
1090
- /**
1091
- * Clears the Dead Letter Queue for a specific queue.
1092
- *
1093
- * @param queue - The name of the queue.
1094
- */
1095
- clearFailed?(queue: string): Promise<void>;
1096
- /**
1097
- * Lists all queues managed by this driver.
1098
- *
1099
- * Useful for monitoring dashboards to discover active queues dynamically.
1100
- *
1101
- * @returns A list of queue names.
1102
- */
1103
- getQueues?(): Promise<string[]>;
1104
- }
1105
-
1106
- /**
1107
- * Configuration for a recurring scheduled job.
1108
- *
1109
- * Defines the schedule (CRON), the job to execute, and metadata tracking execution times.
1110
- *
1111
- * @public
1112
- * @since 3.0.0
1113
- * @example
1114
- * ```typescript
1115
- * const config: ScheduledJobConfig = {
1116
- * id: 'daily-report',
1117
- * cron: '0 0 * * *',
1118
- * queue: 'reports',
1119
- * job: serializedJob,
1120
- * enabled: true
1121
- * };
1122
- * ```
1123
- */
1124
- interface ScheduledJobConfig {
1125
- /** Unique identifier for the scheduled task. */
1126
- id: string;
1127
- /** Cron expression defining the schedule (e.g., '* * * * *'). */
1128
- cron: string;
1129
- /** The target queue name where the job should be pushed. */
1130
- queue: string;
1131
- /** The serialized job data. */
1132
- job: SerializedJob;
1133
- /** Timestamp of the last successful execution in milliseconds. */
1134
- lastRun?: number;
1135
- /** Timestamp of the next scheduled execution in milliseconds. */
1136
- nextRun?: number;
1137
- /** Whether the scheduled job is active. */
1138
- enabled: boolean;
1139
- }
1140
- /**
1141
- * Configuration options for the Scheduler.
1142
- *
1143
- * Defines behavior for scheduling tasks, including distributed lock settings.
1144
- *
1145
- * @public
1146
- * @since 3.1.0
1147
- * @example
1148
- * ```typescript
1149
- * const options: SchedulerOptions = {
1150
- * prefix: 'myapp:queue:',
1151
- * lockTtl: 60000, // Lock held for 60 seconds
1152
- * lockRefreshInterval: 20000 // Auto-renew every 20 seconds
1153
- * };
1154
- * ```
1155
- */
1156
- interface SchedulerOptions {
1157
- /**
1158
- * Prefix for Redis keys.
1159
- *
1160
- * @default 'queue:'
1161
- */
1162
- prefix?: string;
1163
- /**
1164
- * Time-to-live for the distributed lock in milliseconds.
1165
- *
1166
- * Setting a longer TTL ensures long-running tasks are not executed repeatedly
1167
- * due to lock expiration. Recommended to be 2-3 times the expected execution time.
1168
- *
1169
- * @default 60000 (60 seconds)
1170
- */
1171
- lockTtl?: number;
1172
- /**
1173
- * Interval for automatic lock renewal in milliseconds.
1174
- *
1175
- * If set, the lock will be automatically extended every `lockRefreshInterval`.
1176
- * Recommended to be 1/3 of `lockTtl`.
1177
- *
1178
- * @default 20000 (20 seconds)
1179
- */
1180
- lockRefreshInterval?: number;
1181
- /**
1182
- * Number of retries when acquiring a lock fails.
1183
- *
1184
- * @default 0
1185
- */
1186
- lockRetryCount?: number;
1187
- /**
1188
- * Delay between lock acquisition retries in milliseconds.
1189
- *
1190
- * @default 100
1191
- */
1192
- lockRetryDelay?: number;
1193
- /**
1194
- * The interval in milliseconds at which the scheduler checks for due tasks.
1195
- *
1196
- * @default 60000 (1 minute)
1197
- */
1198
- tickInterval?: number;
1199
- /**
1200
- * The time-to-live for the leader lock in milliseconds.
1201
- *
1202
- * Ensures that only one node acts as the scheduler leader.
1203
- * @default 30000 (30 seconds)
1204
- */
1205
- leaderTtl?: number;
1206
- }
1207
- /**
1208
- * Manages recurring tasks and cron jobs.
1209
- *
1210
- * The Scheduler allows you to register jobs to run at specific intervals using CRON syntax.
1211
- * It uses Redis (or a compatible driver) to coordinate distributed execution, ensuring that
1212
- * a scheduled job runs only once per interval across multiple scheduler instances.
1213
- *
1214
- * @public
1215
- * @since 3.0.0
1216
- * @example
1217
- * ```typescript
1218
- * const scheduler = manager.getScheduler();
1219
- * await scheduler.register({
1220
- * id: 'cleanup',
1221
- * cron: '0 * * * *', // Every hour
1222
- * job: new CleanupJob()
1223
- * });
1224
- *
1225
- * // Automatically start the scheduler loop
1226
- * await scheduler.start();
1227
- * ```
1228
- */
1229
- declare class Scheduler {
1230
- private manager;
1231
- private prefix;
1232
- private lockTtl;
1233
- private lockRefreshInterval?;
1234
- private lockRetryCount;
1235
- private lockRetryDelay;
1236
- private tickInterval;
1237
- private leaderTtl;
1238
- private distributedLock?;
1239
- private running;
1240
- private timer;
1241
- private isLeader;
1242
- constructor(manager: QueueManager, options?: SchedulerOptions);
1243
- private get client();
1244
- /**
1245
- * Gets or creates the distributed lock instance.
1246
- *
1247
- * @private
1248
- */
1249
- private getDistributedLock;
1250
- /**
1251
- * Registers a new scheduled job or updates an existing one.
1252
- *
1253
- * Calculates the next run time based on the CRON expression and stores the configuration in Redis.
1254
- *
1255
- * @param config - The job configuration (excluding nextRun and enabled status which are auto-set).
1256
- * @throws {Error} If Redis client does not support pipelining.
1257
- */
1258
- register(config: Omit<ScheduledJobConfig, 'nextRun' | 'enabled'>): Promise<void>;
1259
- /**
1260
- * Removes a scheduled job.
1261
- *
1262
- * Deletes the job metadata and schedule entry from Redis.
1263
- *
1264
- * @param id - The unique identifier of the scheduled job.
1265
- */
1266
- remove(id: string): Promise<void>;
1267
- /**
1268
- * Lists all registered scheduled jobs.
1269
- *
1270
- * @returns An array of all scheduled job configurations.
1271
- */
1272
- list(): Promise<ScheduledJobConfig[]>;
1273
- /**
1274
- * Starts the automatic scheduler loop.
1275
- *
1276
- * Periodically triggers `tick()` to process due jobs. Uses leader election
1277
- * to ensure that only one node performs the scanning in a multi-node environment.
1278
- */
1279
- start(): Promise<void>;
1280
- /**
1281
- * Stops the automatic scheduler loop.
1282
- */
1283
- stop(): Promise<void>;
1284
- /**
1285
- * Acquires the leader lock and performs a tick.
1286
- *
1287
- * @private
1288
- */
1289
- private performTickWithLeaderElection;
1290
- /**
1291
- * Releases the leader lock.
1292
- *
1293
- * @private
1294
- */
1295
- private releaseLeader;
1296
- /**
1297
- * Manually triggers a scheduled job immediately.
1298
- *
1299
- * Forces execution of the job regardless of its schedule, without affecting the next scheduled run time.
1300
- *
1301
- * @param id - The unique identifier of the scheduled job.
1302
- */
1303
- runNow(id: string): Promise<void>;
1304
- /**
1305
- * Checks for and triggers tasks that are due for execution.
1306
- *
1307
- * This method should be called periodically (e.g., via a system cron or a dedicated tick loop).
1308
- * It scans the schedule for tasks with `nextRun <= now`, acquires a distributed lock for each,
1309
- * pushes them to their queue, and updates the `nextRun` time.
1310
- *
1311
- * The distributed lock ensures that in a multi-node environment, each scheduled job is executed
1312
- * only once per interval, even if multiple scheduler instances are running.
1313
- *
1314
- * @returns The number of jobs triggered in this tick.
1315
- */
1316
- tick(): Promise<number>;
1317
- }
1318
-
1319
- /**
1320
- * Job serializer interface.
1321
- *
1322
- * Defines the contract for serializing job objects into storage-friendly formats
1323
- * (strings/buffers) and deserializing them back into executable Job instances.
1324
- *
1325
- * @public
1326
- * @example
1327
- * ```typescript
1328
- * class JSONSerializer implements JobSerializer {
1329
- * serialize(job) { return { ...job, data: JSON.stringify(job) }; }
1330
- * deserialize(serialized) { return JSON.parse(serialized.data); }
1331
- * }
1332
- * ```
1333
- */
1334
- interface JobSerializer {
1335
- /**
1336
- * Converts a Job instance into a serializable object.
1337
- *
1338
- * @param job - The job instance to serialize.
1339
- * @returns A `SerializedJob` object containing the data payload and metadata.
1340
- */
1341
- serialize(job: Job): SerializedJob;
1342
- /**
1343
- * Reconstructs a Job instance from a serialized object.
1344
- *
1345
- * @param serialized - The serialized job data.
1346
- * @returns A fully hydrated Job instance ready for execution.
1347
- */
1348
- deserialize(serialized: SerializedJob): Job;
1349
- }
1350
-
1351
- /**
1352
- * The central manager for queue operations.
1353
- *
1354
- * This class manages multiple queue connections and drivers, exposing a unified API for pushing,
1355
- * popping, and managing jobs. It handles connection pooling, serialization, persistence,
1356
- * and driver lazy-loading.
1357
- *
1358
- * @public
1359
- * @example
1360
- * ```typescript
1361
- * const manager = new QueueManager({
1362
- * default: 'redis',
1363
- * connections: {
1364
- * redis: { driver: 'redis', client: redisClient }
1365
- * }
1366
- * });
1367
- *
1368
- * await manager.push(new SendEmailJob('hello@example.com'));
1369
- * ```
1370
- */
1371
- declare class QueueManager {
1372
- private drivers;
1373
- private serializers;
1374
- private defaultConnection;
1375
- private defaultSerializer;
1376
- private persistence?;
1377
- private scheduler?;
1378
- private debug;
1379
- constructor(config?: QueueConfig);
1380
- /**
1381
- * Log debug message.
1382
- */
1383
- private log;
1384
- /**
1385
- * Registers a new queue connection with the manager.
1386
- *
1387
- * Dynamically loads the required driver implementation based on the configuration.
1388
- *
1389
- * @param name - The name of the connection (e.g., 'primary').
1390
- * @param config - The configuration object for the driver.
1391
- * @throws {Error} If the driver type is missing required dependencies or unsupported.
1392
- *
1393
- * @example
1394
- * ```typescript
1395
- * manager.registerConnection('analytics', { driver: 'sqs', client: sqs });
1396
- * ```
1397
- */
1398
- registerConnection(name: string, config: QueueConnectionConfig): void;
1399
- /**
1400
- * Retrieves the driver instance for a specific connection.
1401
- *
1402
- * @param connection - The name of the connection.
1403
- * @returns The configured QueueDriver instance.
1404
- * @throws {Error} If the connection has not been registered.
1405
- *
1406
- * @example
1407
- * ```typescript
1408
- * const driver = manager.getDriver('redis');
1409
- * ```
1410
- */
1411
- getDriver(connection: string): QueueDriver;
1412
- /**
1413
- * Gets the name of the default connection.
1414
- *
1415
- * @returns The default connection name.
1416
- */
1417
- getDefaultConnection(): string;
1418
- /**
1419
- * Retrieves a serializer instance by type.
1420
- *
1421
- * @param type - The serializer type (e.g., 'json', 'class'). If omitted, returns the default serializer.
1422
- * @returns The JobSerializer instance.
1423
- * @throws {Error} If the requested serializer type is not found.
1424
- */
1425
- getSerializer(type?: string): JobSerializer;
1426
- /**
1427
- * Registers Job classes for the `ClassNameSerializer`.
1428
- *
1429
- * This is required when using 'class' serialization to allow proper hydration of job instances
1430
- * upon deserialization.
1431
- *
1432
- * @param jobClasses - An array of Job class constructors.
1433
- *
1434
- * @example
1435
- * ```typescript
1436
- * manager.registerJobClasses([SendEmailJob, ProcessOrderJob]);
1437
- * ```
1438
- */
1439
- registerJobClasses(jobClasses: Array<new (...args: unknown[]) => Job>): void;
1440
- /**
1441
- * Pushes a single job to the queue.
1442
- *
1443
- * Serializes the job, selects the appropriate driver based on job configuration,
1444
- * and dispatches it. Also handles audit logging if persistence is enabled.
1445
- *
1446
- * @template T - The type of the job (extends Job).
1447
- * @param job - The job instance to enqueue.
1448
- * @param options - Optional overrides for push behavior (priority, delay, etc.).
1449
- * @returns The same job instance (for chaining).
1450
- *
1451
- * @example
1452
- * ```typescript
1453
- * await manager.push(new SendEmailJob('user@example.com'));
1454
- * ```
1455
- */
1456
- push<T extends Job & Queueable>(job: T, options?: JobPushOptions): Promise<T>;
1457
- /**
1458
- * Pushes multiple jobs to the queue in a batch.
1459
- *
1460
- * Optimizes network requests by batching jobs where possible. Groups jobs by connection
1461
- * and queue to maximize throughput.
1462
- *
1463
- * @template T - The type of the jobs.
1464
- * @param jobs - An array of job instances to enqueue.
1465
- * @param options - Configuration for batch size and concurrency.
1466
- * @returns A promise that resolves when all jobs have been pushed.
1467
- *
1468
- * @example
1469
- * ```typescript
1470
- * await manager.pushMany(jobs, { batchSize: 500, concurrency: 5 });
1471
- * ```
1472
- */
1473
- pushMany<T extends Job & Queueable>(jobs: T[], options?: {
1474
- batchSize?: number;
1475
- concurrency?: number;
1476
- }): Promise<void>;
1477
- /**
1478
- * Pops a single job from the queue.
1479
- *
1480
- * Retrieves the next available job from the specified queue.
1481
- *
1482
- * @param queue - The queue name (default: 'default').
1483
- * @param connection - The connection name (defaults to default connection).
1484
- * @returns A Job instance if found, or `null` if the queue is empty.
1485
- *
1486
- * @example
1487
- * ```typescript
1488
- * const job = await manager.pop('priority-queue');
1489
- * if (job) await job.handle();
1490
- * ```
1491
- */
1492
- pop(queue?: string, connection?: string): Promise<Job | null>;
1493
- /**
1494
- * Pops multiple jobs from the queue efficiently.
1495
- *
1496
- * Attempts to retrieve a batch of jobs from the driver. If the driver does not support
1497
- * batching, it falls back to sequential popping.
1498
- *
1499
- * @param queue - The queue name (default: 'default').
1500
- * @param count - The maximum number of jobs to retrieve (default: 10).
1501
- * @param connection - The connection name.
1502
- * @returns An array of Job instances.
1503
- *
1504
- * @example
1505
- * ```typescript
1506
- * const jobs = await manager.popMany('default', 50);
1507
- * ```
1508
- */
1509
- popMany(queue?: string, count?: number, connection?: string): Promise<Job[]>;
1510
- /**
1511
- * Retrieves the current size of a queue.
1512
- *
1513
- * @param queue - The queue name (default: 'default').
1514
- * @param connection - The connection name.
1515
- * @returns The number of waiting jobs.
1516
- *
1517
- * @example
1518
- * ```typescript
1519
- * const count = await manager.size('emails');
1520
- * ```
1521
- */
1522
- size(queue?: string, connection?: string): Promise<number>;
1523
- /**
1524
- * Pops a job from the queue with blocking (wait) behavior.
1525
- *
1526
- * Waits for a job to become available for the specified timeout duration.
1527
- * Useful for reducing polling loop frequency.
1528
- *
1529
- * @param queues - A queue name or array of queue names to listen to.
1530
- * @param timeout - Timeout in seconds (0 = block indefinitely).
1531
- * @param connection - The connection name.
1532
- * @returns A Job instance if found, or `null` if timed out.
1533
- *
1534
- * @example
1535
- * ```typescript
1536
- * // Wait up to 30 seconds for a job
1537
- * const job = await manager.popBlocking('default', 30);
1538
- * ```
1539
- */
1540
- popBlocking(queues?: string | string[], timeout?: number, connection?: string): Promise<Job | null>;
1541
- /**
1542
- * Removes all jobs from a specific queue.
1543
- *
1544
- * @param queue - The queue name to purge.
1545
- * @param connection - The connection name.
1546
- *
1547
- * @example
1548
- * ```typescript
1549
- * await manager.clear('test-queue');
1550
- * ```
1551
- */
1552
- clear(queue?: string, connection?: string): Promise<void>;
1553
- /**
1554
- * Retrieves comprehensive statistics for a queue.
1555
- *
1556
- * Includes counts for pending, processing, delayed, and failed jobs.
1557
- *
1558
- * @param queue - The queue name.
1559
- * @param connection - The connection name.
1560
- * @returns A QueueStats object.
1561
- *
1562
- * @example
1563
- * ```typescript
1564
- * const stats = await manager.stats('default');
1565
- * console.log(stats.size, stats.failed);
1566
- * ```
1567
- */
1568
- stats(queue?: string, connection?: string): Promise<QueueStats>;
1569
- /**
1570
- * Marks a job as successfully completed.
1571
- *
1572
- * Removes the job from the processing state and optionally archives it.
1573
- *
1574
- * @param job - The job instance that finished.
1575
- *
1576
- * @example
1577
- * ```typescript
1578
- * await manager.complete(job);
1579
- * ```
1580
- */
1581
- complete<T extends Job & Queueable>(job: T): Promise<void>;
1582
- /**
1583
- * Marks a job as failed.
1584
- *
1585
- * Moves the job to the failed state (Dead Letter Queue) and optionally archives it.
1586
- * This is typically called after max retry attempts are exhausted.
1587
- *
1588
- * @param job - The job instance that failed.
1589
- * @param error - The error that caused the failure.
1590
- *
1591
- * @example
1592
- * ```typescript
1593
- * await manager.fail(job, new Error('Something went wrong'));
1594
- * ```
1595
- */
1596
- fail<T extends Job & Queueable>(job: T, error: Error): Promise<void>;
1597
- /**
1598
- * Retrieves the configured persistence adapter.
1599
- *
1600
- * @returns The PersistenceAdapter instance, or undefined if not configured.
1601
- */
1602
- getPersistence(): PersistenceAdapter | undefined;
1603
- /**
1604
- * Gets the Scheduler instance associated with this manager.
1605
- *
1606
- * The Scheduler handles delayed jobs and periodic tasks.
1607
- *
1608
- * @returns The Scheduler instance.
1609
- */
1610
- getScheduler(): Scheduler;
1611
- /**
1612
- * Retrieves failed jobs from the Dead Letter Queue.
1613
- *
1614
- * @param queue - The queue name.
1615
- * @param start - The starting index (pagination).
1616
- * @param end - The ending index (pagination).
1617
- * @param connection - The connection name.
1618
- * @returns An array of serialized jobs.
1619
- *
1620
- * @example
1621
- * ```typescript
1622
- * const failedJobs = await manager.getFailed('default', 0, 10);
1623
- * ```
1624
- */
1625
- getFailed(queue: string, start?: number, end?: number, connection?: string): Promise<SerializedJob[]>;
1626
- /**
1627
- * Retries failed jobs from the Dead Letter Queue.
1628
- *
1629
- * Moves jobs from the failed state back to the active queue for re-processing.
1630
- *
1631
- * @param queue - The queue name.
1632
- * @param count - The number of jobs to retry.
1633
- * @param connection - The connection name.
1634
- * @returns The number of jobs successfully retried.
1635
- *
1636
- * @example
1637
- * ```typescript
1638
- * await manager.retryFailed('default', 5);
1639
- * ```
1640
- */
1641
- retryFailed(queue: string, count?: number, connection?: string): Promise<number>;
1642
- /**
1643
- * Clears all failed jobs from the Dead Letter Queue.
1644
- *
1645
- * @param queue - The queue name.
1646
- * @param connection - The connection name.
1647
- *
1648
- * @example
1649
- * ```typescript
1650
- * await manager.clearFailed('default');
1651
- * ```
1652
- */
1653
- clearFailed(queue: string, connection?: string): Promise<void>;
1654
- /**
1655
- * Retrieves high-level statistics across all registered connections and queues.
1656
- *
1657
- * Iterates through all drivers and collects metadata to provide a comprehensive
1658
- * snapshot of the entire queue system's health.
1659
- *
1660
- * @returns A promise resolving to a GlobalStats object.
1661
- */
1662
- getGlobalStats(): Promise<GlobalStats>;
1663
- }
1664
-
1665
- /**
1666
- * Configuration options for the BatchConsumer.
1667
- *
1668
- * @example
1669
- * ```typescript
1670
- * const options: BatchConsumerOptions = {
1671
- * batchSize: 50,
1672
- * autoAck: false
1673
- * };
1674
- * ```
1675
- */
1676
- interface BatchConsumerOptions {
1677
- /**
1678
- * The name of the queue to consume from.
1679
- * @default 'default'
1680
- */
1681
- queue?: string;
1682
- /**
1683
- * The connection name to use.
1684
- * @default The default connection of QueueManager
1685
- */
1686
- connection?: string;
1687
- /**
1688
- * The number of jobs to try to retrieve in each batch.
1689
- * @default 10
1690
- */
1691
- batchSize?: number;
1692
- /**
1693
- * The polling interval in milliseconds when the queue is empty.
1694
- * @default 1000
1695
- */
1696
- pollInterval?: number;
1697
- /**
1698
- * Whether to automatically complete jobs after the handler returns successfully.
1699
- *
1700
- * If set to `false`, the handler function is responsible for calling `manager.complete()`
1701
- * or `manager.fail()` for each job.
1702
- *
1703
- * @default true
1704
- */
1705
- autoAck?: boolean;
1706
- }
1707
- /**
1708
- * Specialized consumer for processing jobs in bulk.
1709
- *
1710
- * Unlike the standard `Consumer` which processes jobs individually (even if fetched in batches),
1711
- * the `BatchConsumer` passes an array of jobs to a single handler function. This is ideal for
1712
- * operations that benefit from bulk processing, such as database inserts or API calls that support batching.
1713
- *
1714
- * @public
1715
- * @example
1716
- * ```typescript
1717
- * const consumer = new BatchConsumer(manager, async (jobs) => {
1718
- * // Process 100 jobs at once
1719
- * await elasticsearch.bulkIndex(jobs.map(j => j.data));
1720
- * }, { batchSize: 100 });
1721
- *
1722
- * consumer.start();
1723
- * ```
1724
- */
1725
- declare class BatchConsumer {
1726
- private manager;
1727
- private handler;
1728
- private running;
1729
- private options;
1730
- constructor(manager: QueueManager, handler: (jobs: Job[]) => Promise<void>, options?: BatchConsumerOptions);
1731
- /**
1732
- * Starts the batch consuming loop.
1733
- *
1734
- * Continuously polls for batches of jobs and passes them to the handler.
1735
- */
1736
- start(): Promise<void>;
1737
- /**
1738
- * Stops the consumer loop.
1739
- *
1740
- * Sets the running flag to false. The loop will exit after the current iteration finishes.
1741
- */
1742
- stop(): void;
1743
- }
1744
-
1745
- /**
1746
- * Sandboxed Worker Implementation.
1747
- *
1748
- * Executes jobs in isolated Worker Threads to provide context isolation,
1749
- * error containment, and resource limits.
1750
- *
1751
- * @public
1752
- */
1753
-
1754
- /**
1755
- * Configuration options for the Sandboxed Worker.
1756
- */
1757
- interface SandboxedWorkerConfig {
1758
- /**
1759
- * Maximum execution time for a job in milliseconds.
1760
- *
1761
- * Jobs exceeding this duration will be forcefully terminated.
1762
- * @default 30000 (30 seconds)
1763
- */
1764
- maxExecutionTime?: number;
1765
- /**
1766
- * Maximum memory limit for the worker in MB.
1767
- *
1768
- * If the worker exceeds this limit, it will be terminated and restarted.
1769
- * Note: Relies on `resourceLimits` which may vary by platform.
1770
- * @default undefined (unlimited)
1771
- */
1772
- maxMemory?: number;
1773
- /**
1774
- * Whether to isolate contexts for each job.
1775
- *
1776
- * If `true`, a new Worker Thread is created for every job execution.
1777
- * If `false`, the Worker Thread is reused across multiple jobs.
1778
- * @default false
1779
- */
1780
- isolateContexts?: boolean;
1781
- /**
1782
- * Idle timeout for the Worker Thread in milliseconds.
1783
- *
1784
- * The worker will be terminated if it remains idle for this duration to save resources.
1785
- * @default 60000 (60 seconds)
1786
- */
1787
- idleTimeout?: number;
1788
- }
1789
- /**
1790
- * Sandboxed Worker.
1791
- *
1792
- * Manages the lifecycle of a Node.js Worker Thread for job execution.
1793
- * Provides features like:
1794
- * - Context Isolation: Run code in a separate thread.
1795
- * - Timeout Enforcement: Terminate hangs or long-running jobs.
1796
- * - Memory Limits: Prevent OOM issues affecting the main process.
1797
- * - Error Containment: Worker crashes do not crash the main application.
1798
- *
1799
- * @example
1800
- * ```typescript
1801
- * const worker = new SandboxedWorker({
1802
- * maxExecutionTime: 30000,
1803
- * maxMemory: 512,
1804
- * isolateContexts: true
1805
- * });
1806
- *
1807
- * await worker.execute(serializedJob);
1808
- * await worker.terminate();
1809
- * ```
1810
- */
1811
- declare class SandboxedWorker {
1812
- private worker;
1813
- private state;
1814
- private config;
1815
- private idleTimer;
1816
- private executionTimer;
1817
- /**
1818
- * Creates a SandboxedWorker instance.
1819
- *
1820
- * @param config - Configuration options for the worker.
1821
- */
1822
- constructor(config?: SandboxedWorkerConfig);
1823
- /**
1824
- * Initializes the Worker Thread.
1825
- *
1826
- * @returns The active Worker Thread instance.
1827
- * @throws {Error} If worker initialization fails or times out.
1828
- */
1829
- private initWorker;
1830
- /**
1831
- * Executes a job in the sandboxed environment.
1832
- *
1833
- * @param job - The serialized job data to execute.
1834
- * @throws {Error} If execution fails, times out, or the worker crashes.
1835
- */
1836
- execute(job: SerializedJob): Promise<void>;
1837
- /**
1838
- * Internal method to send execution message to the worker thread.
1839
- *
1840
- * @param worker - The worker thread instance.
1841
- * @param job - Job data.
1842
- */
1843
- private executeInWorker;
1844
- /**
1845
- * Creates a promise that rejects after the configured timeout.
1846
- */
1847
- private createTimeoutPromise;
1848
- /**
1849
- * Starts the idle timer to auto-terminate the worker.
1850
- */
1851
- private startIdleTimer;
1852
- /**
1853
- * Terminates the Worker Thread immediately.
1854
- *
1855
- * Stops any running job and releases resources.
1856
- */
1857
- terminate(): Promise<void>;
1858
- /**
1859
- * Gets the current state of the worker.
1860
- *
1861
- * @returns The current `WorkerState`.
1862
- */
1863
- getState(): string;
1864
- /**
1865
- * Checks if the worker is ready to accept a job.
1866
- *
1867
- * @returns `true` if ready, `false` otherwise.
1868
- */
1869
- isReady(): boolean;
1870
- /**
1871
- * Checks if the worker is currently executing a job.
1872
- *
1873
- * @returns `true` if busy, `false` otherwise.
1874
- */
1875
- isBusy(): boolean;
1876
- }
1877
-
1878
- /**
1879
- * Configuration options for the Worker.
1880
- *
1881
- * Controls the execution behavior of jobs, including retry limits and timeouts.
1882
- *
1883
- * @example
1884
- * ```typescript
1885
- * const options: WorkerOptions = {
1886
- * maxAttempts: 3,
1887
- * timeout: 30
1888
- * };
1889
- * ```
1890
- */
1891
- interface WorkerOptions {
1892
- /**
1893
- * The maximum number of attempts for a job before it is marked as failed.
1894
- *
1895
- * This value serves as a default fallback if the job itself does not specify `maxAttempts`.
1896
- */
1897
- maxAttempts?: number;
1898
- /**
1899
- * The maximum execution time for a job in seconds.
1900
- *
1901
- * If the job exceeds this duration, it will be timed out and marked as failed.
1902
- */
1903
- timeout?: number;
1904
- /**
1905
- * Callback function triggered when a job permanently fails.
1906
- *
1907
- * This allows for custom error reporting or cleanup logic outside of the job class.
1908
- */
1909
- onFailed?: (job: Job, error: Error) => Promise<void>;
1910
- /**
1911
- * Enable sandboxed execution using Worker Threads.
1912
- *
1913
- * When enabled, jobs are executed in isolated Worker Threads, providing:
1914
- * - Context isolation: Each job runs in a separate execution environment
1915
- * - Crash protection: Worker crashes don't affect the main thread
1916
- * - Memory limits: Prevent memory leaks from affecting the main process
1917
- * - Timeout enforcement: Jobs exceeding the timeout are forcefully terminated
1918
- *
1919
- * @default false
1920
- */
1921
- sandboxed?: boolean;
1922
- /**
1923
- * Sandboxed worker configuration options.
1924
- *
1925
- * Only used when `sandboxed` is true.
1926
- */
1927
- sandboxConfig?: SandboxedWorkerConfig;
1928
- }
1929
- /**
1930
- * Executes background jobs.
1931
- *
1932
- * The Worker is responsible for running the `handle()` method of a job, managing its lifecycle,
1933
- * enforcing timeouts, and handling retries or failures.
1934
- *
1935
- * Supports two execution modes:
1936
- * - **Standard Mode** (default): Executes jobs directly in the current process
1937
- * - **Sandboxed Mode**: Executes jobs in isolated Worker Threads for enhanced security and stability
1938
- *
1939
- * @example
1940
- * ```typescript
1941
- * // Standard mode
1942
- * const worker = new Worker({
1943
- * maxAttempts: 3,
1944
- * timeout: 60
1945
- * });
1946
- *
1947
- * // Sandboxed mode
1948
- * const sandboxedWorker = new Worker({
1949
- * maxAttempts: 3,
1950
- * timeout: 60,
1951
- * sandboxed: true,
1952
- * sandboxConfig: {
1953
- * maxExecutionTime: 30000,
1954
- * maxMemory: 512,
1955
- * isolateContexts: true
1956
- * }
1957
- * });
1958
- *
1959
- * await worker.process(job);
1960
- * ```
1961
- */
1962
- declare class Worker {
1963
- private options;
1964
- private sandboxedWorker?;
1965
- constructor(options?: WorkerOptions);
1966
- /**
1967
- * Processes a single job instance.
1968
- *
1969
- * 1. Checks attempt counts.
1970
- * 2. Enforces execution timeout (if configured).
1971
- * 3. Runs `job.handle()` (either directly or in a sandboxed Worker Thread).
1972
- * 4. Catches errors and invokes failure handlers if max attempts are reached.
1973
- *
1974
- * @param job - The job to process.
1975
- * @throws {Error} If the job execution fails (to trigger retry logic in the consumer).
1976
- */
1977
- process(job: Job): Promise<void>;
1978
- /**
1979
- * Processes a job in standard mode (directly in current process).
1980
- *
1981
- * @param job - The job to process.
1982
- * @param timeout - Optional timeout in seconds.
1983
- */
1984
- private processStandard;
1985
- /**
1986
- * Processes a job in sandboxed mode (in Worker Thread).
1987
- *
1988
- * @param job - The job to process.
1989
- */
1990
- private processSandboxed;
1991
- /**
1992
- * Serializes a Job instance for Worker Thread execution.
1993
- *
1994
- * @param job - The job to serialize.
1995
- * @returns Serialized job data.
1996
- */
1997
- private serializeJob;
1998
- /**
1999
- * Handles the permanent failure of a job.
2000
- *
2001
- * Invokes the job's `failed()` method and any global `onFailed` callback.
2002
- *
2003
- * @param job - The failed job.
2004
- * @param error - The error that caused the failure.
2005
- */
2006
- private handleFailure;
2007
- /**
2008
- * Terminates the sandboxed worker and releases resources.
2009
- *
2010
- * Should be called when the worker is no longer needed.
2011
- * Only applicable when running in sandboxed mode.
2012
- */
2013
- terminate(): Promise<void>;
2014
- }
2015
-
2016
- /**
2017
- * Configuration options for the Consumer.
2018
- *
2019
- * Defines which queues to listen to, connection settings, concurrency levels,
2020
- * and advanced behavior like rate limiting and batch processing.
2021
- *
2022
- * @example
2023
- * ```typescript
2024
- * const options: ConsumerOptions = {
2025
- * queues: ['emails', 'notifications'],
2026
- * concurrency: 5,
2027
- * pollInterval: 2000
2028
- * };
2029
- * ```
2030
- */
2031
- interface ConsumerOptions {
2032
- /**
2033
- * List of queue names to consume jobs from.
2034
- *
2035
- * The consumer will poll these queues in the order provided or based on driver logic.
2036
- */
2037
- queues: string[];
2038
- /**
2039
- * The connection name to use (e.g., 'redis', 'sqs').
2040
- *
2041
- * If not provided, uses the default connection from QueueManager.
2042
- */
2043
- connection?: string;
2044
- /**
2045
- * Configuration options passed to the underlying Worker.
2046
- */
2047
- workerOptions?: WorkerOptions;
2048
- /**
2049
- * The interval in milliseconds to wait before polling again when the queue is empty.
2050
- */
2051
- pollInterval?: number;
2052
- /**
2053
- * Whether to keep the process alive when queues are empty.
2054
- *
2055
- * If false, the consumer will exit the loop when no jobs are found (useful for one-off scripts).
2056
- */
2057
- keepAlive?: boolean;
2058
- /**
2059
- * Monitoring configuration.
2060
- *
2061
- * Can be a boolean to enable default monitoring, or an object for advanced configuration.
2062
- */
2063
- monitor?: boolean | {
2064
- /**
2065
- * The interval in milliseconds for sending heartbeat updates.
2066
- * @default 5000
2067
- */
2068
- interval?: number;
2069
- /**
2070
- * Additional metadata to include in heartbeat payloads.
2071
- */
2072
- extraInfo?: Record<string, unknown>;
2073
- /**
2074
- * Key prefix for monitoring events (e.g. for Redis Pub/Sub).
2075
- */
2076
- prefix?: string;
2077
- };
2078
- /**
2079
- * Rate limiting configuration per queue.
2080
- *
2081
- * Defines the maximum number of jobs to process within a given duration.
2082
- *
2083
- * @example
2084
- * ```typescript
2085
- * { 'emails': { max: 10, duration: 1000 } } // 10 emails per second
2086
- * ```
2087
- */
2088
- rateLimits?: Record<string, {
2089
- max: number;
2090
- duration: number;
2091
- }>;
2092
- /**
2093
- * The maximum number of jobs to process concurrently.
2094
- *
2095
- * @default 1
2096
- */
2097
- concurrency?: number;
2098
- /**
2099
- * Whether to enforce sequential processing for jobs with the same `groupId`.
2100
- *
2101
- * If true, jobs sharing a `groupId` will be processed one after another,
2102
- * even if global concurrency is high.
2103
- *
2104
- * @default true
2105
- */
2106
- groupJobsSequential?: boolean;
2107
- /**
2108
- * The minimum polling interval in milliseconds for adaptive polling.
2109
- *
2110
- * @default 100
2111
- */
2112
- minPollInterval?: number;
2113
- /**
2114
- * The maximum polling interval in milliseconds for adaptive polling.
2115
- *
2116
- * @default 5000
2117
- */
2118
- maxPollInterval?: number;
2119
- /**
2120
- * The multiplier used to increase the polling interval when the queue is empty.
2121
- *
2122
- * @default 1.5
2123
- */
2124
- backoffMultiplier?: number;
2125
- /**
2126
- * The number of jobs to try to fetch in a single request.
2127
- *
2128
- * If supported by the driver, fetching multiple jobs reduces network round-trips.
2129
- *
2130
- * @default 1
2131
- */
2132
- batchSize?: number;
2133
- /**
2134
- * Whether to use blocking operations (like BLPOP in Redis) when polling.
2135
- *
2136
- * Significant optimization for low-latency job pickup. Only applies when `batchSize` is 1.
2137
- *
2138
- * @default true
2139
- */
2140
- useBlocking?: boolean;
2141
- /**
2142
- * The timeout in seconds for blocking operations.
2143
- *
2144
- * @default 5
2145
- */
2146
- blockingTimeout?: number;
2147
- /**
2148
- * Enable verbose debug logging for consumer activities.
2149
- *
2150
- * @default false
2151
- */
2152
- debug?: boolean;
2153
- /**
2154
- * 最大處理請求數量。
2155
- *
2156
- * 當 consumer 處理完這個數量的 job 後會自動停止(觸發 max_requests_reached 事件)。
2157
- * 適用於需要定期重啟 worker 的場景(避免記憶體累積、載入最新程式碼等)。
2158
- *
2159
- * @default undefined (無限制)
2160
- */
2161
- maxRequests?: number;
2162
- /**
2163
- * Optional event callback for external monitoring systems.
2164
- *
2165
- * Called whenever a job lifecycle event occurs (started, processed, failed, etc.).
2166
- */
2167
- onEvent?: (event: string, payload: any) => void;
2168
- }
2169
- /**
2170
- * The Consumer responsible for processing jobs from the queue.
2171
- *
2172
- * It polls the configured queues, retrieves jobs, and delegates execution to a `Worker`.
2173
- * It handles concurrency, rate limiting, adaptive polling, and emits lifecycle events
2174
- * (job:started, job:processed, job:failed, etc.).
2175
- *
2176
- * @public
2177
- * @example
2178
- * ```typescript
2179
- * const consumer = new Consumer(queueManager, {
2180
- * queues: ['default'],
2181
- * concurrency: 10
2182
- * });
2183
- *
2184
- * await consumer.start();
2185
- * ```
2186
- *
2187
- * @emits job:started - When a job begins processing. Payload: { job: Job, queue: string }
2188
- * @emits job:processed - When a job completes successfully. Payload: { job: Job, duration: number, queue: string }
2189
- * @emits job:failed - When a job fails an attempt. Payload: { job: Job, error: Error, duration: number, queue: string }
2190
- * @emits job:retried - When a job is scheduled for a retry. Payload: { job: Job, attempt: number, delay: number }
2191
- * @emits job:failed_permanently - When a job fails all attempts. Payload: { job: Job, error: Error }
2192
- */
2193
- declare class Consumer extends EventEmitter {
2194
- private queueManager;
2195
- private options;
2196
- /**
2197
- * Group limiter 的存活時間(毫秒)。
2198
- * 超過此時間未使用的 group limiter 會被清理,避免記憶體洩漏。
2199
- */
2200
- private static readonly GROUP_LIMITER_TTL;
2201
- private running;
2202
- private stopRequested;
2203
- private workerId;
2204
- private heartbeatTimer;
2205
- private cleanupTimer;
2206
- private groupLimiters;
2207
- private groupLimiterLastUsed;
2208
- private stats;
2209
- constructor(queueManager: QueueManager, options: ConsumerOptions);
2210
- private get connectionName();
2211
- /**
2212
- * Logs a debug message if debug mode is enabled.
2213
- */
2214
- private log;
2215
- /**
2216
- * Starts the consumer loop.
2217
- *
2218
- * Begins polling the queues and processing jobs. This method returns a promise that resolves
2219
- * only when the consumer stops (if `keepAlive` is false) or throws if already running.
2220
- *
2221
- * @throws {Error} If the consumer is already running.
2222
- */
2223
- start(): Promise<void>;
2224
- /**
2225
- * Run a job with concurrency controls and group locking.
2226
- */
2227
- private runJob;
2228
- /**
2229
- * Delegates the actual processing to the worker and handles stats/logging.
2230
- */
2231
- private handleJob;
2232
- private startHeartbeat;
2233
- private stopHeartbeat;
2234
- /**
2235
- * 清理閒置的 group limiters。
2236
- *
2237
- * 定期檢查並移除超過 TTL 且沒有 active/pending jobs 的 group limiters,
2238
- * 避免記憶體洩漏。
2239
- */
2240
- private cleanupGroupLimiters;
2241
- /**
2242
- * 啟動 group limiter 清理計時器。
2243
- */
2244
- private startCleanupTimer;
2245
- /**
2246
- * 停止 group limiter 清理計時器。
2247
- */
2248
- private stopCleanupTimer;
2249
- private publishLog;
2250
- /**
2251
- * Gracefully stops the consumer.
2252
- *
2253
- * Signals the consumer to stop accepting new jobs and waits for currently running jobs
2254
- * to complete.
2255
- *
2256
- * @returns A promise that resolves when the consumer has fully stopped.
2257
- */
2258
- stop(): Promise<void>;
2259
- /**
2260
- * Checks if the consumer is currently active.
2261
- *
2262
- * @returns True if the consumer loop is running.
2263
- */
2264
- isRunning(): boolean;
2265
- /**
2266
- * Retrieves current operational statistics.
2267
- *
2268
- * @returns An object containing processed, failed, retried, and active job counts.
2269
- */
2270
- getStats(): {
2271
- processed: number;
2272
- failed: number;
2273
- retried: number;
2274
- active: number;
2275
- };
2276
- /**
2277
- * Resets the internal statistics counters.
2278
- */
2279
- resetStats(): void;
2280
- }
2281
-
2282
- /**
2283
- * Bull Queue client interface (compatible with bullmq package).
2284
- */
2285
- interface BullQueueClient {
2286
- add(name: string, data: any, options?: any): Promise<any>;
2287
- getJob(id: string): Promise<any | null>;
2288
- count(): Promise<number>;
2289
- process(handler: (job: any) => Promise<void>): void;
2290
- on(event: string, handler: (...args: any[]) => void): void;
2291
- off(event: string, handler: (...args: any[]) => void): void;
2292
- pause(): Promise<void>;
2293
- resume(): Promise<void>;
2294
- clean(grace: number, limit?: number, type?: string): Promise<number>;
2295
- close(): Promise<void>;
2296
- getJobCounts(types?: string[]): Promise<Record<string, number>>;
2297
- getDelayedCount(): Promise<number>;
2298
- getFailedCount(): Promise<number>;
2299
- getActiveCount(): Promise<number>;
2300
- [key: string]: any;
2301
- }
2302
- interface BullWorkerClient {
2303
- [key: string]: any;
2304
- }
2305
- /**
2306
- * Bull Queue driver configuration.
2307
- */
2308
- interface BullMQDriverConfig {
2309
- /**
2310
- * Bull Queue instance (from bullmq package).
2311
- */
2312
- queue: BullQueueClient;
2313
- /**
2314
- * Optional Bull Worker instance for processing jobs.
2315
- */
2316
- worker?: BullWorkerClient;
2317
- /**
2318
- * Connection options (host, port, etc. for Redis).
2319
- */
2320
- connection?: {
2321
- host?: string;
2322
- port?: number;
2323
- password?: string;
2324
- db?: number;
2325
- [key: string]: any;
2326
- };
2327
- /**
2328
- * Default number of concurrent workers (default: 1).
2329
- */
2330
- concurrency?: number;
2331
- /**
2332
- * Key prefix for namespacing queues (default: 'gravito:').
2333
- */
2334
- prefix?: string;
2335
- /**
2336
- * Enable debug logging.
2337
- */
2338
- debug?: boolean;
2339
- }
2340
- /**
2341
- * Bull Queue driver implementation.
2342
- *
2343
- * Provides high-performance, persistent job queuing using Bull Queue (backed by Redis).
2344
- * Supports priority, delays, retries, and group-based FIFO processing.
2345
- *
2346
- * @public
2347
- * @example
2348
- * ```typescript
2349
- * import { Queue } from 'bullmq'
2350
- * import Redis from 'ioredis'
2351
- *
2352
- * const redis = new Redis()
2353
- * const queue = new Queue('gravito-events', { connection: redis })
2354
- * const driver = new BullMQDriver({ queue })
2355
- * ```
2356
- */
2357
- declare class BullMQDriver implements QueueDriver {
2358
- private queue;
2359
- private prefix;
2360
- private debug;
2361
- private queueMap;
2362
- constructor(config: BullMQDriverConfig);
2363
- /**
2364
- * Get or create a queue for the given queue name.
2365
- */
2366
- private getQueue;
2367
- /**
2368
- * Build Job Options from JobPushOptions.
2369
- */
2370
- private buildJobOptions;
2371
- /**
2372
- * Create Bull job data from SerializedJob.
2373
- */
2374
- private createBullJobData;
2375
- /**
2376
- * Pushes a job to Bull Queue.
2377
- */
2378
- push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
2379
- /**
2380
- * Pops a job from Bull Queue.
2381
- * Note: Bull Queue typically uses Workers, not manual pop.
2382
- * This is a fallback implementation.
2383
- */
2384
- pop(queue: string): Promise<SerializedJob | null>;
2385
- /**
2386
- * Returns the size of the queue.
2387
- */
2388
- size(queue: string): Promise<number>;
2389
- /**
2390
- * Clears the queue.
2391
- */
2392
- clear(queue: string): Promise<void>;
2393
- /**
2394
- * Marks a job as failed (moves to failed list).
2395
- */
2396
- fail(queue: string, job: SerializedJob): Promise<void>;
2397
- /**
2398
- * Returns detailed statistics for the queue.
2399
- */
2400
- stats(queue: string): Promise<QueueStats>;
2401
- /**
2402
- * Retrieves failed jobs from the Dead Letter Queue.
2403
- */
2404
- getFailed(queue: string, _start?: number, _end?: number): Promise<SerializedJob[]>;
2405
- /**
2406
- * Retries failed jobs.
2407
- */
2408
- retryFailed(queue: string, _count?: number): Promise<number>;
2409
- /**
2410
- * Clears the Dead Letter Queue.
2411
- */
2412
- clearFailed(queue: string): Promise<void>;
2413
- /**
2414
- * Creates a new queue/topic.
2415
- */
2416
- createTopic(_topic: string, _options?: TopicOptions): Promise<void>;
2417
- /**
2418
- * Deletes a queue/topic.
2419
- */
2420
- deleteTopic(topic: string): Promise<void>;
2421
- /**
2422
- * Pushes multiple jobs in batch.
2423
- */
2424
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
2425
- /**
2426
- * Pops multiple jobs in batch.
2427
- */
2428
- popMany(_queue: string, _count: number): Promise<SerializedJob[]>;
2429
- /**
2430
- * Reports worker heartbeat.
2431
- */
2432
- reportHeartbeat(workerInfo: {
2433
- id: string;
2434
- status: string;
2435
- hostname: string;
2436
- pid: number;
2437
- uptime: number;
2438
- last_ping: string;
2439
- queues: string[];
2440
- metrics?: Record<string, any>;
2441
- [key: string]: any;
2442
- }, _prefix?: string): Promise<void>;
2443
- /**
2444
- * Publishes a log message.
2445
- */
2446
- publishLog(logPayload: {
2447
- level: string;
2448
- message: string;
2449
- workerId: string;
2450
- jobId?: string;
2451
- timestamp: string;
2452
- [key: string]: any;
2453
- }, _prefix?: string): Promise<void>;
2454
- /**
2455
- * Checks rate limit for a queue.
2456
- */
2457
- checkRateLimit(_queue: string, _config: {
2458
- max: number;
2459
- duration: number;
2460
- }): Promise<boolean>;
2461
- /**
2462
- * Retrieves all queue names.
2463
- */
2464
- getQueues(): Promise<string[]>;
2465
- }
2466
-
2467
- /**
2468
- * Generic database service interface.
2469
- *
2470
- * Adapts any SQL database client (e.g., pg, mysql2, sqlite3) for use with the DatabaseDriver.
2471
- * Users must provide an implementation of this interface that wraps their specific DB library.
2472
- */
2473
- interface DatabaseService {
2474
- /**
2475
- * Execute a raw SQL query.
2476
- *
2477
- * @param sql - The SQL query string with placeholders (e.g., $1, ?).
2478
- * @param bindings - The values to bind to the placeholders.
2479
- * @returns The query result (rows or metadata).
2480
- */
2481
- execute<T = unknown>(sql: string, bindings?: unknown[]): Promise<T[] | T>;
2482
- /**
2483
- * Execute multiple queries within a single transaction.
2484
- *
2485
- * @param callback - A function that receives a transaction-scoped service instance.
2486
- * @returns The result of the callback.
2487
- */
2488
- transaction<T>(callback: (tx: DatabaseService) => Promise<T>): Promise<T>;
2489
- }
2490
- /**
2491
- * Configuration options for the DatabaseDriver.
2492
- */
2493
- interface DatabaseDriverConfig {
2494
- /**
2495
- * The name of the table used to store jobs.
2496
- * @default 'jobs'
2497
- */
2498
- table?: string;
2499
- /**
2500
- * The database service adapter instance.
2501
- */
2502
- dbService?: DatabaseService;
2503
- }
2504
- /**
2505
- * Database-backed queue driver.
2506
- *
2507
- * Persists jobs in a SQL database table. Supports delayed jobs, reservation (locking),
2508
- * and reliable delivery. Compatible with PostgreSQL (SKIP LOCKED), MySQL, and SQLite.
2509
- *
2510
- * @public
2511
- * @example
2512
- * ```typescript
2513
- * const driver = new DatabaseDriver({
2514
- * dbService: myDbAdapter,
2515
- * table: 'queue_jobs'
2516
- * });
2517
- * ```
2518
- */
2519
- declare class DatabaseDriver implements QueueDriver {
2520
- private tableName;
2521
- private dbService;
2522
- constructor(config: DatabaseDriverConfig);
2523
- /**
2524
- * Pushes a job to the database queue.
2525
- *
2526
- * Inserts a new row into the jobs table.
2527
- *
2528
- * @param queue - The queue name.
2529
- * @param job - The serialized job.
2530
- */
2531
- push(queue: string, job: SerializedJob): Promise<void>;
2532
- /**
2533
- * Pops the next available job from the queue.
2534
- *
2535
- * Uses transactional locking (SELECT ... FOR UPDATE SKIP LOCKED if supported) to ensure
2536
- * atomic reservation of jobs by workers.
2537
- *
2538
- * @param queue - The queue name.
2539
- * @returns The job or `null`.
2540
- */
2541
- pop(queue: string): Promise<SerializedJob | null>;
2542
- /**
2543
- * Pops multiple jobs from the queue in a single transaction.
2544
- *
2545
- * @param queue - The queue name.
2546
- * @param count - Max jobs to pop.
2547
- */
2548
- popMany(queue: string, count: number): Promise<SerializedJob[]>;
2549
- /**
2550
- * Retrieves queue statistics by querying the table.
2551
- *
2552
- * @param queue - The queue name.
2553
- */
2554
- stats(queue: string): Promise<QueueStats>;
2555
- /**
2556
- * Returns the count of pending jobs.
2557
- *
2558
- * @param queue - The queue name.
2559
- */
2560
- size(queue: string): Promise<number>;
2561
- /**
2562
- * Clears the queue by deleting all rows for the queue.
2563
- *
2564
- * @param queue - The queue name.
2565
- */
2566
- clear(queue: string): Promise<void>;
2567
- /**
2568
- * Pops a job using a polling loop (Blocking simulation).
2569
- *
2570
- * @param queue - The queue name.
2571
- * @param timeout - Timeout in seconds.
2572
- */
2573
- popBlocking(queue: string, timeout: number): Promise<SerializedJob | null>;
2574
- /**
2575
- * Pushes multiple jobs using a transaction.
2576
- *
2577
- * @param queue - The queue name.
2578
- * @param jobs - Array of jobs.
2579
- */
2580
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
2581
- /**
2582
- * Marks a job as permanently failed by moving it to the DLQ (separate logical queue in DB).
2583
- *
2584
- * @param queue - The queue name.
2585
- * @param job - The failed job.
2586
- */
2587
- fail(queue: string, job: SerializedJob): Promise<void>;
2588
- /**
2589
- * Deletes a job row from the database (completion).
2590
- *
2591
- * @param _queue - The queue name (unused).
2592
- * @param job - The job to complete.
2593
- */
2594
- complete(_queue: string, job: SerializedJob): Promise<void>;
2595
- }
2596
-
2597
- declare class GrpcDriver implements QueueDriver {
2598
- private client;
2599
- constructor(config: GrpcDriverConfig);
2600
- private getCredentials;
2601
- push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
2602
- pop(queue: string): Promise<SerializedJob | null>;
2603
- size(queue: string): Promise<number>;
2604
- clear(queue: string): Promise<void>;
2605
- acknowledge(messageId: string): Promise<void>;
2606
- stats(queue: string): Promise<QueueStats>;
2607
- private toProtoJob;
2608
- private fromProtoJob;
2609
- }
2610
-
2611
- /**
2612
- * Kafka driver configuration.
2613
- */
2614
- interface KafkaDriverConfig {
2615
- /**
2616
- * Kafka client instance (kafkajs).
2617
- *
2618
- * Must provide producer, admin, and consumer factories compatible with KafkaJS.
2619
- */
2620
- client: {
2621
- producer: () => {
2622
- connect: () => Promise<void>;
2623
- send: (args: {
2624
- topic: string;
2625
- messages: Array<{
2626
- key?: string;
2627
- value: string;
2628
- }>;
2629
- }) => Promise<void>;
2630
- disconnect: () => Promise<void>;
2631
- };
2632
- admin: () => {
2633
- connect: () => Promise<void>;
2634
- createTopics: (args: {
2635
- topics: Array<{
2636
- topic: string;
2637
- numPartitions?: number;
2638
- replicationFactor?: number;
2639
- }>;
2640
- }) => Promise<void>;
2641
- deleteTopics: (args: {
2642
- topics: string[];
2643
- }) => Promise<void>;
2644
- disconnect: () => Promise<void>;
2645
- };
2646
- consumer: (args: {
2647
- groupId: string;
2648
- }) => {
2649
- connect: () => Promise<void>;
2650
- subscribe: (args: {
2651
- topics: string[];
2652
- }) => Promise<void>;
2653
- run: (args: {
2654
- eachMessage: (args: {
2655
- topic: string;
2656
- partition: number;
2657
- message: {
2658
- key?: Buffer;
2659
- value: Buffer;
2660
- offset: string;
2661
- };
2662
- }) => Promise<void>;
2663
- }) => Promise<void>;
2664
- disconnect: () => Promise<void>;
2665
- };
2666
- };
2667
- /**
2668
- * Consumer group ID used for reading messages.
2669
- * @default 'gravito-workers'
2670
- */
2671
- consumerGroupId?: string;
2672
- }
2673
- /**
2674
- * Kafka-backed queue driver.
2675
- *
2676
- * Uses Apache Kafka topics as queues. Designed for high-throughput streaming
2677
- * rather than traditional job queue semantics (pop/delete).
2678
- * Supports push-based consumption via `subscribe()`.
2679
- *
2680
- * @public
2681
- * @example
2682
- * ```typescript
2683
- * const driver = new KafkaDriver({ client: kafka, consumerGroupId: 'my-app' });
2684
- * ```
2685
- */
2686
- declare class KafkaDriver implements QueueDriver {
2687
- private client;
2688
- private consumerGroupId;
2689
- private producer?;
2690
- private admin?;
2691
- constructor(config: KafkaDriverConfig);
2692
- /**
2693
- * Ensure the producer is connected.
2694
- */
2695
- private ensureProducer;
2696
- /**
2697
- * Ensure the admin client is connected.
2698
- */
2699
- private ensureAdmin;
2700
- /**
2701
- * Pushes a job to a Kafka topic.
2702
- *
2703
- * @param queue - The topic name.
2704
- * @param job - The job to publish.
2705
- */
2706
- push(queue: string, job: SerializedJob): Promise<void>;
2707
- /**
2708
- * Pop is not supported for Kafka (Push-based).
2709
- *
2710
- * Kafka consumers typically stream messages. Use `subscribe()` instead.
2711
- *
2712
- * @throws {Error} Always throws as Kafka does not support polling individual messages in this manner.
2713
- */
2714
- pop(_queue: string): Promise<SerializedJob | null>;
2715
- /**
2716
- * Returns 0 as Kafka does not expose a simple "queue size".
2717
- *
2718
- * Monitoring lag requires external tools or Admin API checks not implemented here.
2719
- */
2720
- size(_queue: string): Promise<number>;
2721
- /**
2722
- * Clears a queue by deleting the topic.
2723
- *
2724
- * @param queue - The topic name.
2725
- */
2726
- clear(queue: string): Promise<void>;
2727
- /**
2728
- * Pushes multiple jobs to a Kafka topic.
2729
- *
2730
- * @param queue - The topic name.
2731
- * @param jobs - Array of jobs.
2732
- */
2733
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
2734
- /**
2735
- * Creates a new Kafka topic.
2736
- *
2737
- * @param topic - The topic name.
2738
- * @param options - Config for partitions/replication.
2739
- */
2740
- createTopic(topic: string, options?: TopicOptions): Promise<void>;
2741
- /**
2742
- * Deletes a Kafka topic.
2743
- *
2744
- * @param topic - The topic name.
2745
- */
2746
- deleteTopic(topic: string): Promise<void>;
2747
- /**
2748
- * Subscribes to a topic for streaming jobs.
2749
- *
2750
- * Starts a Kafka consumer group and processes messages as they arrive.
2751
- *
2752
- * @param queue - The topic name.
2753
- * @param callback - Function to handle the job.
2754
- */
2755
- subscribe(queue: string, callback: (job: SerializedJob) => Promise<void>): Promise<void>;
2756
- }
2757
-
2758
- /**
2759
- * Configuration options for the MemoryDriver.
2760
- */
2761
- interface MemoryDriverConfig {
2762
- /**
2763
- * The maximum number of jobs allowed in a single queue.
2764
- *
2765
- * @default Infinity
2766
- */
2767
- maxSize?: number;
2768
- }
2769
- /**
2770
- * In-memory queue driver.
2771
- *
2772
- * Stores jobs in a local JavaScript Map. Ideal for development, testing, and simple
2773
- * use cases where persistence across restarts is not required.
2774
- * It supports basic delay handling but data is volatile.
2775
- *
2776
- * @public
2777
- * @example
2778
- * ```typescript
2779
- * const driver = new MemoryDriver({ maxSize: 1000 });
2780
- * await driver.push('default', job);
2781
- * ```
2782
- */
2783
- declare class MemoryDriver implements QueueDriver {
2784
- private queues;
2785
- private maxSize;
2786
- constructor(config?: MemoryDriverConfig);
2787
- /**
2788
- * Pushes a job to the in-memory queue.
2789
- *
2790
- * @param queue - The queue name.
2791
- * @param job - The serialized job.
2792
- * @throws {Error} If the queue has reached `maxSize`.
2793
- */
2794
- push(queue: string, job: SerializedJob): Promise<void>;
2795
- /**
2796
- * Pops the next available job from the queue.
2797
- *
2798
- * Respects `delaySeconds` by checking the job's `createdAt` timestamp.
2799
- *
2800
- * @param queue - The queue name.
2801
- * @returns The job or `null`.
2802
- */
2803
- pop(queue: string): Promise<SerializedJob | null>;
2804
- /**
2805
- * Returns the number of jobs in the queue.
2806
- *
2807
- * @param queue - The queue name.
2808
- */
2809
- size(queue: string): Promise<number>;
2810
- /**
2811
- * Clears all jobs from the queue.
2812
- *
2813
- * @param queue - The queue name.
2814
- */
2815
- clear(queue: string): Promise<void>;
2816
- /**
2817
- * Moves a job to the failed (DLQ) list.
2818
- *
2819
- * In MemoryDriver, this simply pushes to a `failed:{queue}` list.
2820
- *
2821
- * @param queue - The original queue name.
2822
- * @param job - The failed job.
2823
- */
2824
- fail(queue: string, job: SerializedJob): Promise<void>;
2825
- /**
2826
- * Retrieves statistics for the queue.
2827
- *
2828
- * Calculates pending, delayed, and failed counts by iterating through the list.
2829
- *
2830
- * @param queue - The queue name.
2831
- */
2832
- stats(queue: string): Promise<QueueStats>;
2833
- /**
2834
- * Pushes multiple jobs to the queue.
2835
- *
2836
- * @param queue - The queue name.
2837
- * @param jobs - Array of jobs.
2838
- */
2839
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
2840
- /**
2841
- * Pops multiple jobs from the queue.
2842
- *
2843
- * @param queue - The queue name.
2844
- * @param count - Max jobs to pop.
2845
- */
2846
- popMany(queue: string, count: number): Promise<SerializedJob[]>;
2847
- /**
2848
- * Lists all active queues in memory.
2849
- */
2850
- getQueues(): Promise<string[]>;
2851
- }
2852
-
2853
- /**
2854
- * RabbitMQ driver configuration.
2855
- */
2856
- interface RabbitMQDriverConfig {
2857
- /**
2858
- * RabbitMQ client (amqplib) Connection or Channel.
2859
- * If a Connection is provided, the driver will create and manage a Channel.
2860
- */
2861
- client: any;
2862
- /**
2863
- * Exchange name (optional).
2864
- */
2865
- exchange?: string;
2866
- /**
2867
- * Exchange type (default: 'fanout').
2868
- */
2869
- exchangeType?: 'direct' | 'topic' | 'headers' | 'fanout' | 'match';
2870
- }
2871
- /**
2872
- * RabbitMQ (AMQP) queue driver.
2873
- *
2874
- * Uses RabbitMQ as the backend. Supports standard AMQP queues, exchanges,
2875
- * and reliable message acknowledgements.
2876
- *
2877
- * @public
2878
- * @example
2879
- * ```typescript
2880
- * import amqp from 'amqplib';
2881
- * const conn = await amqp.connect('amqp://localhost');
2882
- * const driver = new RabbitMQDriver({ client: conn });
2883
- * ```
2884
- */
2885
- declare class RabbitMQDriver implements QueueDriver {
2886
- private connection;
2887
- private channel;
2888
- private exchange?;
2889
- private exchangeType;
2890
- constructor(config: RabbitMQDriverConfig);
2891
- /**
2892
- * Ensure channel is created.
2893
- */
2894
- ensureChannel(): Promise<any>;
2895
- /**
2896
- * Get the underlying connection.
2897
- */
2898
- getRawConnection(): any;
2899
- /**
2900
- * Pushes a job to a RabbitMQ queue or exchange.
2901
- *
2902
- * @param queue - The queue name.
2903
- * @param job - The serialized job.
2904
- */
2905
- push(queue: string, job: SerializedJob): Promise<void>;
2906
- /**
2907
- * Pops a job from the queue.
2908
- *
2909
- * @param queue - The queue name.
2910
- */
2911
- pop(queue: string): Promise<SerializedJob | null>;
2912
- /**
2913
- * Pops multiple jobs.
2914
- *
2915
- * @param queue - The queue name.
2916
- * @param count - Max jobs.
2917
- */
2918
- popMany(queue: string, count: number): Promise<SerializedJob[]>;
2919
- /**
2920
- * Acknowledges a message.
2921
- *
2922
- * @param messageId - The message object (RabbitMQ requires object reference).
2923
- */
2924
- acknowledge(messageId: string): Promise<void>;
2925
- /**
2926
- * Negative acknowledge a message.
2927
- */
2928
- nack(message: any, requeue?: boolean): Promise<void>;
2929
- /**
2930
- * Reject a message.
2931
- */
2932
- reject(message: any, requeue?: boolean): Promise<void>;
2933
- /**
2934
- * Subscribes to a queue.
2935
- */
2936
- subscribe(queue: string, callback: (job: SerializedJob) => Promise<void>, options?: {
2937
- autoAck?: boolean;
2938
- prefetch?: number;
2939
- }): Promise<void>;
2940
- /**
2941
- * Returns the number of messages in the queue.
2942
- *
2943
- * @param queue - The queue name.
2944
- */
2945
- size(queue: string): Promise<number>;
2946
- /**
2947
- * Purges the queue.
2948
- *
2949
- * @param queue - The queue name.
2950
- */
2951
- clear(queue: string): Promise<void>;
2952
- }
2953
-
2954
- /**
2955
- * Interface for Redis clients (compatible with ioredis and node-redis).
2956
- */
2957
- interface RedisClient {
2958
- lpush(key: string, ...values: string[]): Promise<number>;
2959
- rpop(key: string, count?: number): Promise<string | string[] | null>;
2960
- llen(key: string): Promise<number>;
2961
- del(key: string, ...keys: string[]): Promise<number>;
2962
- lpushx?(key: string, ...values: string[]): Promise<number>;
2963
- rpoplpush?(src: string, dst: string): Promise<string | null>;
2964
- zadd?(key: string, score: number, member: string): Promise<number>;
2965
- zrange?(key: string, start: number, end: number, ...args: string[]): Promise<string[]>;
2966
- zrem?(key: string, ...members: string[]): Promise<number>;
2967
- get?(key: string): Promise<string | null>;
2968
- set?(key: string, value: string, ...args: any[]): Promise<'OK' | null>;
2969
- ltrim?(key: string, start: number, stop: number): Promise<'OK'>;
2970
- lrange?(key: string, start: number, stop: number): Promise<string[]>;
2971
- publish?(channel: string, message: string): Promise<number>;
2972
- pipeline?(): any;
2973
- defineCommand?(name: string, options: {
2974
- numberOfKeys: number;
2975
- lua: string;
2976
- }): void;
2977
- incr?(key: string): Promise<number>;
2978
- expire?(key: string, seconds: number): Promise<number>;
2979
- eval(script: string, numKeys: number, ...args: (string | number)[]): Promise<any>;
2980
- [key: string]: any;
2981
- }
2982
- /**
2983
- * Extended Redis client with custom commands.
2984
- */
2985
- interface CustomRedisClient extends RedisClient {
2986
- pushGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string, payload: string): Promise<number>;
2987
- completeGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string): Promise<number>;
2988
- popMany(queue: string, prefix: string, count: number, now: string): Promise<string[]>;
2989
- }
2990
- /**
2991
- * Extended Redis client with custom group commands (Legacy name).
2992
- */
2993
- type GroupRedisClient = CustomRedisClient;
2994
- /**
2995
- * Redis driver configuration.
2996
- */
2997
- interface RedisDriverConfig {
2998
- /**
2999
- * Redis client instance (ioredis or node-redis).
3000
- */
3001
- client: RedisClient;
3002
- /**
3003
- * Key prefix (default: `queue:`).
3004
- */
3005
- prefix?: string;
3006
- }
3007
- /**
3008
- * High-performance Redis queue driver.
3009
- *
3010
- * Implements FIFO queues using Redis Lists, reliable priority support, delayed jobs via Sorted Sets,
3011
- * and rate limiting. Uses Lua scripts for atomic operations and advanced features like
3012
- * group-based sequential processing.
3013
- *
3014
- * @public
3015
- * @example
3016
- * ```typescript
3017
- * import Redis from 'ioredis';
3018
- * const redis = new Redis();
3019
- * const driver = new RedisDriver({ client: redis });
3020
- * ```
3021
- */
3022
- declare class RedisDriver implements QueueDriver {
3023
- private prefix;
3024
- private client;
3025
- private static PUSH_SCRIPT;
3026
- private static COMPLETE_SCRIPT;
3027
- private static POP_MANY_SCRIPT;
3028
- constructor(config: RedisDriverConfig);
3029
- /**
3030
- * Get full Redis key for a queue.
3031
- */
3032
- private getKey;
3033
- /**
3034
- * Pushes a job to Redis.
3035
- *
3036
- * Handles regular jobs (LPUSH), delayed jobs (ZADD), and grouped jobs (custom Lua logic).
3037
- *
3038
- * @param queue - The queue name.
3039
- * @param job - The serialized job.
3040
- * @param options - Push options.
3041
- */
3042
- push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
3043
- /**
3044
- * Completes a job.
3045
- *
3046
- * Crucial for Group FIFO logic to unlock the next job in the group.
3047
- *
3048
- * @param queue - The queue name.
3049
- * @param job - The job to complete.
3050
- */
3051
- complete(queue: string, job: SerializedJob): Promise<void>;
3052
- /**
3053
- * Pops a job from the queue.
3054
- *
3055
- * Checks priorities in order (critical -> high -> default -> low).
3056
- * Also checks for due delayed jobs and moves them to the active list.
3057
- *
3058
- * @param queue - The queue name.
3059
- * @returns The job or `null`.
3060
- */
3061
- pop(queue: string): Promise<SerializedJob | null>;
3062
- /**
3063
- * Manual fallback for pop if Lua fails.
3064
- */
3065
- private popManualFallback;
3066
- /**
3067
- * Pops a job using blocking Redis commands (BRPOP).
3068
- *
3069
- * Efficiently waits for a job to arrive without polling.
3070
- *
3071
- * @param queues - The queues to listen to.
3072
- * @param timeout - Timeout in seconds.
3073
- */
3074
- popBlocking(queues: string | string[], timeout: number): Promise<SerializedJob | null>;
3075
- /**
3076
- * Parse Redis payload.
3077
- */
3078
- private parsePayload;
3079
- /**
3080
- * Returns the length of the queue (Redis List length).
3081
- *
3082
- * @param queue - The queue name.
3083
- */
3084
- size(queue: string): Promise<number>;
3085
- /**
3086
- * Marks a job as permanently failed by moving it to a DLQ list.
3087
- *
3088
- * @param queue - The queue name.
3089
- * @param job - The failed job.
3090
- */
3091
- fail(queue: string, job: SerializedJob): Promise<void>;
3092
- /**
3093
- * Clears the queue and its associated delayed/active sets.
3094
- *
3095
- * @param queue - The queue name.
3096
- */
3097
- clear(queue: string): Promise<void>;
3098
- /**
3099
- * Retrieves full stats for the queue using Redis Pipelining.
3100
- *
3101
- * Aggregates counts from all priority lists and the DLQ.
3102
- *
3103
- * @param queue - The queue name.
3104
- */
3105
- stats(queue: string): Promise<QueueStats>;
3106
- /**
3107
- * Pushes multiple jobs to the queue.
3108
- *
3109
- * Uses pipeline for batch efficiency. Falls back to individual pushes if complex logic (groups/priority) is involved.
3110
- *
3111
- * @param queue - The queue name.
3112
- * @param jobs - Array of jobs.
3113
- */
3114
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
3115
- /**
3116
- * Pops multiple jobs from the queue.
3117
- *
3118
- * Uses a Lua script for atomic retrieval across priorities.
3119
- *
3120
- * @param queue - The queue name.
3121
- * @param count - Max jobs to pop.
3122
- */
3123
- popMany(queue: string, count: number): Promise<SerializedJob[]>;
3124
- /**
3125
- * Reports a worker heartbeat.
3126
- *
3127
- * Stores worker metadata in a key with an expiration (TTL).
3128
- */
3129
- reportHeartbeat(workerInfo: any, prefix?: string): Promise<void>;
3130
- /**
3131
- * Publishes monitoring logs.
3132
- *
3133
- * Uses Redis Pub/Sub for real-time logs and a capped List for history.
3134
- */
3135
- publishLog(logPayload: any, prefix?: string): Promise<void>;
3136
- /**
3137
- * Checks the rate limit for a queue.
3138
- *
3139
- * Uses a simple Fixed Window counter (INCR + EXPIRE).
3140
- *
3141
- * @param queue - The queue name.
3142
- * @param config - Rate limit rules.
3143
- */
3144
- checkRateLimit(queue: string, config: {
3145
- max: number;
3146
- duration: number;
3147
- }): Promise<boolean>;
3148
- /**
3149
- * Retrieves failed jobs from the DLQ.
3150
- *
3151
- * @param queue - The queue name.
3152
- * @param start - Start index.
3153
- * @param end - End index.
3154
- */
3155
- getFailed(queue: string, start?: number, end?: number): Promise<SerializedJob[]>;
3156
- /**
3157
- * Retries failed jobs.
3158
- *
3159
- * Pops from DLQ and pushes back to the active queue (RPOPLPUSH equivalent logic).
3160
- *
3161
- * @param queue - The queue name.
3162
- * @param count - Jobs to retry.
3163
- */
3164
- retryFailed(queue: string, count?: number): Promise<number>;
3165
- /**
3166
- * Clears the Dead Letter Queue.
3167
- *
3168
- * @param queue - The queue name.
3169
- */
3170
- clearFailed(queue: string): Promise<void>;
3171
- /**
3172
- * Retrieves all discovered queue names from Redis.
3173
- */
3174
- getQueues(): Promise<string[]>;
3175
- }
3176
-
3177
- /**
3178
- * SQS driver configuration.
3179
- */
3180
- interface SQSDriverConfig {
3181
- /**
3182
- * SQS client instance (`@aws-sdk/client-sqs`).
3183
- */
3184
- client: {
3185
- send: (command: unknown) => Promise<{
3186
- MessageId?: string;
3187
- Messages?: Array<{
3188
- MessageId?: string;
3189
- ReceiptHandle?: string;
3190
- Body?: string;
3191
- }>;
3192
- }>;
3193
- };
3194
- /**
3195
- * Queue URL prefix (used to build full queue URLs).
3196
- */
3197
- queueUrlPrefix?: string;
3198
- /**
3199
- * Visibility timeout (seconds, default: 30).
3200
- */
3201
- visibilityTimeout?: number;
3202
- /**
3203
- * Long-polling duration (seconds, default: 20).
3204
- */
3205
- waitTimeSeconds?: number;
3206
- }
3207
- /**
3208
- * Amazon SQS queue driver.
3209
- *
3210
- * Wraps the AWS SDK for SQS. Supports standard and FIFO queues, long polling,
3211
- * and visibility timeouts.
3212
- *
3213
- * @public
3214
- * @example
3215
- * ```typescript
3216
- * import { SQSClient } from '@aws-sdk/client-sqs';
3217
- * const sqs = new SQSClient({ region: 'us-east-1' });
3218
- * const driver = new SQSDriver({ client: sqs });
3219
- * ```
3220
- */
3221
- declare class SQSDriver implements QueueDriver {
3222
- private client;
3223
- private queueUrlPrefix;
3224
- private visibilityTimeout;
3225
- private waitTimeSeconds;
3226
- private queueUrls;
3227
- constructor(config: SQSDriverConfig);
3228
- /**
3229
- * Resolve the full queue URL.
3230
- */
3231
- private getQueueUrl;
3232
- /**
3233
- * Pushes a job to SQS.
3234
- *
3235
- * @param queue - The queue name (or URL).
3236
- * @param job - The serialized job.
3237
- */
3238
- push(queue: string, job: SerializedJob): Promise<void>;
3239
- /**
3240
- * Pops a job from SQS (using long polling).
3241
- *
3242
- * @param queue - The queue name (or URL).
3243
- */
3244
- pop(queue: string): Promise<SerializedJob | null>;
3245
- /**
3246
- * Pops multiple jobs (up to 10).
3247
- *
3248
- * @param queue - The queue name.
3249
- * @param count - Max jobs (capped at 10 by SQS).
3250
- */
3251
- popMany(queue: string, count: number): Promise<SerializedJob[]>;
3252
- /**
3253
- * Returns the approximate number of messages in the queue.
3254
- *
3255
- * @param queue - The queue name.
3256
- */
3257
- size(queue: string): Promise<number>;
3258
- /**
3259
- * Clears the queue by continuously receiving and deleting messages.
3260
- *
3261
- * SQS does not have a "purge" command in the client data plane easily accessible here,
3262
- * so we drain the queue.
3263
- *
3264
- * @param queue - The queue name.
3265
- */
3266
- clear(queue: string): Promise<void>;
3267
- /**
3268
- * Pushes multiple jobs using SQS batch API.
3269
- *
3270
- * @param queue - The queue name.
3271
- * @param jobs - Array of jobs.
3272
- */
3273
- pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
3274
- /**
3275
- * Throws error as SQS requires ReceiptHandle, not just MessageId.
3276
- */
3277
- acknowledge(_messageId: string): Promise<void>;
3278
- /**
3279
- * Deletes a message using its ReceiptHandle (ACK).
3280
- *
3281
- * @param queue - The queue name.
3282
- * @param receiptHandle - The SQS receipt handle.
3283
- */
3284
- deleteMessage(queue: string, receiptHandle: string): Promise<void>;
3285
- }
3286
-
3287
- /**
3288
- * Configuration options for distributed locks.
3289
- *
3290
- * Defines the time-to-live (TTL), retry strategy, and automatic renewal behavior for a lock.
3291
- *
3292
- * @public
3293
- * @since 3.1.0
3294
- * @example
3295
- * ```typescript
3296
- * const options: LockOptions = {
3297
- * ttl: 60000, // Lock held for 60 seconds
3298
- * retryCount: 3, // Retry 3 times on failure
3299
- * retryDelay: 100, // Wait 100ms between retries
3300
- * refreshInterval: 20000 // Auto-renew every 20 seconds
3301
- * };
3302
- * ```
3303
- */
3304
- interface LockOptions {
3305
- /**
3306
- * Time-to-live for the lock in milliseconds.
3307
- *
3308
- * The lock will automatically expire if the holder does not release or renew it
3309
- * before this duration elapses.
3310
- */
3311
- ttl: number;
3312
- /**
3313
- * Number of retry attempts if lock acquisition fails.
3314
- *
3315
- * Set to 0 to disable retries.
3316
- */
3317
- retryCount: number;
3318
- /**
3319
- * Delay between retry attempts in milliseconds.
3320
- */
3321
- retryDelay: number;
3322
- /**
3323
- * Interval for automatic lock renewal in milliseconds.
3324
- *
3325
- * If set, the lock will automatically extend its TTL every `refreshInterval`.
3326
- * Recommended value is 1/3 of the `ttl`.
3327
- *
3328
- * @optional
3329
- */
3330
- refreshInterval?: number;
3331
- }
3332
- /**
3333
- * Distributed lock implementation based on Redis (Redlock style).
3334
- *
3335
- * Provides mutual exclusion in a distributed environment, ensuring only one node
3336
- * holds a specific lock at a time. Supports automatic renewal, retry mechanisms,
3337
- * and safe release (only the holder can release).
3338
- *
3339
- * @public
3340
- * @since 3.1.0
3341
- * @example
3342
- * ```typescript
3343
- * const lock = new DistributedLock(redisClient);
3344
- *
3345
- * const acquired = await lock.acquire('my-resource', {
3346
- * ttl: 60000,
3347
- * retryCount: 3,
3348
- * retryDelay: 100,
3349
- * refreshInterval: 20000
3350
- * });
3351
- *
3352
- * if (acquired) {
3353
- * try {
3354
- * // Perform exclusive operation
3355
- * } finally {
3356
- * await lock.release('my-resource');
3357
- * }
3358
- * }
3359
- * ```
3360
- */
3361
- declare class DistributedLock {
3362
- private client;
3363
- /**
3364
- * Unique identifier for this lock instance.
3365
- * Used to ensure only the owner can release the lock.
3366
- */
3367
- private lockId;
3368
- /**
3369
- * Timer for automatic renewal.
3370
- */
3371
- private refreshTimer;
3372
- /**
3373
- * The key of the currently held lock.
3374
- */
3375
- private currentLockKey;
3376
- /**
3377
- * Creates a DistributedLock instance.
3378
- *
3379
- * @param client - Redis client instance. Must support SET, DEL, and EVAL commands.
3380
- */
3381
- constructor(client: GroupRedisClient);
3382
- /**
3383
- * Attempts to acquire a distributed lock for the specified key.
3384
- *
3385
- * Uses Redis `SET key value EX ttl NX` for atomic acquisition.
3386
- * If the lock is held by another node, it retries according to `retryCount`.
3387
- * Upon success, if `refreshInterval` is set, automatic renewal starts.
3388
- *
3389
- * @param key - The lock key. Use a meaningful resource identifier.
3390
- * @param options - Configuration options for the lock.
3391
- * @returns `true` if the lock was acquired, `false` otherwise.
3392
- *
3393
- * @throws {Error} If the Redis client does not support the SET command.
3394
- *
3395
- * @example
3396
- * ```typescript
3397
- * const acquired = await lock.acquire('schedule:job-123', {
3398
- * ttl: 30000,
3399
- * retryCount: 5,
3400
- * retryDelay: 200
3401
- * });
3402
- *
3403
- * if (!acquired) {
3404
- * console.log('Resource is currently locked by another node');
3405
- * }
3406
- * ```
3407
- */
3408
- acquire(key: string, options: LockOptions): Promise<boolean>;
3409
- /**
3410
- * Releases the lock for the specified key.
3411
- *
3412
- * Uses a Lua script to ensure atomicity: the lock is deleted ONLY if the value matches
3413
- * this instance's `lockId`. This prevents deleting locks held by others.
3414
- * Stops the auto-renewal timer upon success.
3415
- *
3416
- * @param key - The lock key to release.
3417
- *
3418
- * @throws {Error} If the Redis client does not support the EVAL command.
3419
- *
3420
- * @example
3421
- * ```typescript
3422
- * await lock.release('schedule:job-123');
3423
- * ```
3424
- */
3425
- release(key: string): Promise<void>;
3426
- /**
3427
- * Starts the automatic renewal mechanism.
3428
- *
3429
- * Periodically extends the lock's TTL to prevent expiration during long-running tasks.
3430
- * Uses a Lua script to ensure only owned locks are renewed.
3431
- *
3432
- * @param key - The lock key.
3433
- * @param options - Lock options containing `refreshInterval`.
3434
- */
3435
- private startRefresh;
3436
- /**
3437
- * Stops the automatic renewal timer.
3438
- */
3439
- private stopRefresh;
3440
- /**
3441
- * Helper for delay.
3442
- *
3443
- * @param ms - Milliseconds to sleep.
3444
- */
3445
- private sleep;
3446
- /**
3447
- * Checks if the specified lock is currently held by this instance.
3448
- *
3449
- * @param key - The lock key.
3450
- * @returns `true` if held, `false` otherwise.
3451
- *
3452
- * @example
3453
- * ```typescript
3454
- * if (lock.isHeld('schedule:job-123')) {
3455
- * console.log('Lock is active');
3456
- * }
3457
- * ```
3458
- */
3459
- isHeld(key: string): boolean;
3460
- }
3461
-
3462
- /**
3463
- * Configuration options for the OrbitStream extension.
3464
- *
3465
- * Extends the standard `QueueConfig` with specific options for integration into
3466
- * the Gravito lifecycle, such as auto-starting workers in development environments.
3467
- *
3468
- * @public
3469
- * @example
3470
- * ```typescript
3471
- * const options: OrbitStreamOptions = {
3472
- * default: 'redis',
3473
- * autoStartWorker: true
3474
- * };
3475
- * ```
3476
- */
3477
- interface OrbitStreamOptions extends QueueConfig {
3478
- /**
3479
- * Automatically start an embedded worker process.
3480
- *
3481
- * If set to `true`, a background worker will be spawned within the main process.
3482
- * This is recommended for development or simple deployments but should be
3483
- * avoided in high-scale production to keep web servers stateless.
3484
- */
3485
- autoStartWorker?: boolean;
3486
- /**
3487
- * Configuration options for the embedded worker.
3488
- *
3489
- * Only used if `autoStartWorker` is true. Defines concurrency, polling intervals, etc.
3490
- */
3491
- workerOptions?: ConsumerOptions;
3492
- /**
3493
- * Configuration for the Stream Monitoring Dashboard API.
3494
- *
3495
- * If enabled, registers API routes for monitoring queue health and job history.
3496
- *
3497
- * @default false
3498
- */
3499
- dashboard?: boolean | {
3500
- /**
3501
- * Base path for the dashboard API routes.
3502
- * @default '/_flux'
3503
- */
3504
- path?: string;
3505
- };
3506
- }
3507
- /**
3508
- * The Queue Orbit (Plugin) for Gravito Framework.
3509
- *
3510
- * This class acts as the integration layer between the `@gravito/stream` package
3511
- * and the Gravito core system (`PlanetCore`). It registers the `QueueManager`
3512
- * service, injects it into the request context, and manages the lifecycle of
3513
- * embedded workers.
3514
- *
3515
- * @public
3516
- * @example
3517
- * ```typescript
3518
- * const stream = OrbitStream.configure({
3519
- * default: 'redis',
3520
- * connections: {
3521
- * redis: { driver: 'redis', client: redis }
3522
- * }
3523
- * });
3524
- *
3525
- * await PlanetCore.boot({ orbits: [stream] });
3526
- * ```
3527
- */
3528
- declare class OrbitStream implements GravitoOrbit {
3529
- private options;
3530
- private queueManager?;
3531
- private consumer?;
3532
- private core?;
3533
- constructor(options?: OrbitStreamOptions);
3534
- /**
3535
- * Factory method for creating and configuring an OrbitStream instance.
3536
- *
3537
- * Provides a fluent way to instantiate the orbit during application bootstrap.
3538
- *
3539
- * @param options - Configuration options.
3540
- * @returns A new OrbitStream instance.
3541
- *
3542
- * @example
3543
- * ```typescript
3544
- * const orbit = OrbitStream.configure({ default: 'memory' });
3545
- * ```
3546
- */
3547
- static configure(options: OrbitStreamOptions): OrbitStream;
3548
- /**
3549
- * Installs the Queue system into the Gravito PlanetCore.
3550
- *
3551
- * This lifecycle method:
3552
- * 1. Initializes the `QueueManager`.
3553
- * 2. Registers the `queue` service in the dependency injection container.
3554
- * 3. Sets up a global middleware to inject `QueueManager` into the request context (`c.get('queue')`).
3555
- * 4. Automatically detects and registers database connections if available in the context.
3556
- * 5. Starts the embedded worker if configured.
3557
- *
3558
- * @param core - The PlanetCore instance.
3559
- */
3560
- install(core: PlanetCore): void;
3561
- /**
3562
- * Starts the embedded worker process.
3563
- *
3564
- * Launches a `Consumer` instance to process jobs in the background.
3565
- * Throws an error if `QueueManager` is not initialized or if a worker is already running.
3566
- *
3567
- * @param options - Consumer configuration options.
3568
- * @throws {Error} If QueueManager is missing or worker is already active.
3569
- *
3570
- * @example
3571
- * ```typescript
3572
- * orbit.startWorker({ queues: ['default'] });
3573
- * ```
3574
- */
3575
- startWorker(options: ConsumerOptions): void;
3576
- /**
3577
- * Stops the embedded worker process.
3578
- *
3579
- * Gracefully shuts down the consumer, waiting for active jobs to complete.
3580
- *
3581
- * @returns A promise that resolves when the worker has stopped.
3582
- *
3583
- * @example
3584
- * ```typescript
3585
- * await orbit.stopWorker();
3586
- * ```
3587
- */
3588
- stopWorker(): Promise<void>;
3589
- /**
3590
- * Retrieves the underlying QueueManager instance.
3591
- *
3592
- * @returns The active QueueManager, or undefined if not installed.
3593
- *
3594
- * @example
3595
- * ```typescript
3596
- * const manager = orbit.getQueueManager();
3597
- * ```
3598
- */
3599
- getQueueManager(): QueueManager | undefined;
3600
- }
3601
- declare module '@gravito/core' {
3602
- interface GravitoVariables {
3603
- /** Queue manager for job processing */
3604
- queue?: QueueManager;
3605
- /** Database service (from orbit-db) */
3606
- db?: unknown;
3607
- }
3608
- }
3609
-
3610
- /**
3611
- * Buffered Persistence Wrapper.
3612
- *
3613
- * Decorates any `PersistenceAdapter` to add write buffering. Instead of writing
3614
- * to the database immediately for every event, it collects jobs and logs in memory
3615
- * and flushes them in batches. This significantly reduces database I/O for high-throughput queues.
3616
- *
3617
- * @public
3618
- * @example
3619
- * ```typescript
3620
- * const mysqlAdapter = new MySQLPersistence(db);
3621
- * const bufferedAdapter = new BufferedPersistence(mysqlAdapter, {
3622
- * maxBufferSize: 100,
3623
- * flushInterval: 500
3624
- * });
3625
- * ```
3626
- */
3627
- declare class BufferedPersistence implements PersistenceAdapter {
3628
- private adapter;
3629
- private jobBuffer;
3630
- private logBuffer;
3631
- private flushTimer;
3632
- private maxBufferSize;
3633
- private flushInterval;
3634
- constructor(adapter: PersistenceAdapter, options?: {
3635
- maxBufferSize?: number;
3636
- flushInterval?: number;
3637
- });
3638
- /**
3639
- * Buffers a job archive request.
3640
- *
3641
- * @param queue - The queue name.
3642
- * @param job - The serialized job.
3643
- * @param status - The final job status.
3644
- */
3645
- archive(queue: string, job: SerializedJob, status: 'completed' | 'failed' | 'waiting' | string): Promise<void>;
3646
- /**
3647
- * Delegates find to the underlying adapter (no buffering for reads).
3648
- */
3649
- find(queue: string, id: string): Promise<SerializedJob | null>;
3650
- /**
3651
- * Delegates list to the underlying adapter (no buffering for reads).
3652
- */
3653
- list(queue: string, options?: {
3654
- limit?: number;
3655
- offset?: number;
3656
- status?: 'completed' | 'failed' | 'waiting' | string;
3657
- jobId?: string;
3658
- startTime?: Date;
3659
- endTime?: Date;
3660
- }): Promise<SerializedJob[]>;
3661
- /**
3662
- * Archives multiple jobs directly (bypassing buffer, or flushing first).
3663
- *
3664
- * Actually, for consistency, this might just pass through.
3665
- */
3666
- archiveMany(jobs: Array<{
3667
- queue: string;
3668
- job: SerializedJob;
3669
- status: 'completed' | 'failed' | 'waiting' | string;
3670
- }>): Promise<void>;
3671
- /**
3672
- * Delegates cleanup to the underlying adapter.
3673
- */
3674
- cleanup(days: number): Promise<number>;
3675
- /**
3676
- * Flushes all buffered data to the underlying adapter.
3677
- *
3678
- * Uses `archiveMany` and `archiveLogMany` if supported by the adapter for batch efficiency.
3679
- */
3680
- flush(): Promise<void>;
3681
- /**
3682
- * Delegates count to the underlying adapter.
3683
- */
3684
- count(queue: string, options?: {
3685
- status?: 'completed' | 'failed' | 'waiting' | string;
3686
- jobId?: string;
3687
- startTime?: Date;
3688
- endTime?: Date;
3689
- }): Promise<number>;
3690
- /**
3691
- * Buffers a log message.
3692
- */
3693
- archiveLog(log: {
3694
- level: string;
3695
- message: string;
3696
- workerId: string;
3697
- queue?: string;
3698
- timestamp: Date;
3699
- }): Promise<void>;
3700
- /**
3701
- * Archives multiple logs directly.
3702
- */
3703
- archiveLogMany(logs: Array<{
3704
- level: string;
3705
- message: string;
3706
- workerId: string;
3707
- queue?: string;
3708
- timestamp: Date;
3709
- }>): Promise<void>;
3710
- /**
3711
- * Delegates listLogs to the underlying adapter.
3712
- */
3713
- listLogs(options?: {
3714
- limit?: number;
3715
- offset?: number;
3716
- level?: string;
3717
- workerId?: string;
3718
- queue?: string;
3719
- search?: string;
3720
- startTime?: Date;
3721
- endTime?: Date;
3722
- }): Promise<any[]>;
3723
- /**
3724
- * Delegates countLogs to the underlying adapter.
3725
- */
3726
- countLogs(options?: {
3727
- level?: string;
3728
- workerId?: string;
3729
- queue?: string;
3730
- search?: string;
3731
- startTime?: Date;
3732
- endTime?: Date;
3733
- }): Promise<number>;
3734
- /**
3735
- * Ensures the auto-flush timer is running.
3736
- */
3737
- private ensureFlushTimer;
3738
- }
3739
-
3740
- /**
3741
- * MySQL Persistence Adapter.
3742
- *
3743
- * Implements the `PersistenceAdapter` interface for MySQL databases.
3744
- * Stores job history and logs in relational tables for long-term retention and auditing.
3745
- *
3746
- * @public
3747
- * @example
3748
- * ```typescript
3749
- * const persistence = new MySQLPersistence(dbConnection);
3750
- * ```
3751
- */
3752
- declare class MySQLPersistence implements PersistenceAdapter {
3753
- private db;
3754
- private table;
3755
- private logsTable;
3756
- /**
3757
- * @param db - An Atlas DB instance or compatible QueryBuilder.
3758
- * @param table - The name of the table to store archived jobs.
3759
- * @param logsTable - The name of the table to store system logs.
3760
- * @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
3761
- */
3762
- constructor(db: ConnectionContract, table?: string, logsTable?: string, _options?: {
3763
- maxBufferSize?: number;
3764
- flushInterval?: number;
3765
- });
3766
- /**
3767
- * Archives a single job.
3768
- */
3769
- archive(queue: string, job: SerializedJob, status: 'completed' | 'failed' | 'waiting' | string): Promise<void>;
3770
- /**
3771
- * Archives multiple jobs in a batch.
3772
- */
3773
- archiveMany(jobs: Array<{
3774
- queue: string;
3775
- job: SerializedJob;
3776
- status: 'completed' | 'failed' | 'waiting' | string;
3777
- }>): Promise<void>;
3778
- /**
3779
- * No-op. Use BufferedPersistence if flushing is needed.
3780
- */
3781
- flush(): Promise<void>;
3782
- /**
3783
- * Finds an archived job by ID.
3784
- */
3785
- find(queue: string, id: string): Promise<SerializedJob | null>;
3786
- /**
3787
- * List jobs from the archive.
3788
- */
3789
- list(queue: string, options?: {
3790
- limit?: number;
3791
- offset?: number;
3792
- status?: 'completed' | 'failed' | 'waiting' | string;
3793
- jobId?: string;
3794
- startTime?: Date;
3795
- endTime?: Date;
3796
- }): Promise<SerializedJob[]>;
3797
- /**
3798
- * Search jobs from the archive.
3799
- *
3800
- * @param query - Search string (matches ID, payload, or error).
3801
- * @param options - Filter options.
3802
- */
3803
- search(query: string, options?: {
3804
- limit?: number;
3805
- offset?: number;
3806
- queue?: string;
3807
- }): Promise<SerializedJob[]>;
3808
- /**
3809
- * Archive a system log message.
3810
- */
3811
- archiveLog(log: {
3812
- level: string;
3813
- message: string;
3814
- workerId: string;
3815
- queue?: string;
3816
- timestamp: Date;
3817
- }): Promise<void>;
3818
- /**
3819
- * Archive multiple log messages.
3820
- */
3821
- archiveLogMany(logs: Array<{
3822
- level: string;
3823
- message: string;
3824
- workerId: string;
3825
- queue?: string;
3826
- timestamp: Date;
3827
- }>): Promise<void>;
3828
- /**
3829
- * List system logs from the archive.
3830
- */
3831
- listLogs(options?: {
3832
- limit?: number;
3833
- offset?: number;
3834
- level?: string;
3835
- workerId?: string;
3836
- queue?: string;
3837
- search?: string;
3838
- startTime?: Date;
3839
- endTime?: Date;
3840
- }): Promise<any[]>;
3841
- /**
3842
- * Count system logs in the archive.
3843
- */
3844
- countLogs(options?: {
3845
- level?: string;
3846
- workerId?: string;
3847
- queue?: string;
3848
- search?: string;
3849
- startTime?: Date;
3850
- endTime?: Date;
3851
- }): Promise<number>;
3852
- /**
3853
- * Remove old records from the archive.
3854
- */
3855
- cleanup(days: number): Promise<number>;
3856
- /**
3857
- * Count jobs in the archive.
3858
- */
3859
- count(queue: string, options?: {
3860
- status?: 'completed' | 'failed' | 'waiting' | string;
3861
- jobId?: string;
3862
- startTime?: Date;
3863
- endTime?: Date;
3864
- }): Promise<number>;
3865
- /**
3866
- * Helper to create necessary tables if they don't exist.
3867
- */
3868
- setupTable(): Promise<void>;
3869
- private setupJobsTable;
3870
- private setupLogsTable;
3871
- }
3872
-
3873
- /**
3874
- * SQLite Persistence Adapter.
3875
- *
3876
- * Archives jobs into a local SQLite database for zero-config persistence.
3877
- * Uses transactions to optimize write performance for batches.
3878
- *
3879
- * @public
3880
- * @example
3881
- * ```typescript
3882
- * const persistence = new SQLitePersistence(db);
3883
- * ```
3884
- */
3885
- declare class SQLitePersistence implements PersistenceAdapter {
3886
- private db;
3887
- private table;
3888
- private logsTable;
3889
- /**
3890
- * @param db - An Atlas DB instance (SQLite driver).
3891
- * @param table - The name of the table to store archived jobs.
3892
- * @param logsTable - The name of the table to store system logs.
3893
- * @param options - Buffering options (Deprecated: Use BufferedPersistence wrapper instead).
3894
- */
3895
- constructor(db: ConnectionContract, table?: string, logsTable?: string, _options?: {
3896
- maxBufferSize?: number;
3897
- flushInterval?: number;
3898
- });
3899
- /**
3900
- * Archives a single job.
3901
- */
3902
- archive(queue: string, job: SerializedJob, status: 'completed' | 'failed' | 'waiting' | string): Promise<void>;
3903
- /**
3904
- * Archives multiple jobs in a batch.
3905
- *
3906
- * Optimized for SQLite by wrapping chunks in transactions.
3907
- */
3908
- archiveMany(jobs: Array<{
3909
- queue: string;
3910
- job: SerializedJob;
3911
- status: 'completed' | 'failed' | 'waiting' | string;
3912
- }>): Promise<void>;
3913
- /**
3914
- * No-op. Use BufferedPersistence if flushing is needed.
3915
- */
3916
- flush(): Promise<void>;
3917
- /**
3918
- * Finds an archived job by ID.
3919
- */
3920
- find(queue: string, id: string): Promise<SerializedJob | null>;
3921
- /**
3922
- * List jobs from the archive.
3923
- */
3924
- list(queue: string, options?: {
3925
- limit?: number;
3926
- offset?: number;
3927
- status?: 'completed' | 'failed' | 'waiting' | string | string[];
3928
- jobId?: string;
3929
- startTime?: Date;
3930
- endTime?: Date;
3931
- }): Promise<SerializedJob[]>;
3932
- /**
3933
- * Search jobs from the archive.
3934
- */
3935
- search(query: string, options?: {
3936
- limit?: number;
3937
- offset?: number;
3938
- queue?: string;
3939
- }): Promise<SerializedJob[]>;
3940
- /**
3941
- * Archive a system log message (buffered).
3942
- */
3943
- archiveLog(log: {
3944
- level: string;
3945
- message: string;
3946
- workerId: string;
3947
- queue?: string;
3948
- timestamp: Date;
3949
- }): Promise<void>;
3950
- /**
3951
- * Archive multiple log messages (direct batch write).
3952
- */
3953
- archiveLogMany(logs: Array<{
3954
- level: string;
3955
- message: string;
3956
- workerId: string;
3957
- queue?: string;
3958
- timestamp: Date;
3959
- }>): Promise<void>;
3960
- /**
3961
- * List system logs from the archive.
3962
- */
3963
- listLogs(options?: {
3964
- limit?: number;
3965
- offset?: number;
3966
- level?: string;
3967
- workerId?: string;
3968
- queue?: string;
3969
- search?: string;
3970
- startTime?: Date;
3971
- endTime?: Date;
3972
- }): Promise<any[]>;
3973
- /**
3974
- * Count system logs in the archive.
3975
- */
3976
- countLogs(options?: {
3977
- level?: string;
3978
- workerId?: string;
3979
- queue?: string;
3980
- search?: string;
3981
- startTime?: Date;
3982
- endTime?: Date;
3983
- }): Promise<number>;
3984
- /**
3985
- * Remove old records from the archive.
3986
- */
3987
- cleanup(days: number): Promise<number>;
3988
- /**
3989
- * Count jobs in the archive.
3990
- */
3991
- count(queue: string, options?: {
3992
- status?: 'completed' | 'failed' | 'waiting' | string | string[];
3993
- jobId?: string;
3994
- startTime?: Date;
3995
- endTime?: Date;
3996
- }): Promise<number>;
3997
- /**
3998
- * Setup table for SQLite.
3999
- */
4000
- setupTable(): Promise<void>;
4001
- private setupJobsTable;
4002
- private setupLogsTable;
4003
- }
4004
-
4005
- /**
4006
- * Retry strategy for events.
4007
- *
4008
- * - 'bull': Use Bull Queue's built-in retry mechanism
4009
- * - 'core': Use EventPriorityQueue's retry logic (not implemented in Stream backend)
4010
- * - 'hybrid': Bull retries + Core DLQ fallback (future phase)
4011
- */
4012
- type RetryStrategy = 'bull' | 'core' | 'hybrid';
4013
- /**
4014
- * Configuration for StreamEventBackend.
4015
- */
4016
- interface StreamEventBackendConfig {
4017
- /**
4018
- * Retry strategy to use.
4019
- * @default 'bull'
4020
- */
4021
- retryStrategy?: RetryStrategy;
4022
- /**
4023
- * Whether to integrate with CircuitBreaker.
4024
- * @default false
4025
- */
4026
- circuitBreakerIntegration?: boolean;
4027
- /**
4028
- * Optional DLQ (Dead Letter Queue) handler for failed events.
4029
- */
4030
- dlqHandler?: {
4031
- handle(event: EventTask, error: Error, attempt: number): Promise<void>;
4032
- };
4033
- /**
4034
- * CircuitBreaker getter (injected from core)
4035
- */
4036
- getCircuitBreaker?: (hook: string) => any;
4037
- }
4038
- /**
4039
- * Event backend implementation using Gravito Stream (Bull Queue).
4040
- *
4041
- * Provides persistent, distributed event processing with:
4042
- * - Bull Queue persistence (Redis-backed)
4043
- * - Configurable retry strategies
4044
- * - Optional CircuitBreaker integration
4045
- * - DLQ support for failed events
4046
- *
4047
- * @example
4048
- * ```typescript
4049
- * const queueManager = new QueueManager({
4050
- * default: 'bullmq',
4051
- * connections: {
4052
- * bullmq: {
4053
- * driver: 'bullmq',
4054
- * queue: new Queue('gravito-events', { connection: redis })
4055
- * }
4056
- * }
4057
- * })
4058
- *
4059
- * const backend = new StreamEventBackend(queueManager, {
4060
- * retryStrategy: 'bull',
4061
- * circuitBreakerIntegration: true
4062
- * })
4063
- * ```
4064
- */
4065
- declare class StreamEventBackend implements EventBackend {
4066
- private queueManager;
4067
- private config;
4068
- constructor(queueManager: QueueManager, config?: StreamEventBackendConfig);
4069
- /**
4070
- * Build Job Push Options from EventOptions.
4071
- *
4072
- * Maps EventOptions to Bull Queue JobPushOptions with retry strategy applied.
4073
- */
4074
- private buildJobOptions;
4075
- /**
4076
- * Enqueue an event task to the stream queue.
4077
- *
4078
- * Applies retry strategy and CircuitBreaker checks based on configuration.
4079
- * Supports DLQ routing for failed events.
4080
- */
4081
- enqueue(task: EventTask): Promise<void>;
4082
- /**
4083
- * Apply retry strategy to the job based on configuration.
4084
- */
4085
- private applyRetryStrategy;
4086
- /**
4087
- * Handle job failure and route to DLQ if configured.
4088
- *
4089
- * Called when a job exhausts all retry attempts.
4090
- */
4091
- handleJobFailure(task: EventTask, error: Error, attempt: number): Promise<void>;
4092
- /**
4093
- * Record a job failure for CircuitBreaker state management.
4094
- *
4095
- * Called when a job fails, regardless of retry status.
4096
- */
4097
- recordJobFailure(task: EventTask, error: Error): void;
4098
- /**
4099
- * Record a job success for CircuitBreaker state management.
4100
- *
4101
- * Called when a job completes successfully.
4102
- */
4103
- recordJobSuccess(task: EventTask): void;
4104
- /**
4105
- * Get the retry strategy configuration.
4106
- */
4107
- getRetryStrategy(): RetryStrategy;
4108
- /**
4109
- * Check if CircuitBreaker integration is enabled.
4110
- */
4111
- isCircuitBreakerEnabled(): boolean;
4112
- /**
4113
- * Get the DLQ handler, if configured.
4114
- */
4115
- getDLQHandler(): StreamEventBackendConfig['dlqHandler'] | undefined;
4116
- }
4117
-
4118
- /**
4119
- * SystemEventJob - Internal job for processing Gravito async hooks.
4120
- *
4121
- * @internal
4122
- */
4123
- declare class SystemEventJob extends Job {
4124
- readonly hook: string;
4125
- readonly args: unknown;
4126
- readonly options: Record<string, any>;
4127
- /**
4128
- * Optional failure callback for DLQ handling.
4129
- */
4130
- private onFailedCallback?;
4131
- constructor(hook: string, args: unknown, options?: Record<string, any>);
4132
- /**
4133
- * Set failure callback for DLQ handling.
4134
- *
4135
- * @param callback - Called when job fails permanently
4136
- * @returns Self for chaining
4137
- */
4138
- onFailed(callback: (error: Error, attempt: number) => Promise<void>): this;
4139
- /**
4140
- * Execute the hook listeners in the worker process.
4141
- */
4142
- handle(): Promise<void>;
4143
- /**
4144
- * Called when job fails permanently after all retries.
4145
- *
4146
- * This method is invoked by the worker when job exhausts all retry attempts.
4147
- */
4148
- failed(error: Error, attempt?: number): Promise<void>;
4149
- }
4150
-
4151
- /**
4152
- * Class Name Serializer (Laravel-style).
4153
- *
4154
- * Serializes jobs by storing their class name along with their properties.
4155
- * During deserialization, it looks up the registered class constructor and creates a new instance,
4156
- * populating it with the stored properties.
4157
- *
4158
- * This is the recommended serializer for most use cases as it preserves the behavior (methods)
4159
- * of the job class.
4160
- *
4161
- * @public
4162
- * @example
4163
- * ```typescript
4164
- * const serializer = new ClassNameSerializer();
4165
- * serializer.register(SendEmailJob);
4166
- *
4167
- * const serialized = serializer.serialize(new SendEmailJob('foo@bar.com'));
4168
- * const job = serializer.deserialize(serialized); // instanceof SendEmailJob
4169
- * ```
4170
- */
4171
- declare class ClassNameSerializer implements JobSerializer {
4172
- /**
4173
- * Registry of job classes, mapped by class name.
4174
- */
4175
- private jobClasses;
4176
- /**
4177
- * Registers a Job class for serialization.
4178
- *
4179
- * @param jobClass - The job class constructor.
4180
- */
4181
- register(jobClass: new (...args: unknown[]) => Job): void;
4182
- /**
4183
- * Registers multiple Job classes at once.
4184
- *
4185
- * @param jobClasses - An array of job class constructors.
4186
- */
4187
- registerMany(jobClasses: Array<new (...args: unknown[]) => Job>): void;
4188
- /**
4189
- * Serializes a Job instance.
4190
- *
4191
- * Captures the class name and all enumerable properties.
4192
- *
4193
- * @param job - The job to serialize.
4194
- */
4195
- serialize(job: Job): SerializedJob;
4196
- /**
4197
- * Deserializes a Job instance.
4198
- *
4199
- * Instantiates the class matching `className` and assigns properties.
4200
- *
4201
- * @param serialized - The serialized job.
4202
- * @throws {Error} If the job class is not registered.
4203
- */
4204
- deserialize(serialized: SerializedJob): Job;
4205
- }
4206
-
4207
- /**
4208
- * JSON Serializer.
4209
- *
4210
- * Serializes jobs to standard JSON. This is the simplest serializer but has limitations:
4211
- * it cannot restore class instances (methods are lost) or handle complex types like Maps/Sets.
4212
- * Deserialized jobs will be plain objects that must be manually handled or cast.
4213
- *
4214
- * @public
4215
- * @example
4216
- * ```typescript
4217
- * const serializer = new JsonSerializer();
4218
- * ```
4219
- */
4220
- declare class JsonSerializer implements JobSerializer {
4221
- /**
4222
- * Serializes a job to a JSON object.
4223
- */
4224
- serialize(job: Job): SerializedJob;
4225
- /**
4226
- * Deserializes a JSON object into a basic Job-like object.
4227
- *
4228
- * Note: The result is NOT an instance of the original Job class.
4229
- */
4230
- deserialize(serialized: SerializedJob): Job;
4231
- }
4232
-
4233
- /**
4234
- * Worker Pool Implementation.
4235
- *
4236
- * Manages a pool of Sandboxed Workers to provide concurrency control,
4237
- * worker reuse, load balancing, and health monitoring.
4238
- *
4239
- * @public
4240
- */
4241
-
4242
- /**
4243
- * Configuration options for the Worker Pool.
4244
- */
4245
- interface WorkerPoolConfig extends SandboxedWorkerConfig {
4246
- /**
4247
- * The maximum number of workers allowed in the pool.
4248
- *
4249
- * @default 4
4250
- */
4251
- poolSize?: number;
4252
- /**
4253
- * The minimum number of workers to keep alive.
4254
- *
4255
- * The pool will pre-warm and maintain at least this many ready workers.
4256
- * @default 0
4257
- */
4258
- minWorkers?: number;
4259
- /**
4260
- * Interval for performing health checks in milliseconds.
4261
- *
4262
- * Periodically scans for and removes terminated or unhealthy workers.
4263
- * @default 30000 (30 seconds)
4264
- */
4265
- healthCheckInterval?: number;
4266
- }
4267
- /**
4268
- * Runtime statistics for the Worker Pool.
4269
- */
4270
- interface WorkerPoolStats {
4271
- /** Total number of workers (ready + busy). */
4272
- total: number;
4273
- /** Number of idle workers ready for new jobs. */
4274
- ready: number;
4275
- /** Number of workers currently executing jobs. */
4276
- busy: number;
4277
- /** Number of workers in terminated state awaiting cleanup. */
4278
- terminated: number;
4279
- /** Number of jobs waiting in the queue. */
4280
- pending: number;
4281
- /** Total number of successfully completed jobs. */
4282
- completed: number;
4283
- /** Total number of failed jobs. */
4284
- failed: number;
4285
- }
4286
- /**
4287
- * Worker Pool.
4288
- *
4289
- * Orchestrates multiple `SandboxedWorker` instances to execute jobs concurrently.
4290
- *
4291
- * Key features:
4292
- * - **Concurrency Control**: Limits the number of simultaneous job executions (`poolSize`).
4293
- * - **Queueing**: Queues jobs when all workers are busy.
4294
- * - **Lifecycle Management**: Automatically creates, reuses, and terminates workers.
4295
- * - **Health Monitoring**: Periodically cleans up dead workers and maintains `minWorkers`.
4296
- *
4297
- * @example
4298
- * ```typescript
4299
- * const pool = new WorkerPool({
4300
- * poolSize: 8,
4301
- * minWorkers: 2,
4302
- * maxExecutionTime: 30000
4303
- * });
4304
- *
4305
- * await pool.execute(job);
4306
- * await pool.shutdown();
4307
- * ```
4308
- */
4309
- declare class WorkerPool {
4310
- private workers;
4311
- private config;
4312
- private queue;
4313
- private healthCheckTimer;
4314
- private stats;
4315
- /**
4316
- * Creates a WorkerPool instance.
4317
- *
4318
- * @param config - Configuration options for the pool.
4319
- */
4320
- constructor(config?: WorkerPoolConfig);
4321
- /**
4322
- * Pre-warms the pool by creating the minimum number of workers.
4323
- */
4324
- private warmUp;
4325
- /**
4326
- * Creates a new SandboxedWorker and adds it to the pool.
4327
- *
4328
- * @returns The newly created worker.
4329
- */
4330
- private createWorker;
4331
- /**
4332
- * Retrieves an available worker from the pool.
4333
- *
4334
- * Priorities:
4335
- * 1. Reuse an existing ready worker.
4336
- * 2. Create a new worker if the pool is not full.
4337
- * 3. Return `null` if the pool is saturated.
4338
- *
4339
- * @returns An available worker or `null`.
4340
- */
4341
- private getAvailableWorker;
4342
- /**
4343
- * Executes a job using the worker pool.
4344
- *
4345
- * If a worker is available, the job starts immediately.
4346
- * Otherwise, it is added to the pending queue.
4347
- *
4348
- * @param job - The serialized job data.
4349
- * @throws {Error} If execution fails.
4350
- */
4351
- execute(job: SerializedJob): Promise<void>;
4352
- /**
4353
- * Processes the next job in the queue if a worker is available.
4354
- */
4355
- private processQueue;
4356
- /**
4357
- * Starts the periodic health check.
4358
- */
4359
- private startHealthCheck;
4360
- /**
4361
- * Performs a health check on the pool.
4362
- *
4363
- * Removes terminated workers and ensures `minWorkers` are available.
4364
- */
4365
- private performHealthCheck;
4366
- /**
4367
- * Gets the current statistics of the worker pool.
4368
- *
4369
- * @returns Snapshot of pool statistics.
4370
- */
4371
- getStats(): WorkerPoolStats;
4372
- /**
4373
- * Shuts down the worker pool.
4374
- *
4375
- * Terminates all workers and rejects any pending jobs.
4376
- */
4377
- shutdown(): Promise<void>;
4378
- /**
4379
- * Waits for all active and pending jobs to complete.
4380
- *
4381
- * @param timeout - Maximum wait time in milliseconds. 0 for infinite.
4382
- * @throws {Error} If the timeout is reached.
4383
- */
4384
- waitForCompletion(timeout?: number): Promise<void>;
4385
- }
4386
-
4387
- export { BatchConsumer, type BatchConsumerOptions, BufferedPersistence, BullMQDriver, type BullMQDriverConfig, ClassNameSerializer, Consumer, type ConsumerOptions, DatabaseDriver, type DatabaseDriverConfig, DistributedLock, GrpcDriver, type GrpcDriverConfig, Job, type JobSerializer, JsonSerializer, KafkaDriver, type KafkaDriverConfig, type LockOptions, MemoryDriver, MySQLPersistence, OrbitStream, type OrbitStreamOptions, type PersistenceAdapter, type QueueConfig, type QueueConnectionConfig, type QueueDriver, QueueManager, type Queueable, RabbitMQDriver, type RabbitMQDriverConfig, RedisDriver, type RedisDriverConfig, type RetryStrategy, SQLitePersistence, SQSDriver, type SQSDriverConfig, SandboxedWorker, type SandboxedWorkerConfig, type ScheduledJobConfig, Scheduler, type SchedulerOptions, type SerializedJob, StreamEventBackend, type StreamEventBackendConfig, SystemEventJob, type TopicOptions, Worker, type WorkerOptions, WorkerPool, type WorkerPoolConfig, type WorkerPoolStats };
18
+ * // Push a Job
19
+ * await queue.push(new SendEmail())
20
+ * ```
21
+ */
22
+ export { BatchConsumer, type BatchConsumerOptions } from './BatchConsumer';
23
+ export type { ConsumerOptions } from './Consumer';
24
+ export { Consumer } from './Consumer';
25
+ export type { ConsumerStrategy } from './consumer/ConsumerStrategy';
26
+ export { PollingStrategy } from './consumer/PollingStrategy';
27
+ export { ReactiveStrategy } from './consumer/ReactiveStrategy';
28
+ export { decodeBinaryJobFrame, encodeBinaryJobFrame, Flags as BinaryJobFrameFlags, isGravitoJobFrame, MAGIC as BINARY_JOB_FRAME_MAGIC, VERSION as BINARY_JOB_FRAME_VERSION, } from './drivers/BinaryJobFrame';
29
+ export type { BullMQDriverConfig } from './drivers/BullMQDriver';
30
+ export { BullMQDriver } from './drivers/BullMQDriver';
31
+ export type { DatabaseDriverConfig } from './drivers/DatabaseDriver';
32
+ export { DatabaseDriver } from './drivers/DatabaseDriver';
33
+ export type { GrpcDriverConfig } from './drivers/GrpcDriver';
34
+ export { GrpcDriver } from './drivers/GrpcDriver';
35
+ export type { KafkaDriverConfig } from './drivers/KafkaDriver';
36
+ export { KafkaDriver } from './drivers/KafkaDriver';
37
+ export { MemoryDriver } from './drivers/MemoryDriver';
38
+ export type { QueueDriver } from './drivers/QueueDriver';
39
+ export type { RabbitMQDriverConfig } from './drivers/RabbitMQDriver';
40
+ export { RabbitMQDriver } from './drivers/RabbitMQDriver';
41
+ export type { RedisClient, RedisDriverConfig } from './drivers/RedisDriver';
42
+ export { RedisDriver } from './drivers/RedisDriver';
43
+ export type { SQSDriverConfig } from './drivers/SQSDriver';
44
+ export { SQSDriver } from './drivers/SQSDriver';
45
+ export { Job } from './Job';
46
+ export type { LockOptions } from './locks/DistributedLock';
47
+ export { DistributedLock } from './locks/DistributedLock';
48
+ export type { OrbitStreamOptions } from './OrbitStream';
49
+ export { OrbitStream } from './OrbitStream';
50
+ export { BufferedPersistence } from './persistence/BufferedPersistence';
51
+ export { BunBufferedPersistence, type BunBufferedPersistenceOptions, createBufferedPersistence, } from './persistence/BunBufferedPersistence';
52
+ export { MySQLPersistence } from './persistence/MySQLPersistence';
53
+ export { SQLitePersistence } from './persistence/SQLitePersistence';
54
+ export type { Queueable } from './Queueable';
55
+ export { QueueManager } from './QueueManager';
56
+ export type { ScheduledJobConfig, SchedulerOptions } from './Scheduler';
57
+ export { Scheduler } from './Scheduler';
58
+ export { type RetryStrategy, StreamEventBackend, type StreamEventBackendConfig, } from './StreamEventBackend';
59
+ export { SystemEventJob } from './SystemEventJob';
60
+ export { BinarySerializer } from './serializers/BinarySerializer';
61
+ export { CborNativeSerializer } from './serializers/CborNativeSerializer';
62
+ export { ClassNameSerializer } from './serializers/ClassNameSerializer';
63
+ export type { JobSerializer } from './serializers/JobSerializer';
64
+ export { JsonlSerializer } from './serializers/JsonlSerializer';
65
+ export { JsonSerializer } from './serializers/JsonSerializer';
66
+ export type { PersistenceAdapter, QueueConfig, QueueConnectionConfig, SerializedJob, TopicOptions, } from './types';
67
+ export type { WorkerOptions } from './Worker';
68
+ export { Worker } from './Worker';
69
+ export { SandboxedWorker, type SandboxedWorkerConfig, WorkerPool, type WorkerPoolConfig, type WorkerPoolStats, } from './workers';