@gravito/stream 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +127 -285
  2. package/README.zh-TW.md +146 -13
  3. package/dist/BatchConsumer.d.ts +81 -0
  4. package/dist/Consumer.d.ts +215 -0
  5. package/dist/DashboardProvider.d.ts +20 -0
  6. package/dist/Job.d.ts +183 -0
  7. package/dist/OrbitStream.d.ts +151 -0
  8. package/dist/QueueManager.d.ts +319 -0
  9. package/dist/Queueable.d.ts +91 -0
  10. package/dist/Scheduler.d.ts +214 -0
  11. package/dist/StreamEventBackend.d.ts +114 -0
  12. package/dist/SystemEventJob.d.ts +33 -0
  13. package/dist/Worker.d.ts +139 -0
  14. package/dist/benchmarks/PerformanceReporter.d.ts +99 -0
  15. package/dist/consumer/ConcurrencyGate.d.ts +55 -0
  16. package/dist/consumer/ConsumerStrategy.d.ts +41 -0
  17. package/dist/consumer/GroupSequencer.d.ts +57 -0
  18. package/dist/consumer/HeartbeatManager.d.ts +65 -0
  19. package/dist/consumer/JobExecutor.d.ts +61 -0
  20. package/dist/consumer/JobSourceGenerator.d.ts +31 -0
  21. package/dist/consumer/PollingStrategy.d.ts +42 -0
  22. package/dist/consumer/ReactiveStrategy.d.ts +41 -0
  23. package/dist/consumer/StreamingConsumer.d.ts +88 -0
  24. package/dist/consumer/index.d.ts +13 -0
  25. package/dist/consumer/types.d.ts +102 -0
  26. package/dist/drivers/BinaryJobFrame.d.ts +78 -0
  27. package/dist/drivers/BullMQDriver.d.ts +186 -0
  28. package/dist/drivers/DatabaseDriver.d.ts +131 -0
  29. package/dist/drivers/GrpcDriver.d.ts +16 -0
  30. package/dist/drivers/KafkaDriver.d.ts +148 -0
  31. package/dist/drivers/MemoryDriver.d.ts +108 -0
  32. package/dist/drivers/QueueDriver.d.ts +250 -0
  33. package/dist/drivers/RabbitMQDriver.d.ts +102 -0
  34. package/dist/drivers/RedisDriver.d.ts +294 -0
  35. package/dist/drivers/SQSDriver.d.ts +111 -0
  36. package/dist/drivers/kafka/BackpressureController.d.ts +60 -0
  37. package/dist/drivers/kafka/BatchProcessor.d.ts +50 -0
  38. package/dist/drivers/kafka/ConsumerLifecycleManager.d.ts +80 -0
  39. package/dist/drivers/kafka/ErrorCategorizer.d.ts +39 -0
  40. package/dist/drivers/kafka/ErrorRecoveryManager.d.ts +100 -0
  41. package/dist/drivers/kafka/HeartbeatManager.d.ts +57 -0
  42. package/dist/drivers/kafka/KafkaDriver.d.ts +138 -0
  43. package/dist/drivers/kafka/KafkaMetrics.d.ts +88 -0
  44. package/dist/drivers/kafka/KafkaNotifier.d.ts +54 -0
  45. package/dist/drivers/kafka/MessageBuffer.d.ts +71 -0
  46. package/dist/drivers/kafka/OffsetTracker.d.ts +63 -0
  47. package/dist/drivers/kafka/PerformanceMonitor.d.ts +88 -0
  48. package/dist/drivers/kafka/RateLimiter.d.ts +52 -0
  49. package/dist/drivers/kafka/RebalanceHandler.d.ts +104 -0
  50. package/dist/drivers/kafka/RingBuffer.d.ts +63 -0
  51. package/dist/drivers/kafka/index.d.ts +22 -0
  52. package/dist/drivers/kafka/types.d.ts +553 -0
  53. package/dist/drivers/prepareJobForTransport.d.ts +10 -0
  54. package/dist/index.cjs +6274 -3777
  55. package/dist/index.cjs.map +71 -0
  56. package/dist/index.d.ts +60 -2233
  57. package/dist/index.js +6955 -4446
  58. package/dist/index.js.map +71 -0
  59. package/dist/locks/DistributedLock.d.ts +175 -0
  60. package/dist/persistence/BufferedPersistence.d.ts +130 -0
  61. package/dist/persistence/BunBufferedPersistence.d.ts +173 -0
  62. package/dist/persistence/MySQLPersistence.d.ts +134 -0
  63. package/dist/persistence/SQLitePersistence.d.ts +133 -0
  64. package/dist/serializers/BinarySerializer.d.ts +42 -0
  65. package/dist/serializers/CachedSerializer.d.ts +38 -0
  66. package/dist/serializers/CborNativeSerializer.d.ts +56 -0
  67. package/dist/serializers/ClassNameSerializer.d.ts +58 -0
  68. package/dist/serializers/JobSerializer.d.ts +33 -0
  69. package/dist/serializers/JsonSerializer.d.ts +28 -0
  70. package/dist/serializers/JsonlSerializer.d.ts +90 -0
  71. package/dist/serializers/MessagePackSerializer.d.ts +29 -0
  72. package/dist/types.d.ts +653 -0
  73. package/dist/workers/BinaryWorkerProtocol.d.ts +77 -0
  74. package/dist/workers/BunWorker.d.ts +179 -0
  75. package/dist/workers/SandboxedWorker.d.ts +132 -0
  76. package/dist/workers/WorkerFactory.d.ts +128 -0
  77. package/dist/workers/WorkerPool.d.ts +186 -0
  78. package/dist/workers/bun-job-executor.d.ts +14 -0
  79. package/dist/workers/index.d.ts +13 -0
  80. package/dist/workers/job-executor.d.ts +9 -0
  81. package/package.json +13 -6
  82. package/proto/queue.proto +101 -0
  83. package/dist/index.d.cts +0 -2242
@@ -0,0 +1,42 @@
1
+ import type { Job } from '../Job';
2
+ import type { QueueManager } from '../QueueManager';
3
+ import type { ConsumerStrategy } from './ConsumerStrategy';
4
+ /**
5
+ * PollingStrategy implements pull-based job consumption.
6
+ *
7
+ * Continuously polls queues at adaptive intervals, backing off when empty
8
+ * and resetting when jobs are found.
9
+ *
10
+ * Characteristics:
11
+ * - Simple and reliable
12
+ * - Predictable polling intervals
13
+ * - Higher latency and resource usage vs reactive
14
+ * - No external dependencies (works with any driver)
15
+ *
16
+ * @public
17
+ */
18
+ export declare class PollingStrategy implements ConsumerStrategy {
19
+ private queueManager;
20
+ private queues;
21
+ private connectionName;
22
+ private options;
23
+ private running;
24
+ constructor(queueManager: QueueManager, queues: string[], connectionName: string, options: {
25
+ minPollInterval: number;
26
+ maxPollInterval: number;
27
+ backoffMultiplier: number;
28
+ batchSize: number;
29
+ useBlocking: boolean;
30
+ blockingTimeout: number;
31
+ concurrency: number;
32
+ stats: {
33
+ active: number;
34
+ };
35
+ debug: boolean;
36
+ log: (message: string, data?: unknown) => void;
37
+ });
38
+ start(): Promise<void>;
39
+ stop(): Promise<void>;
40
+ isRunning(): boolean;
41
+ fetchJobs(): Promise<Job[]>;
42
+ }
@@ -0,0 +1,41 @@
1
+ import type { Job } from '../Job';
2
+ import type { QueueManager } from '../QueueManager';
3
+ import type { ConsumerStrategy } from './ConsumerStrategy';
4
+ /**
5
+ * ReactiveStrategy implements push-based job consumption.
6
+ *
7
+ * Listens for queue notifications and pulls jobs reactively when they arrive.
8
+ * Implements drain-loop pattern with optional polling fallback.
9
+ *
10
+ * Characteristics:
11
+ * - Low latency (immediate job pickup)
12
+ * - Efficient resource usage (no continuous polling)
13
+ * - Requires driver support for onNotify()
14
+ * - Includes polling fallback to prevent starvation
15
+ *
16
+ * @public
17
+ */
18
+ export declare class ReactiveStrategy implements ConsumerStrategy {
19
+ private queueManager;
20
+ private queues;
21
+ private connectionName;
22
+ private options;
23
+ private running;
24
+ private lastNotificationTime;
25
+ private fallbackPollingTimer;
26
+ constructor(queueManager: QueueManager, queues: string[], connectionName: string, options: {
27
+ concurrency: number;
28
+ batchSize: number;
29
+ stats: {
30
+ active: number;
31
+ };
32
+ reactivePollingFallback: number;
33
+ debug: boolean;
34
+ log: (message: string, data?: unknown) => void;
35
+ });
36
+ start(): Promise<void>;
37
+ stop(): Promise<void>;
38
+ isRunning(): boolean;
39
+ fetchJobs(): Promise<Job[]>;
40
+ private startFallbackPolling;
41
+ }
@@ -0,0 +1,88 @@
1
+ import { EventEmitter } from 'node:events';
2
+ import type { ConsumerOptions } from '../Consumer';
3
+ import type { QueueManager } from '../QueueManager';
4
+ import type { ConsumerStats } from './types';
5
+ /**
6
+ * StreamingConsumer 是 Consumer 管線的核心實作。
7
+ *
8
+ * 組合以下元件,提供高效能、可測試的 job 消費管線:
9
+ * - JobSourceGenerator:抓取 job 的 Async Generator
10
+ * - ConcurrencyGate:Promise-based Semaphore,取代 busy-wait
11
+ * - GroupSequencer:確保相同 groupId 的 job 依序執行
12
+ * - JobExecutor:完整的 job 執行生命週期
13
+ * - HeartbeatManager:定時心跳與監控日誌
14
+ *
15
+ * 主管線採用 `for await...of` 模式,
16
+ * 完全消除原始 Consumer 中的 `setTimeout(50)` busy-wait。
17
+ *
18
+ * @example
19
+ * ```typescript
20
+ * const streaming = new StreamingConsumer(queueManager, options)
21
+ * streaming.on('job:processed', (payload) => console.log('Done:', payload.job.id))
22
+ *
23
+ * await streaming.start()
24
+ * ```
25
+ */
26
+ export declare class StreamingConsumer extends EventEmitter {
27
+ private readonly queueManager;
28
+ private readonly options;
29
+ private running;
30
+ private stopRequested;
31
+ private readonly workerId;
32
+ private readonly stats;
33
+ private gate;
34
+ private sequencer;
35
+ private heartbeat;
36
+ private abortController;
37
+ constructor(queueManager: QueueManager, options: ConsumerOptions);
38
+ /**
39
+ * 啟動主消費管線。
40
+ *
41
+ * 初始化所有元件,然後進入 `for await...of` 管線迴圈:
42
+ * 1. 從 jobSourceGenerator 取得 FetchResult
43
+ * 2. 對每個 job,acquire ConcurrencyGate
44
+ * 3. 透過 GroupSequencer 確保 group 序列
45
+ * 4. 由 JobExecutor 執行 job
46
+ * 5. release ConcurrencyGate
47
+ *
48
+ * @throws {Error} 如果 consumer 已在執行中
49
+ */
50
+ start(): Promise<void>;
51
+ /**
52
+ * 請求優雅停止。
53
+ *
54
+ * 設定 stopRequested 旗標,並中止 generator,
55
+ * 等待目前執行中的 job 完成後停止。
56
+ *
57
+ * @returns 等待完全停止的 Promise
58
+ */
59
+ requestStop(): Promise<void>;
60
+ /**
61
+ * 檢查是否仍有 job 在執行中。
62
+ */
63
+ isActive(): boolean;
64
+ /**
65
+ * 取得目前的統計資料快照。
66
+ */
67
+ getStats(): ConsumerStats;
68
+ /**
69
+ * 重置統計計數器。
70
+ */
71
+ resetStats(): void;
72
+ /**
73
+ * Consumer 是否正在執行中。
74
+ */
75
+ isRunning(): boolean;
76
+ /**
77
+ * 等待所有正在執行的 job 完成。
78
+ */
79
+ private waitForAllJobsToComplete;
80
+ /**
81
+ * 清理所有資源。
82
+ */
83
+ private cleanup;
84
+ /**
85
+ * 輸出 debug 日誌(僅在 debug=true 時)。
86
+ */
87
+ private log;
88
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Consumer 管線模組 - 重新導出所有公開類型與類別。
3
+ *
4
+ * 供測試及進階使用者直接引用各個子元件。
5
+ */
6
+ export { ConcurrencyGate } from './ConcurrencyGate';
7
+ export { GroupSequencer } from './GroupSequencer';
8
+ export type { HeartbeatManagerOptions } from './HeartbeatManager';
9
+ export { HeartbeatManager } from './HeartbeatManager';
10
+ export { JobExecutor } from './JobExecutor';
11
+ export { jobSourceGenerator } from './JobSourceGenerator';
12
+ export { StreamingConsumer } from './StreamingConsumer';
13
+ export type { ConsumerStats, ExecutionResult, ExecutorOptions, FetchResult, JobSourceOptions, StopSignal, } from './types';
@@ -0,0 +1,102 @@
1
+ import type { Job } from '../Job';
2
+ /**
3
+ * 從佇列抓取 job 的結果。
4
+ *
5
+ * JobSourceGenerator 每次 yield 一個 FetchResult,
6
+ * 包含本輪抓取的所有 job,以及是否使用阻塞式等待的資訊。
7
+ */
8
+ export interface FetchResult {
9
+ /** 本次抓取到的 job 列表(可能為空) */
10
+ jobs: Job[];
11
+ /**
12
+ * 是否使用了阻塞式 pop(如 BLPOP)。
13
+ * 若為 true,代表已在 driver 層等待,不需額外 backoff。
14
+ */
15
+ didBlock: boolean;
16
+ }
17
+ /**
18
+ * JobSourceGenerator 的選項設定。
19
+ */
20
+ export interface JobSourceOptions {
21
+ /** 要消費的佇列名稱列表 */
22
+ queues: string[];
23
+ /** 連線名稱,不指定時使用預設連線 */
24
+ connection?: string;
25
+ /** 每次嘗試抓取的 job 數量,大於 1 時使用批次抓取 */
26
+ batchSize: number;
27
+ /** 是否使用阻塞式 pop */
28
+ useBlocking: boolean;
29
+ /** 阻塞式 pop 的逾時秒數 */
30
+ blockingTimeout: number;
31
+ /** 佇列為空時是否持續存活 */
32
+ keepAlive: boolean;
33
+ /** 最小輪詢間隔(毫秒) */
34
+ minPollInterval: number;
35
+ /** 最大輪詢間隔(毫秒) */
36
+ maxPollInterval: number;
37
+ /** 自適應退避乘數 */
38
+ backoffMultiplier: number;
39
+ /** Rate limit 設定,key 為佇列名稱 */
40
+ rateLimits?: Record<string, {
41
+ max: number;
42
+ duration: number;
43
+ }>;
44
+ /** 並發容量 getter,讓 generator 知道可以抓多少 job */
45
+ getCapacity: () => number;
46
+ }
47
+ /**
48
+ * JobExecutor 執行選項。
49
+ */
50
+ export interface ExecutorOptions {
51
+ /** debug 模式,會輸出詳細 log */
52
+ debug: boolean;
53
+ /** Worker ID,用於 log 識別 */
54
+ workerId: string;
55
+ /** Worker 選項(maxAttempts 等) */
56
+ workerOptions?: {
57
+ maxAttempts?: number;
58
+ };
59
+ /** 監控設定 */
60
+ monitor?: boolean | {
61
+ interval?: number;
62
+ extraInfo?: Record<string, unknown>;
63
+ prefix?: string;
64
+ };
65
+ /** 最大請求數量,達到後自動停止 */
66
+ maxRequests?: number;
67
+ /** 外部事件回呼 */
68
+ onEvent?: (event: string, payload: unknown) => void;
69
+ }
70
+ /**
71
+ * JobExecutor 執行一個 job 後的結果。
72
+ */
73
+ export interface ExecutionResult {
74
+ /** job 是否成功完成 */
75
+ success: boolean;
76
+ /** 執行耗時(毫秒) */
77
+ duration: number;
78
+ /** 是否因達到 maxRequests 而需要停止 consumer */
79
+ shouldStop: boolean;
80
+ /** 如果失敗,包含錯誤資訊 */
81
+ error?: Error;
82
+ }
83
+ /**
84
+ * Consumer 執行統計資訊。
85
+ */
86
+ export interface ConsumerStats {
87
+ /** 已成功處理的 job 數量 */
88
+ processed: number;
89
+ /** 失敗(含重試失敗)的 job 數量 */
90
+ failed: number;
91
+ /** 重試的 job 數量 */
92
+ retried: number;
93
+ /** 目前正在執行的 job 數量 */
94
+ active: number;
95
+ }
96
+ /**
97
+ * 停止訊號,用於 AbortController 整合。
98
+ */
99
+ export interface StopSignal {
100
+ /** 是否已發出停止訊號 */
101
+ readonly aborted: boolean;
102
+ }
@@ -0,0 +1,78 @@
1
+ import type { SerializedJob } from '../types';
2
+ /**
3
+ * BinaryJobFrame - Gravito Job 的二進制幀格式
4
+ *
5
+ * 幀結構(Frame Format):
6
+ * Magic(2B) + Version(1B) + Flags(1B) + MetaLen(2B Big-Endian) + Metadata(CBOR) + Payload
7
+ *
8
+ * - Magic:固定為 [0x47, 0x4A]("GJ" = Gravito Job)
9
+ * - Version:目前為 0x01
10
+ * - Flags:位元旗標,目前定義:
11
+ * Bit 0 (HAS_CLASS_NAME):是否包含 className 欄位
12
+ * Bit 1 (HAS_GROUP_ID):是否包含 groupId 欄位
13
+ * Bit 2 (HAS_DELAY):是否包含 delaySeconds 欄位
14
+ * Bit 3 (HAS_PRIORITY):是否包含 priority 欄位
15
+ * Bit 4 (HAS_ERROR):是否包含 error / failedAt 欄位
16
+ * - MetaLen:Metadata CBOR 資料的長度(2 bytes Big-Endian,最大 65535 bytes)
17
+ * - Metadata:以 CBOR 或 JSON 編碼的 metadata 物件(id, createdAt, attempts 等)
18
+ * - Payload:原始 Uint8Array 資料(job.data)
19
+ *
20
+ * @module BinaryJobFrame
21
+ */
22
+ /** Magic bytes,標識 Gravito Job 二進制幀 */
23
+ export declare const MAGIC: readonly [71, 74];
24
+ /** 幀格式版本號 */
25
+ export declare const VERSION = 1;
26
+ /** 幀 header 的固定大小(Magic 2B + Version 1B + Flags 1B + MetaLen 2B = 6 bytes) */
27
+ export declare const HEADER_SIZE = 6;
28
+ /** Flags 位元定義 */
29
+ export declare const Flags: {
30
+ /** 包含 className 欄位 */
31
+ readonly HAS_CLASS_NAME: 1;
32
+ /** 包含 groupId 欄位 */
33
+ readonly HAS_GROUP_ID: 2;
34
+ /** 包含 delaySeconds 欄位 */
35
+ readonly HAS_DELAY: 4;
36
+ /** 包含 priority 欄位 */
37
+ readonly HAS_PRIORITY: 8;
38
+ /** 包含 error / failedAt 欄位 */
39
+ readonly HAS_ERROR: 16;
40
+ };
41
+ /**
42
+ * Metadata 短鍵映射(節省序列化空間)
43
+ * 完整鍵名 → 短鍵名
44
+ */
45
+ export declare const META_KEY_MAP: Record<string, string>;
46
+ /**
47
+ * 短鍵反向映射(反序列化用)
48
+ * 短鍵名 → 完整鍵名
49
+ */
50
+ export declare const META_KEY_REVERSE_MAP: Record<string, string>;
51
+ /**
52
+ * 判斷給定的資料是否為 Gravito Job 二進制幀
53
+ *
54
+ * 透過 Magic byte 嗅探識別:檢查前 2 bytes 是否為 [0x47, 0x4A]
55
+ *
56
+ * @param data - 要檢查的資料(Uint8Array 或 Buffer)
57
+ * @returns 如果是 Gravito Job 幀則回傳 true
58
+ */
59
+ export declare function isGravitoJobFrame(data: Uint8Array | Buffer): boolean;
60
+ /**
61
+ * 將 SerializedJob 編碼為二進制幀格式
62
+ *
63
+ * 僅適用於 type === 'binary' 且 data 為 Uint8Array 的 Job。
64
+ *
65
+ * @param job - 要編碼的 SerializedJob(必須是 binary type)
66
+ * @returns 編碼後的 Uint8Array
67
+ * @throws {TypeError} 如果 job.data 不是 Uint8Array
68
+ * @throws {Error} 如果 metadata 長度超過 65535 bytes
69
+ */
70
+ export declare function encodeBinaryJobFrame(job: SerializedJob): Uint8Array;
71
+ /**
72
+ * 將二進制幀解碼為 SerializedJob
73
+ *
74
+ * @param data - 要解碼的二進制幀(Uint8Array 或 Buffer)
75
+ * @returns 解碼後的 SerializedJob,data 為 Uint8Array
76
+ * @throws {Error} 如果資料格式無效
77
+ */
78
+ export declare function decodeBinaryJobFrame(data: Uint8Array | Buffer): SerializedJob;
@@ -0,0 +1,186 @@
1
+ import type { JobPushOptions, QueueStats, SerializedJob, TopicOptions } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ /**
4
+ * Bull Queue client interface (compatible with bullmq package).
5
+ */
6
+ export interface BullQueueClient {
7
+ add(name: string, data: any, options?: any): Promise<any>;
8
+ getJob(id: string): Promise<any | null>;
9
+ count(): Promise<number>;
10
+ process(handler: (job: any) => Promise<void>): void;
11
+ on(event: string, handler: (...args: any[]) => void): void;
12
+ off(event: string, handler: (...args: any[]) => void): void;
13
+ pause(): Promise<void>;
14
+ resume(): Promise<void>;
15
+ clean(grace: number, limit?: number, type?: string): Promise<number>;
16
+ close(): Promise<void>;
17
+ getJobCounts(types?: string[]): Promise<Record<string, number>>;
18
+ getDelayedCount(): Promise<number>;
19
+ getFailedCount(): Promise<number>;
20
+ getActiveCount(): Promise<number>;
21
+ [key: string]: any;
22
+ }
23
+ export interface BullWorkerClient {
24
+ [key: string]: any;
25
+ }
26
+ /**
27
+ * Bull Queue driver configuration.
28
+ */
29
+ export interface BullMQDriverConfig {
30
+ /**
31
+ * Bull Queue instance (from bullmq package).
32
+ */
33
+ queue: BullQueueClient;
34
+ /**
35
+ * Optional Bull Worker instance for processing jobs.
36
+ */
37
+ worker?: BullWorkerClient;
38
+ /**
39
+ * Connection options (host, port, etc. for Redis).
40
+ */
41
+ connection?: {
42
+ host?: string;
43
+ port?: number;
44
+ password?: string;
45
+ db?: number;
46
+ [key: string]: any;
47
+ };
48
+ /**
49
+ * Default number of concurrent workers (default: 1).
50
+ */
51
+ concurrency?: number;
52
+ /**
53
+ * Key prefix for namespacing queues (default: 'gravito:').
54
+ */
55
+ prefix?: string;
56
+ /**
57
+ * Enable debug logging.
58
+ */
59
+ debug?: boolean;
60
+ }
61
+ /**
62
+ * Bull Queue driver implementation.
63
+ *
64
+ * Provides high-performance, persistent job queuing using Bull Queue (backed by Redis).
65
+ * Supports priority, delays, retries, and group-based FIFO processing.
66
+ *
67
+ * @public
68
+ * @example
69
+ * ```typescript
70
+ * import { Queue } from 'bullmq'
71
+ * import Redis from 'ioredis'
72
+ *
73
+ * const redis = new Redis()
74
+ * const queue = new Queue('gravito-events', { connection: redis })
75
+ * const driver = new BullMQDriver({ queue })
76
+ * ```
77
+ */
78
+ export declare class BullMQDriver implements QueueDriver {
79
+ private queue;
80
+ private prefix;
81
+ private debug;
82
+ private queueMap;
83
+ constructor(config: BullMQDriverConfig);
84
+ /**
85
+ * Get or create a queue for the given queue name.
86
+ */
87
+ private getQueue;
88
+ /**
89
+ * Build Job Options from JobPushOptions.
90
+ */
91
+ private buildJobOptions;
92
+ /**
93
+ * Create Bull job data from SerializedJob.
94
+ */
95
+ private createBullJobData;
96
+ /**
97
+ * Pushes a job to Bull Queue.
98
+ */
99
+ push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
100
+ /**
101
+ * Pops a job from Bull Queue.
102
+ * Note: Bull Queue typically uses Workers, not manual pop.
103
+ * This is a fallback implementation.
104
+ */
105
+ pop(queue: string): Promise<SerializedJob | null>;
106
+ /**
107
+ * Returns the size of the queue.
108
+ */
109
+ size(queue: string): Promise<number>;
110
+ /**
111
+ * Clears the queue.
112
+ */
113
+ clear(queue: string): Promise<void>;
114
+ /**
115
+ * Marks a job as failed (moves to failed list).
116
+ */
117
+ fail(queue: string, job: SerializedJob): Promise<void>;
118
+ /**
119
+ * Returns detailed statistics for the queue.
120
+ */
121
+ stats(queue: string): Promise<QueueStats>;
122
+ /**
123
+ * Retrieves failed jobs from the Dead Letter Queue.
124
+ */
125
+ getFailed(queue: string, _start?: number, _end?: number): Promise<SerializedJob[]>;
126
+ /**
127
+ * Retries failed jobs.
128
+ */
129
+ retryFailed(queue: string, _count?: number): Promise<number>;
130
+ /**
131
+ * Clears the Dead Letter Queue.
132
+ */
133
+ clearFailed(queue: string): Promise<void>;
134
+ /**
135
+ * Creates a new queue/topic.
136
+ */
137
+ createTopic(_topic: string, _options?: TopicOptions): Promise<void>;
138
+ /**
139
+ * Deletes a queue/topic.
140
+ */
141
+ deleteTopic(topic: string): Promise<void>;
142
+ /**
143
+ * Pushes multiple jobs in batch.
144
+ */
145
+ pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
146
+ /**
147
+ * Pops multiple jobs in batch.
148
+ */
149
+ popMany(_queue: string, _count: number): Promise<SerializedJob[]>;
150
+ /**
151
+ * Reports worker heartbeat.
152
+ */
153
+ reportHeartbeat(workerInfo: {
154
+ id: string;
155
+ status: string;
156
+ hostname: string;
157
+ pid: number;
158
+ uptime: number;
159
+ last_ping: string;
160
+ queues: string[];
161
+ metrics?: Record<string, any>;
162
+ [key: string]: any;
163
+ }, _prefix?: string): Promise<void>;
164
+ /**
165
+ * Publishes a log message.
166
+ */
167
+ publishLog(logPayload: {
168
+ level: string;
169
+ message: string;
170
+ workerId: string;
171
+ jobId?: string;
172
+ timestamp: string;
173
+ [key: string]: any;
174
+ }, _prefix?: string): Promise<void>;
175
+ /**
176
+ * Checks rate limit for a queue.
177
+ */
178
+ checkRateLimit(_queue: string, _config: {
179
+ max: number;
180
+ duration: number;
181
+ }): Promise<boolean>;
182
+ /**
183
+ * Retrieves all queue names.
184
+ */
185
+ getQueues(): Promise<string[]>;
186
+ }
@@ -0,0 +1,131 @@
1
+ import type { QueueStats, SerializedJob } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ /**
4
+ * Generic database service interface.
5
+ *
6
+ * Adapts any SQL database client (e.g., pg, mysql2, sqlite3) for use with the DatabaseDriver.
7
+ * Users must provide an implementation of this interface that wraps their specific DB library.
8
+ */
9
+ export interface DatabaseService {
10
+ /**
11
+ * Execute a raw SQL query.
12
+ *
13
+ * @param sql - The SQL query string with placeholders (e.g., $1, ?).
14
+ * @param bindings - The values to bind to the placeholders.
15
+ * @returns The query result (rows or metadata).
16
+ */
17
+ execute<T = unknown>(sql: string, bindings?: unknown[]): Promise<T[] | T>;
18
+ /**
19
+ * Execute multiple queries within a single transaction.
20
+ *
21
+ * @param callback - A function that receives a transaction-scoped service instance.
22
+ * @returns The result of the callback.
23
+ */
24
+ transaction<T>(callback: (tx: DatabaseService) => Promise<T>): Promise<T>;
25
+ }
26
+ /**
27
+ * Configuration options for the DatabaseDriver.
28
+ */
29
+ export interface DatabaseDriverConfig {
30
+ /**
31
+ * The name of the table used to store jobs.
32
+ * @default 'jobs'
33
+ */
34
+ table?: string;
35
+ /**
36
+ * The database service adapter instance.
37
+ */
38
+ dbService?: DatabaseService;
39
+ }
40
+ /**
41
+ * Database-backed queue driver.
42
+ *
43
+ * Persists jobs in a SQL database table. Supports delayed jobs, reservation (locking),
44
+ * and reliable delivery. Compatible with PostgreSQL (SKIP LOCKED), MySQL, and SQLite.
45
+ *
46
+ * @public
47
+ * @example
48
+ * ```typescript
49
+ * const driver = new DatabaseDriver({
50
+ * dbService: myDbAdapter,
51
+ * table: 'queue_jobs'
52
+ * });
53
+ * ```
54
+ */
55
+ export declare class DatabaseDriver implements QueueDriver {
56
+ private tableName;
57
+ private dbService;
58
+ constructor(config: DatabaseDriverConfig);
59
+ /**
60
+ * Pushes a job to the database queue.
61
+ *
62
+ * Inserts a new row into the jobs table.
63
+ *
64
+ * @param queue - The queue name.
65
+ * @param job - The serialized job.
66
+ */
67
+ push(queue: string, job: SerializedJob): Promise<void>;
68
+ /**
69
+ * Pops the next available job from the queue.
70
+ *
71
+ * Uses transactional locking (SELECT ... FOR UPDATE SKIP LOCKED if supported) to ensure
72
+ * atomic reservation of jobs by workers.
73
+ *
74
+ * @param queue - The queue name.
75
+ * @returns The job or `null`.
76
+ */
77
+ pop(queue: string): Promise<SerializedJob | null>;
78
+ /**
79
+ * Pops multiple jobs from the queue in a single transaction.
80
+ *
81
+ * @param queue - The queue name.
82
+ * @param count - Max jobs to pop.
83
+ */
84
+ popMany(queue: string, count: number): Promise<SerializedJob[]>;
85
+ /**
86
+ * Retrieves queue statistics by querying the table.
87
+ *
88
+ * @param queue - The queue name.
89
+ */
90
+ stats(queue: string): Promise<QueueStats>;
91
+ /**
92
+ * Returns the count of pending jobs.
93
+ *
94
+ * @param queue - The queue name.
95
+ */
96
+ size(queue: string): Promise<number>;
97
+ /**
98
+ * Clears the queue by deleting all rows for the queue.
99
+ *
100
+ * @param queue - The queue name.
101
+ */
102
+ clear(queue: string): Promise<void>;
103
+ /**
104
+ * Pops a job using a polling loop (Blocking simulation).
105
+ *
106
+ * @param queue - The queue name.
107
+ * @param timeout - Timeout in seconds.
108
+ */
109
+ popBlocking(queue: string, timeout: number): Promise<SerializedJob | null>;
110
+ /**
111
+ * Pushes multiple jobs using a transaction.
112
+ *
113
+ * @param queue - The queue name.
114
+ * @param jobs - Array of jobs.
115
+ */
116
+ pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
117
+ /**
118
+ * Marks a job as permanently failed by moving it to the DLQ (separate logical queue in DB).
119
+ *
120
+ * @param queue - The queue name.
121
+ * @param job - The failed job.
122
+ */
123
+ fail(queue: string, job: SerializedJob): Promise<void>;
124
+ /**
125
+ * Deletes a job row from the database (completion).
126
+ *
127
+ * @param _queue - The queue name (unused).
128
+ * @param job - The job to complete.
129
+ */
130
+ complete(_queue: string, job: SerializedJob): Promise<void>;
131
+ }
@@ -0,0 +1,16 @@
1
+ import type { GrpcDriverConfig, JobPushOptions, QueueStats, SerializedJob } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ export type { GrpcDriverConfig } from '../types';
4
+ export declare class GrpcDriver implements QueueDriver {
5
+ private client;
6
+ constructor(config: GrpcDriverConfig);
7
+ private getCredentials;
8
+ push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
9
+ pop(queue: string): Promise<SerializedJob | null>;
10
+ size(queue: string): Promise<number>;
11
+ clear(queue: string): Promise<void>;
12
+ acknowledge(messageId: string): Promise<void>;
13
+ stats(queue: string): Promise<QueueStats>;
14
+ private toProtoJob;
15
+ private fromProtoJob;
16
+ }