@gravito/stream 2.0.2 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +27 -1
  2. package/dist/BatchConsumer.d.ts +81 -0
  3. package/dist/Consumer.d.ts +215 -0
  4. package/dist/DashboardProvider.d.ts +29 -0
  5. package/dist/Job.d.ts +183 -0
  6. package/dist/OrbitStream.d.ts +151 -0
  7. package/dist/QueueManager.d.ts +321 -0
  8. package/dist/Queueable.d.ts +91 -0
  9. package/dist/Scheduler.d.ts +215 -0
  10. package/dist/StreamEventBackend.d.ts +120 -0
  11. package/dist/SystemEventJob.d.ts +41 -0
  12. package/dist/Worker.d.ts +139 -0
  13. package/dist/benchmarks/PerformanceReporter.d.ts +99 -0
  14. package/dist/consumer/ConcurrencyGate.d.ts +55 -0
  15. package/dist/consumer/ConsumerStrategy.d.ts +41 -0
  16. package/dist/consumer/GroupSequencer.d.ts +57 -0
  17. package/dist/consumer/HeartbeatManager.d.ts +65 -0
  18. package/dist/consumer/JobExecutor.d.ts +61 -0
  19. package/dist/consumer/JobSourceGenerator.d.ts +31 -0
  20. package/dist/consumer/PollingStrategy.d.ts +42 -0
  21. package/dist/consumer/ReactiveStrategy.d.ts +41 -0
  22. package/dist/consumer/StreamingConsumer.d.ts +90 -0
  23. package/dist/consumer/index.d.ts +13 -0
  24. package/dist/consumer/types.d.ts +102 -0
  25. package/dist/drivers/BinaryJobFrame.d.ts +78 -0
  26. package/dist/drivers/BullMQDriver.d.ts +237 -0
  27. package/dist/drivers/DatabaseDriver.d.ts +131 -0
  28. package/dist/drivers/GrpcDriver.d.ts +16 -0
  29. package/dist/drivers/KafkaDriver.d.ts +161 -0
  30. package/dist/drivers/MemoryDriver.d.ts +119 -0
  31. package/dist/drivers/QueueDriver.d.ts +250 -0
  32. package/dist/drivers/RabbitMQDriver.d.ts +140 -0
  33. package/dist/drivers/RedisDriver.d.ts +328 -0
  34. package/dist/drivers/SQSDriver.d.ts +114 -0
  35. package/dist/drivers/kafka/BackpressureController.d.ts +60 -0
  36. package/dist/drivers/kafka/BatchProcessor.d.ts +50 -0
  37. package/dist/drivers/kafka/ConsumerLifecycleManager.d.ts +80 -0
  38. package/dist/drivers/kafka/ErrorCategorizer.d.ts +39 -0
  39. package/dist/drivers/kafka/ErrorRecoveryManager.d.ts +100 -0
  40. package/dist/drivers/kafka/HeartbeatManager.d.ts +57 -0
  41. package/dist/drivers/kafka/KafkaDriver.d.ts +138 -0
  42. package/dist/drivers/kafka/KafkaMetrics.d.ts +88 -0
  43. package/dist/drivers/kafka/KafkaNotifier.d.ts +70 -0
  44. package/dist/drivers/kafka/MessageBuffer.d.ts +71 -0
  45. package/dist/drivers/kafka/OffsetTracker.d.ts +65 -0
  46. package/dist/drivers/kafka/PerformanceMonitor.d.ts +88 -0
  47. package/dist/drivers/kafka/RateLimiter.d.ts +52 -0
  48. package/dist/drivers/kafka/RebalanceHandler.d.ts +104 -0
  49. package/dist/drivers/kafka/RingBuffer.d.ts +63 -0
  50. package/dist/drivers/kafka/index.d.ts +22 -0
  51. package/dist/drivers/kafka/types.d.ts +553 -0
  52. package/dist/drivers/prepareJobForTransport.d.ts +10 -0
  53. package/dist/index.cjs +72 -7826
  54. package/dist/index.cjs.map +9 -0
  55. package/dist/index.d.ts +60 -4378
  56. package/dist/index.js +39 -7797
  57. package/dist/index.js.map +9 -0
  58. package/dist/locks/DistributedLock.d.ts +175 -0
  59. package/dist/persistence/BufferedPersistence.d.ts +130 -0
  60. package/dist/persistence/BunBufferedPersistence.d.ts +173 -0
  61. package/dist/persistence/MySQLPersistence.d.ts +134 -0
  62. package/dist/persistence/SQLitePersistence.d.ts +133 -0
  63. package/dist/serializers/BinarySerializer.d.ts +42 -0
  64. package/dist/serializers/CachedSerializer.d.ts +42 -0
  65. package/dist/serializers/CborNativeSerializer.d.ts +56 -0
  66. package/dist/serializers/ClassNameSerializer.d.ts +58 -0
  67. package/dist/serializers/JobSerializer.d.ts +33 -0
  68. package/dist/serializers/JsonSerializer.d.ts +28 -0
  69. package/dist/serializers/JsonlSerializer.d.ts +90 -0
  70. package/dist/serializers/MessagePackSerializer.d.ts +29 -0
  71. package/dist/types.d.ts +672 -0
  72. package/dist/workers/BinaryWorkerProtocol.d.ts +77 -0
  73. package/dist/workers/BunWorker.d.ts +179 -0
  74. package/dist/workers/SandboxedWorker.d.ts +132 -0
  75. package/dist/workers/WorkerFactory.d.ts +128 -0
  76. package/dist/workers/WorkerPool.d.ts +186 -0
  77. package/dist/workers/bun-job-executor.d.ts +14 -0
  78. package/dist/workers/index.d.ts +13 -0
  79. package/dist/workers/job-executor.d.ts +9 -0
  80. package/package.json +6 -4
  81. package/dist/index.d.cts +0 -4387
@@ -0,0 +1,140 @@
1
+ import type { SerializedJob } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ interface RabbitMessage {
4
+ content: Buffer;
5
+ }
6
+ interface RabbitChannel {
7
+ assertExchange(exchange: string, type: string, options: {
8
+ durable: boolean;
9
+ }): Promise<unknown>;
10
+ assertQueue(queue: string, options: {
11
+ durable: boolean;
12
+ }): Promise<unknown>;
13
+ bindQueue(queue: string, exchange: string, routingKey: string): Promise<unknown>;
14
+ publish(exchange: string, routingKey: string, content: Buffer, options: {
15
+ persistent: boolean;
16
+ }): boolean;
17
+ sendToQueue(queue: string, content: Buffer, options: {
18
+ persistent: boolean;
19
+ }): boolean;
20
+ get(queue: string, options: {
21
+ noAck: boolean;
22
+ }): Promise<RabbitMessage | false | null>;
23
+ ack(message: RabbitMessage): void;
24
+ nack(message: RabbitMessage, allUpTo: boolean, requeue: boolean): void;
25
+ reject(message: RabbitMessage, requeue: boolean): void;
26
+ consume(queue: string, onMessage: (message: RabbitMessage | null) => Promise<void>, options: {
27
+ noAck: boolean;
28
+ }): Promise<unknown>;
29
+ prefetch(count: number): Promise<unknown>;
30
+ checkQueue(queue: string): Promise<{
31
+ messageCount: number;
32
+ }>;
33
+ purgeQueue(queue: string): Promise<unknown>;
34
+ }
35
+ interface RabbitConnection {
36
+ createChannel(): Promise<RabbitChannel>;
37
+ }
38
+ type RabbitClient = RabbitConnection | RabbitChannel;
39
+ /**
40
+ * RabbitMQ driver configuration.
41
+ */
42
+ export interface RabbitMQDriverConfig {
43
+ /**
44
+ * RabbitMQ client (amqplib) Connection or Channel.
45
+ * If a Connection is provided, the driver will create and manage a Channel.
46
+ */
47
+ client: RabbitClient;
48
+ /**
49
+ * Exchange name (optional).
50
+ */
51
+ exchange?: string;
52
+ /**
53
+ * Exchange type (default: 'fanout').
54
+ */
55
+ exchangeType?: 'direct' | 'topic' | 'headers' | 'fanout' | 'match';
56
+ }
57
+ /**
58
+ * RabbitMQ (AMQP) queue driver.
59
+ *
60
+ * Uses RabbitMQ as the backend. Supports standard AMQP queues, exchanges,
61
+ * and reliable message acknowledgements.
62
+ *
63
+ * @public
64
+ * @example
65
+ * ```typescript
66
+ * import amqp from 'amqplib';
67
+ * const conn = await amqp.connect('amqp://localhost');
68
+ * const driver = new RabbitMQDriver({ client: conn });
69
+ * ```
70
+ */
71
+ export declare class RabbitMQDriver implements QueueDriver {
72
+ private connection;
73
+ private channel?;
74
+ private exchange?;
75
+ private exchangeType;
76
+ constructor(config: RabbitMQDriverConfig);
77
+ private isRabbitConnection;
78
+ /**
79
+ * Ensure channel is created.
80
+ */
81
+ ensureChannel(): Promise<RabbitChannel>;
82
+ /**
83
+ * Get the underlying connection.
84
+ */
85
+ getRawConnection(): RabbitClient;
86
+ /**
87
+ * Pushes a job to a RabbitMQ queue or exchange.
88
+ *
89
+ * @param queue - The queue name.
90
+ * @param job - The serialized job.
91
+ */
92
+ push(queue: string, job: SerializedJob): Promise<void>;
93
+ /**
94
+ * Pops a job from the queue.
95
+ *
96
+ * @param queue - The queue name.
97
+ */
98
+ pop(queue: string): Promise<SerializedJob | null>;
99
+ /**
100
+ * Pops multiple jobs.
101
+ *
102
+ * @param queue - The queue name.
103
+ * @param count - Max jobs.
104
+ */
105
+ popMany(queue: string, count: number): Promise<SerializedJob[]>;
106
+ /**
107
+ * Acknowledges a message.
108
+ *
109
+ * @param messageId - The message object (RabbitMQ requires object reference).
110
+ */
111
+ acknowledge(messageId: string): Promise<void>;
112
+ /**
113
+ * Negative acknowledge a message.
114
+ */
115
+ nack(message: RabbitMessage, requeue?: boolean): Promise<void>;
116
+ /**
117
+ * Reject a message.
118
+ */
119
+ reject(message: RabbitMessage, requeue?: boolean): Promise<void>;
120
+ /**
121
+ * Subscribes to a queue.
122
+ */
123
+ subscribe(queue: string, callback: (job: SerializedJob) => Promise<void>, options?: {
124
+ autoAck?: boolean;
125
+ prefetch?: number;
126
+ }): Promise<void>;
127
+ /**
128
+ * Returns the number of messages in the queue.
129
+ *
130
+ * @param queue - The queue name.
131
+ */
132
+ size(queue: string): Promise<number>;
133
+ /**
134
+ * Purges the queue.
135
+ *
136
+ * @param queue - The queue name.
137
+ */
138
+ clear(queue: string): Promise<void>;
139
+ }
140
+ export {};
@@ -0,0 +1,328 @@
1
+ import type { JobPushOptions, QueueStats, SerializedJob } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ /**
4
+ * Interface for Redis clients (compatible with ioredis and node-redis).
5
+ */
6
+ export interface RedisClient {
7
+ lpush(key: string, ...values: string[]): Promise<number>;
8
+ rpop(key: string, count?: number): Promise<string | string[] | null>;
9
+ brpop?(...args: Array<string | number>): Promise<[string, string] | null>;
10
+ llen(key: string): Promise<number>;
11
+ del(key: string, ...keys: string[]): Promise<number>;
12
+ lpushx?(key: string, ...values: string[]): Promise<number>;
13
+ rpoplpush?(src: string, dst: string): Promise<string | null>;
14
+ zadd?(key: string, score: number, member: string): Promise<number>;
15
+ zcard?(key: string): Promise<number>;
16
+ zrange?(key: string, start: number, end: number, ...args: string[]): Promise<string[]>;
17
+ zrangebyscore?(key: string, min: string | number, max: string | number): Promise<string[]>;
18
+ zrem?(key: string, ...members: string[]): Promise<number>;
19
+ get?(key: string): Promise<string | null>;
20
+ hgetall?(key: string): Promise<Record<string, string>>;
21
+ set?(key: string, value: string, ...args: Array<string | number>): Promise<'OK' | null>;
22
+ ltrim?(key: string, start: number, stop: number): Promise<'OK'>;
23
+ lrange?(key: string, start: number, stop: number): Promise<string[]>;
24
+ publish?(channel: string, message: string): Promise<number>;
25
+ pipeline?(): RedisPipeline;
26
+ defineCommand?(name: string, options: {
27
+ numberOfKeys: number;
28
+ lua: string;
29
+ }): void;
30
+ incr?(key: string): Promise<number>;
31
+ expire?(key: string, seconds: number): Promise<number>;
32
+ eval(script: string, numKeys: number, ...args: (string | number)[]): Promise<unknown>;
33
+ subscribe?(...channels: string[]): Promise<unknown>;
34
+ on?(event: 'message', handler: (channel: string, message: string) => void | Promise<void>): unknown;
35
+ on?(event: string, handler: (...args: unknown[]) => void): unknown;
36
+ /**
37
+ * Binary-capable RPOP(ioredis Buffer mode)
38
+ * 回傳 Buffer 而非 string,支援 binary frame 的零拷貝讀取
39
+ */
40
+ rpopBuffer?(key: string, count?: number): Promise<Buffer | Buffer[] | null>;
41
+ /**
42
+ * Binary-capable BRPOP(ioredis Buffer mode)
43
+ * 支援 blocking pop 的 binary frame 讀取
44
+ */
45
+ brpopBuffer?(...args: Array<string | number>): Promise<[Buffer, Buffer] | null>;
46
+ /**
47
+ * Binary-capable LRANGE(ioredis Buffer mode)
48
+ * 用於從 DLQ 讀取 binary frame
49
+ */
50
+ lrangeBuffer?(key: string, start: number, stop: number): Promise<Buffer[]>;
51
+ /**
52
+ * Binary-capable LPUSH(接受 Buffer)
53
+ * 用於將 binary frame 寫入 Redis list
54
+ */
55
+ lpushBuffer?(key: string, ...values: Buffer[]): Promise<number>;
56
+ [key: string]: unknown;
57
+ }
58
+ interface PipelineReply<T = unknown> {
59
+ 0: Error | null;
60
+ 1: T;
61
+ }
62
+ interface RedisPipeline {
63
+ llen(key: string): RedisPipeline;
64
+ zcard(key: string): RedisPipeline;
65
+ lpush(key: string, ...values: string[]): RedisPipeline;
66
+ ltrim(key: string, start: number, stop: number): RedisPipeline;
67
+ rpop(key: string): RedisPipeline;
68
+ del(key: string, ...keys: string[]): RedisPipeline;
69
+ hset(key: string, values: Record<string, string | number | boolean>): RedisPipeline;
70
+ zadd(key: string, score: number, member: string): RedisPipeline;
71
+ zrem(key: string, ...members: string[]): RedisPipeline;
72
+ pushGroupJob?(waitList: string, activeSet: string, pendingList: string, groupId: string, payload: string): RedisPipeline;
73
+ lpushBuffer?(key: string, ...values: Buffer[]): RedisPipeline;
74
+ exec(): Promise<PipelineReply[] | null>;
75
+ }
76
+ interface WorkerHeartbeatPayload {
77
+ id: string;
78
+ [key: string]: unknown;
79
+ }
80
+ interface LogPayload {
81
+ [key: string]: unknown;
82
+ }
83
+ /**
84
+ * Extended Redis client with custom commands.
85
+ */
86
+ export interface CustomRedisClient extends RedisClient {
87
+ pushGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string, payload: string): Promise<number>;
88
+ completeGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string): Promise<number>;
89
+ popMany(queue: string, prefix: string, count: number, now: string): Promise<string[]>;
90
+ }
91
+ /**
92
+ * Extended Redis client with custom group commands (Legacy name).
93
+ */
94
+ export type GroupRedisClient = CustomRedisClient;
95
+ /**
96
+ * Redis driver configuration.
97
+ */
98
+ export interface RedisDriverConfig {
99
+ /**
100
+ * Redis client instance (ioredis or node-redis).
101
+ */
102
+ client: RedisClient;
103
+ /**
104
+ * Key prefix (default: `queue:`).
105
+ */
106
+ prefix?: string;
107
+ }
108
+ /**
109
+ * High-performance Redis queue driver.
110
+ *
111
+ * Implements FIFO queues using Redis Lists, reliable priority support, delayed jobs via Sorted Sets,
112
+ * and rate limiting. Uses Lua scripts for atomic operations and advanced features like
113
+ * group-based sequential processing.
114
+ *
115
+ * 支援 binary frame 格式:
116
+ * - push/pushMany:type === 'binary' 且 data instanceof Uint8Array 時,使用 BinaryJobFrame 格式
117
+ * - pop/popMany:透過 magic byte 嗅探自動偵測格式(binary frame 或 JSON)
118
+ *
119
+ * @public
120
+ * @example
121
+ * ```typescript
122
+ * import Redis from 'ioredis';
123
+ * const redis = new Redis();
124
+ * const driver = new RedisDriver({ client: redis });
125
+ * ```
126
+ */
127
+ export declare class RedisDriver implements QueueDriver {
128
+ private prefix;
129
+ private client;
130
+ private static PUSH_SCRIPT;
131
+ private static COMPLETE_SCRIPT;
132
+ private static POP_MANY_SCRIPT;
133
+ constructor(config: RedisDriverConfig);
134
+ /**
135
+ * Get full Redis key for a queue.
136
+ */
137
+ private getKey;
138
+ /**
139
+ * 判斷 job 是否應使用 binary frame 格式傳輸
140
+ *
141
+ * 當 type === 'binary' 且 data 為 Uint8Array 時啟用 binary path
142
+ */
143
+ private isBinaryJob;
144
+ /**
145
+ * 序列化 Job 為適合 Redis 儲存的格式
146
+ *
147
+ * Binary path:使用 BinaryJobFrame 格式(Uint8Array → Buffer)
148
+ * Legacy path:使用 JSON.stringify(string)
149
+ *
150
+ * @returns Buffer(binary path)或 string(legacy path)
151
+ */
152
+ private serializeJobForTransport;
153
+ private serializeJsonJobForTransport;
154
+ /**
155
+ * 自動偵測並解析 Redis payload 格式
156
+ *
157
+ * 1. 若輸入為 Buffer/Uint8Array 且以 Magic bytes 開頭 → 使用 BinaryJobFrame 解碼
158
+ * 2. 否則 → 使用 JSON.parse(legacy path)
159
+ *
160
+ * @param payload - Redis 回傳的資料(string 或 Buffer)
161
+ * @returns 解析後的 SerializedJob
162
+ */
163
+ private parsePayloadAuto;
164
+ /**
165
+ * 解析 JSON 格式的 payload(legacy path)
166
+ */
167
+ private parseJsonPayload;
168
+ /**
169
+ * Pushes a job to Redis.
170
+ *
171
+ * 支援 binary path 和 legacy path:
172
+ * - Binary path:job.type === 'binary' 且 data instanceof Uint8Array → 使用 BinaryJobFrame
173
+ * - Legacy path:其他格式 → 使用 JSON.stringify
174
+ *
175
+ * @param queue - The queue name.
176
+ * @param job - The serialized job.
177
+ * @param options - Push options.
178
+ */
179
+ push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
180
+ /**
181
+ * Completes a job.
182
+ *
183
+ * Crucial for Group FIFO logic to unlock the next job in the group.
184
+ *
185
+ * @param queue - The queue name.
186
+ * @param job - The job to complete.
187
+ */
188
+ complete(queue: string, job: SerializedJob): Promise<void>;
189
+ /**
190
+ * Pops a job from the queue.
191
+ *
192
+ * 支援 binary frame 自動偵測:
193
+ * - 若 client 支援 rpopBuffer → 嘗試讀取 Buffer 並用 magic byte 判斷格式
194
+ * - 否則降級為 string rpop,使用 parsePayloadAuto 解析
195
+ *
196
+ * @param queue - The queue name.
197
+ * @returns The job or `null`.
198
+ */
199
+ pop(queue: string): Promise<SerializedJob | null>;
200
+ /**
201
+ * Manual fallback for pop if Lua fails.
202
+ *
203
+ * 優先使用 rpopBuffer 取得 binary frame,否則降級為 string
204
+ */
205
+ private popManualFallback;
206
+ /**
207
+ * Pops a job using blocking Redis commands (BRPOP).
208
+ *
209
+ * 支援 brpopBuffer 取得 binary frame,否則降級為 string
210
+ *
211
+ * @param queues - The queues to listen to.
212
+ * @param timeout - Timeout in seconds.
213
+ */
214
+ popBlocking(queues: string | string[], timeout: number): Promise<SerializedJob | null>;
215
+ /**
216
+ * Returns the length of the queue (Redis List length).
217
+ *
218
+ * @param queue - The queue name.
219
+ */
220
+ size(queue: string): Promise<number>;
221
+ /**
222
+ * Marks a job as permanently failed by moving it to a DLQ list.
223
+ *
224
+ * 支援 binary path:binary job 使用 BinaryJobFrame 格式存入 DLQ
225
+ *
226
+ * @param queue - The queue name.
227
+ * @param job - The failed job.
228
+ */
229
+ fail(queue: string, job: SerializedJob): Promise<void>;
230
+ /**
231
+ * Clears the queue and its associated delayed/active sets.
232
+ *
233
+ * @param queue - The queue name.
234
+ */
235
+ clear(queue: string): Promise<void>;
236
+ /**
237
+ * Retrieves full stats for the queue using Redis Pipelining.
238
+ *
239
+ * @param queue - The queue name.
240
+ */
241
+ stats(queue: string): Promise<QueueStats>;
242
+ /**
243
+ * Pushes multiple jobs to the queue.
244
+ *
245
+ * 批次 push 支援 binary path 和 legacy path 混合使用。
246
+ * 對於含有 binary job 的批次,透過 pipeline 分批處理。
247
+ *
248
+ * @param queue - The queue name.
249
+ * @param jobs - Array of jobs.
250
+ */
251
+ pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
252
+ /**
253
+ * Pops multiple jobs from the queue.
254
+ *
255
+ * 支援 binary frame 自動偵測格式。
256
+ *
257
+ * @param queue - The queue name.
258
+ * @param count - Max jobs to pop.
259
+ */
260
+ popMany(queue: string, count: number): Promise<SerializedJob[]>;
261
+ /**
262
+ * Reports a worker heartbeat.
263
+ */
264
+ reportHeartbeat(workerInfo: WorkerHeartbeatPayload, prefix?: string): Promise<void>;
265
+ /**
266
+ * Publishes monitoring logs.
267
+ */
268
+ publishLog(logPayload: LogPayload, prefix?: string): Promise<void>;
269
+ /**
270
+ * Checks the rate limit for a queue.
271
+ */
272
+ checkRateLimit(queue: string, config: {
273
+ max: number;
274
+ duration: number;
275
+ }): Promise<boolean>;
276
+ /**
277
+ * Retrieves failed jobs from the DLQ.
278
+ *
279
+ * 支援 lrangeBuffer 讀取 binary frame,否則降級為 string lrange
280
+ *
281
+ * @param queue - The queue name.
282
+ * @param start - Start index.
283
+ * @param end - End index.
284
+ */
285
+ getFailed(queue: string, start?: number, end?: number): Promise<SerializedJob[]>;
286
+ /**
287
+ * Retries failed jobs.
288
+ *
289
+ * @param queue - The queue name.
290
+ * @param count - Jobs to retry.
291
+ */
292
+ retryFailed(queue: string, count?: number): Promise<number>;
293
+ /**
294
+ * Clears the Dead Letter Queue.
295
+ *
296
+ * @param queue - The queue name.
297
+ */
298
+ clearFailed(queue: string): Promise<void>;
299
+ /**
300
+ * Retrieves all discovered queue names from Redis.
301
+ */
302
+ getQueues(): Promise<string[]>;
303
+ /**
304
+ * Enables real-time notifications for job arrivals via Redis Pub/Sub.
305
+ *
306
+ * Sets up the pub/sub connection and prepares for notification callbacks.
307
+ *
308
+ * @throws {Error} If the Redis client doesn't support pub/sub operations.
309
+ */
310
+ enableNotifications(): Promise<void>;
311
+ /**
312
+ * Disables real-time notifications for job arrivals.
313
+ *
314
+ * Stops listening to notification channels.
315
+ */
316
+ disableNotifications(): Promise<void>;
317
+ /**
318
+ * Registers a notification listener for one or more queues.
319
+ *
320
+ * Uses Redis Pub/Sub to listen for job arrivals. When a job is pushed,
321
+ * the driver publishes a notification that triggers the callback.
322
+ *
323
+ * @param queues - Queue name(s) to listen for.
324
+ * @param callback - Function called with queue name when a job arrives.
325
+ */
326
+ onNotify(queues: string | string[], callback: (queue: string) => Promise<void>): Promise<void>;
327
+ }
328
+ export {};
@@ -0,0 +1,114 @@
1
+ import type { SerializedJob } from '../types';
2
+ import type { QueueDriver } from './QueueDriver';
3
+ /**
4
+ * SQS driver configuration.
5
+ */
6
+ export interface SQSDriverConfig {
7
+ /**
8
+ * SQS client instance (`@aws-sdk/client-sqs`).
9
+ */
10
+ client: {
11
+ send: (command: unknown) => Promise<{
12
+ Attributes?: {
13
+ ApproximateNumberOfMessages?: string;
14
+ };
15
+ MessageId?: string;
16
+ Messages?: Array<{
17
+ MessageId?: string;
18
+ ReceiptHandle?: string;
19
+ Body?: string;
20
+ }>;
21
+ }>;
22
+ };
23
+ /**
24
+ * Queue URL prefix (used to build full queue URLs).
25
+ */
26
+ queueUrlPrefix?: string;
27
+ /**
28
+ * Visibility timeout (seconds, default: 30).
29
+ */
30
+ visibilityTimeout?: number;
31
+ /**
32
+ * Long-polling duration (seconds, default: 20).
33
+ */
34
+ waitTimeSeconds?: number;
35
+ }
36
+ /**
37
+ * Amazon SQS queue driver.
38
+ *
39
+ * Wraps the AWS SDK for SQS. Supports standard and FIFO queues, long polling,
40
+ * and visibility timeouts.
41
+ *
42
+ * @public
43
+ * @example
44
+ * ```typescript
45
+ * import { SQSClient } from '@aws-sdk/client-sqs';
46
+ * const sqs = new SQSClient({ region: 'us-east-1' });
47
+ * const driver = new SQSDriver({ client: sqs });
48
+ * ```
49
+ */
50
+ export declare class SQSDriver implements QueueDriver {
51
+ private client;
52
+ private queueUrlPrefix;
53
+ private visibilityTimeout;
54
+ private waitTimeSeconds;
55
+ private queueUrls;
56
+ constructor(config: SQSDriverConfig);
57
+ /**
58
+ * Resolve the full queue URL.
59
+ */
60
+ private getQueueUrl;
61
+ /**
62
+ * Pushes a job to SQS.
63
+ *
64
+ * @param queue - The queue name (or URL).
65
+ * @param job - The serialized job.
66
+ */
67
+ push(queue: string, job: SerializedJob): Promise<void>;
68
+ /**
69
+ * Pops a job from SQS (using long polling).
70
+ *
71
+ * @param queue - The queue name (or URL).
72
+ */
73
+ pop(queue: string): Promise<SerializedJob | null>;
74
+ /**
75
+ * Pops multiple jobs (up to 10).
76
+ *
77
+ * @param queue - The queue name.
78
+ * @param count - Max jobs (capped at 10 by SQS).
79
+ */
80
+ popMany(queue: string, count: number): Promise<SerializedJob[]>;
81
+ /**
82
+ * Returns the approximate number of messages in the queue.
83
+ *
84
+ * @param queue - The queue name.
85
+ */
86
+ size(queue: string): Promise<number>;
87
+ /**
88
+ * Clears the queue by continuously receiving and deleting messages.
89
+ *
90
+ * SQS does not have a "purge" command in the client data plane easily accessible here,
91
+ * so we drain the queue.
92
+ *
93
+ * @param queue - The queue name.
94
+ */
95
+ clear(queue: string): Promise<void>;
96
+ /**
97
+ * Pushes multiple jobs using SQS batch API.
98
+ *
99
+ * @param queue - The queue name.
100
+ * @param jobs - Array of jobs.
101
+ */
102
+ pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
103
+ /**
104
+ * Throws error as SQS requires ReceiptHandle, not just MessageId.
105
+ */
106
+ acknowledge(_messageId: string): Promise<void>;
107
+ /**
108
+ * Deletes a message using its ReceiptHandle (ACK).
109
+ *
110
+ * @param queue - The queue name.
111
+ * @param receiptHandle - The SQS receipt handle.
112
+ */
113
+ deleteMessage(queue: string, receiptHandle: string): Promise<void>;
114
+ }
@@ -0,0 +1,60 @@
1
+ import type { BackpressureConfig } from './types';
2
+ /**
3
+ * Monitors buffer watermarks and signals pause/resume to the Kafka consumer.
4
+ *
5
+ * Decoupled from KafkaDriver to enable independent testing and reuse.
6
+ * Uses callback pattern for minimal overhead.
7
+ *
8
+ * @public
9
+ */
10
+ export declare class BackpressureController {
11
+ private readonly bufferSize;
12
+ private readonly highWatermark;
13
+ private readonly lowWatermark;
14
+ private readonly maxInFlight;
15
+ private paused;
16
+ private inFlightCount;
17
+ private onPause;
18
+ private onResume;
19
+ constructor(bufferSize: number, config?: BackpressureConfig);
20
+ /**
21
+ * Register pause/resume callbacks.
22
+ */
23
+ onBackpressure(callbacks: {
24
+ pause: (topics: string[]) => void;
25
+ resume: (topics: string[]) => void;
26
+ }): void;
27
+ /**
28
+ * Check if a topic should be paused based on buffer level.
29
+ */
30
+ shouldPause(bufferLength: number): boolean;
31
+ /**
32
+ * Check if a topic should be resumed based on buffer level.
33
+ */
34
+ shouldResume(bufferLength: number): boolean;
35
+ /**
36
+ * Acquire a slot for in-flight callback processing.
37
+ * Returns false if max in-flight limit reached.
38
+ */
39
+ acquireSlot(): boolean;
40
+ /**
41
+ * Release a slot after callback processing completes.
42
+ */
43
+ releaseSlot(): void;
44
+ /**
45
+ * Evaluate backpressure for a topic and trigger callbacks if needed.
46
+ */
47
+ evaluate(bufferLength: number, topics: string[]): void;
48
+ /**
49
+ * Check if consumer is currently paused.
50
+ */
51
+ isPaused(): boolean;
52
+ /**
53
+ * Get current in-flight callback count.
54
+ */
55
+ getInFlightCount(): number;
56
+ /**
57
+ * Reset state (for cleanup/testing).
58
+ */
59
+ reset(): void;
60
+ }
@@ -0,0 +1,50 @@
1
+ import type { SerializedJob } from '../../types';
2
+ import type { MessageBuffer } from './MessageBuffer';
3
+ import type { BatchConfig, BatchResult, BufferedMessage, KafkaProducerClient } from './types';
4
+ /**
5
+ * 批次處理引擎,管理 Kafka 的批次發送和消費。
6
+ *
7
+ * 功能:
8
+ * - 並行管線推送(pushBatches):最多 concurrency 個並行批次發送
9
+ * - 批次收集(collectBatch):等待 linger 時間收集滿批次
10
+ * - 並行回調處理(processBatch):控制並行度 + 逾時 + 部分失敗追蹤
11
+ *
12
+ * @public
13
+ */
14
+ export declare class BatchProcessor {
15
+ private readonly maxBatchSize;
16
+ private readonly batchLingerMs;
17
+ private readonly concurrency;
18
+ private readonly enablePipeline;
19
+ constructor(config?: BatchConfig);
20
+ /**
21
+ * 並行管線推送多個批次到 Kafka producer。
22
+ *
23
+ * 將 jobs 分割成 maxBatchSize 的批次,
24
+ * 最多 concurrency 個並行發送。
25
+ */
26
+ pushBatches(producer: KafkaProducerClient, topic: string, jobs: SerializedJob[], serialize: (job: SerializedJob) => string | Buffer): Promise<void>;
27
+ /**
28
+ * 從 buffer 收集一個批次的訊息。
29
+ *
30
+ * 立即取出現有訊息,如果未滿則等待 linger 時間收集更多。
31
+ */
32
+ collectBatch(buffer: MessageBuffer, topic: string, maxWaitMs: number): Promise<BufferedMessage[]>;
33
+ /**
34
+ * 並行執行批次回調並追蹤結果。
35
+ *
36
+ * 使用信號量控制並行度,支援逾時和部分失敗。
37
+ */
38
+ processBatch(messages: BufferedMessage[], callback: (job: SerializedJob) => Promise<void>, options: {
39
+ concurrency: number;
40
+ timeoutMs: number;
41
+ }): Promise<BatchResult>;
42
+ /**
43
+ * 取得當前配置快照。
44
+ */
45
+ getConfig(): Required<BatchConfig>;
46
+ /**
47
+ * 內部:發送單一批次到 Kafka。
48
+ */
49
+ private sendBatch;
50
+ }