@gravito/stream 2.0.1 → 2.1.0

package/dist/drivers/RedisDriver.d.ts

@@ -0,0 +1,294 @@
+import type { JobPushOptions, QueueStats, SerializedJob } from '../types';
+import type { QueueDriver } from './QueueDriver';
+/**
+ * Interface for Redis clients (compatible with ioredis and node-redis).
+ */
+export interface RedisClient {
+    lpush(key: string, ...values: string[]): Promise<number>;
+    rpop(key: string, count?: number): Promise<string | string[] | null>;
+    llen(key: string): Promise<number>;
+    del(key: string, ...keys: string[]): Promise<number>;
+    lpushx?(key: string, ...values: string[]): Promise<number>;
+    rpoplpush?(src: string, dst: string): Promise<string | null>;
+    zadd?(key: string, score: number, member: string): Promise<number>;
+    zrange?(key: string, start: number, end: number, ...args: string[]): Promise<string[]>;
+    zrem?(key: string, ...members: string[]): Promise<number>;
+    get?(key: string): Promise<string | null>;
+    set?(key: string, value: string, ...args: any[]): Promise<'OK' | null>;
+    ltrim?(key: string, start: number, stop: number): Promise<'OK'>;
+    lrange?(key: string, start: number, stop: number): Promise<string[]>;
+    publish?(channel: string, message: string): Promise<number>;
+    pipeline?(): any;
+    defineCommand?(name: string, options: {
+        numberOfKeys: number;
+        lua: string;
+    }): void;
+    incr?(key: string): Promise<number>;
+    expire?(key: string, seconds: number): Promise<number>;
+    eval(script: string, numKeys: number, ...args: (string | number)[]): Promise<any>;
+    /**
+     * Binary-capable RPOP（ioredis Buffer mode）
+     * 回傳 Buffer 而非 string，支援 binary frame 的零拷貝讀取
+     */
+    rpopBuffer?(key: string, count?: number): Promise<Buffer | Buffer[] | null>;
+    /**
+     * Binary-capable BRPOP（ioredis Buffer mode）
+     * 支援 blocking pop 的 binary frame 讀取
+     */
+    brpopBuffer?(...args: any[]): Promise<[Buffer, Buffer] | null>;
+    /**
+     * Binary-capable LRANGE（ioredis Buffer mode）
+     * 用於從 DLQ 讀取 binary frame
+     */
+    lrangeBuffer?(key: string, start: number, stop: number): Promise<Buffer[]>;
+    /**
+     * Binary-capable LPUSH（接受 Buffer）
+     * 用於將 binary frame 寫入 Redis list
+     */
+    lpushBuffer?(key: string, ...values: Buffer[]): Promise<number>;
+    [key: string]: any;
+}
+/**
+ * Extended Redis client with custom commands.
+ */
+export interface CustomRedisClient extends RedisClient {
+    pushGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string, payload: string): Promise<number>;
+    completeGroupJob(waitList: string, activeSet: string, pendingList: string, groupId: string): Promise<number>;
+    popMany(queue: string, prefix: string, count: number, now: string): Promise<string[]>;
+}
+/**
+ * Extended Redis client with custom group commands (Legacy name).
+ */
+export type GroupRedisClient = CustomRedisClient;
+/**
+ * Redis driver configuration.
+ */
+export interface RedisDriverConfig {
+    /**
+     * Redis client instance (ioredis or node-redis).
+     */
+    client: RedisClient;
+    /**
+     * Key prefix (default: `queue:`).
+     */
+    prefix?: string;
+}
+/**
+ * High-performance Redis queue driver.
+ *
+ * Implements FIFO queues using Redis Lists, reliable priority support, delayed jobs via Sorted Sets,
+ * and rate limiting. Uses Lua scripts for atomic operations and advanced features like
+ * group-based sequential processing.
+ *
+ * 支援 binary frame 格式：
+ * - push/pushMany：type === 'binary' 且 data instanceof Uint8Array 時，使用 BinaryJobFrame 格式
+ * - pop/popMany：透過 magic byte 嗅探自動偵測格式（binary frame 或 JSON）
+ *
+ * @public
+ * @example
+ * ```typescript
+ * import Redis from 'ioredis';
+ * const redis = new Redis();
+ * const driver = new RedisDriver({ client: redis });
+ * ```
+ */
+export declare class RedisDriver implements QueueDriver {
+    private prefix;
+    private client;
+    private static PUSH_SCRIPT;
+    private static COMPLETE_SCRIPT;
+    private static POP_MANY_SCRIPT;
+    constructor(config: RedisDriverConfig);
+    /**
+     * Get full Redis key for a queue.
+     */
+    private getKey;
+    /**
+     * 判斷 job 是否應使用 binary frame 格式傳輸
+     *
+     * 當 type === 'binary' 且 data 為 Uint8Array 時啟用 binary path
+     */
+    private isBinaryJob;
+    /**
+     * 序列化 Job 為適合 Redis 儲存的格式
+     *
+     * Binary path：使用 BinaryJobFrame 格式（Uint8Array → Buffer）
+     * Legacy path：使用 JSON.stringify（string）
+     *
+     * @returns Buffer（binary path）或 string（legacy path）
+     */
+    private serializeJobForTransport;
+    /**
+     * 自動偵測並解析 Redis payload 格式
+     *
+     * 1. 若輸入為 Buffer/Uint8Array 且以 Magic bytes 開頭 → 使用 BinaryJobFrame 解碼
+     * 2. 否則 → 使用 JSON.parse（legacy path）
+     *
+     * @param payload - Redis 回傳的資料（string 或 Buffer）
+     * @returns 解析後的 SerializedJob
+     */
+    private parsePayloadAuto;
+    /**
+     * 解析 JSON 格式的 payload（legacy path）
+     */
+    private parseJsonPayload;
+    /**
+     * Pushes a job to Redis.
+     *
+     * 支援 binary path 和 legacy path：
+     * - Binary path：job.type === 'binary' 且 data instanceof Uint8Array → 使用 BinaryJobFrame
+     * - Legacy path：其他格式 → 使用 JSON.stringify
+     *
+     * @param queue - The queue name.
+     * @param job - The serialized job.
+     * @param options - Push options.
+     */
+    push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
+    /**
+     * Completes a job.
+     *
+     * Crucial for Group FIFO logic to unlock the next job in the group.
+     *
+     * @param queue - The queue name.
+     * @param job - The job to complete.
+     */
+    complete(queue: string, job: SerializedJob): Promise<void>;
+    /**
+     * Pops a job from the queue.
+     *
+     * 支援 binary frame 自動偵測：
+     * - 若 client 支援 rpopBuffer → 嘗試讀取 Buffer 並用 magic byte 判斷格式
+     * - 否則降級為 string rpop，使用 parsePayloadAuto 解析
+     *
+     * @param queue - The queue name.
+     * @returns The job or `null`.
+     */
+    pop(queue: string): Promise<SerializedJob | null>;
+    /**
+     * Manual fallback for pop if Lua fails.
+     *
+     * 優先使用 rpopBuffer 取得 binary frame，否則降級為 string
+     */
+    private popManualFallback;
+    /**
+     * Pops a job using blocking Redis commands (BRPOP).
+     *
+     * 支援 brpopBuffer 取得 binary frame，否則降級為 string
+     *
+     * @param queues - The queues to listen to.
+     * @param timeout - Timeout in seconds.
+     */
+    popBlocking(queues: string | string[], timeout: number): Promise<SerializedJob | null>;
+    /**
+     * Returns the length of the queue (Redis List length).
+     *
+     * @param queue - The queue name.
+     */
+    size(queue: string): Promise<number>;
+    /**
+     * Marks a job as permanently failed by moving it to a DLQ list.
+     *
+     * 支援 binary path：binary job 使用 BinaryJobFrame 格式存入 DLQ
+     *
+     * @param queue - The queue name.
+     * @param job - The failed job.
+     */
+    fail(queue: string, job: SerializedJob): Promise<void>;
+    /**
+     * Clears the queue and its associated delayed/active sets.
+     *
+     * @param queue - The queue name.
+     */
+    clear(queue: string): Promise<void>;
+    /**
+     * Retrieves full stats for the queue using Redis Pipelining.
+     *
+     * @param queue - The queue name.
+     */
+    stats(queue: string): Promise<QueueStats>;
+    /**
+     * Pushes multiple jobs to the queue.
+     *
+     * 批次 push 支援 binary path 和 legacy path 混合使用。
+     * 對於含有 binary job 的批次，透過 pipeline 分批處理。
+     *
+     * @param queue - The queue name.
+     * @param jobs - Array of jobs.
+     */
+    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
+    /**
+     * Pops multiple jobs from the queue.
+     *
+     * 支援 binary frame 自動偵測格式。
+     *
+     * @param queue - The queue name.
+     * @param count - Max jobs to pop.
+     */
+    popMany(queue: string, count: number): Promise<SerializedJob[]>;
+    /**
+     * Reports a worker heartbeat.
+     */
+    reportHeartbeat(workerInfo: any, prefix?: string): Promise<void>;
+    /**
+     * Publishes monitoring logs.
+     */
+    publishLog(logPayload: any, prefix?: string): Promise<void>;
+    /**
+     * Checks the rate limit for a queue.
+     */
+    checkRateLimit(queue: string, config: {
+        max: number;
+        duration: number;
+    }): Promise<boolean>;
+    /**
+     * Retrieves failed jobs from the DLQ.
+     *
+     * 支援 lrangeBuffer 讀取 binary frame，否則降級為 string lrange
+     *
+     * @param queue - The queue name.
+     * @param start - Start index.
+     * @param end - End index.
+     */
+    getFailed(queue: string, start?: number, end?: number): Promise<SerializedJob[]>;
+    /**
+     * Retries failed jobs.
+     *
+     * @param queue - The queue name.
+     * @param count - Jobs to retry.
+     */
+    retryFailed(queue: string, count?: number): Promise<number>;
+    /**
+     * Clears the Dead Letter Queue.
+     *
+     * @param queue - The queue name.
+     */
+    clearFailed(queue: string): Promise<void>;
+    /**
+     * Retrieves all discovered queue names from Redis.
+     */
+    getQueues(): Promise<string[]>;
+    /**
+     * Enables real-time notifications for job arrivals via Redis Pub/Sub.
+     *
+     * Sets up the pub/sub connection and prepares for notification callbacks.
+     *
+     * @throws {Error} If the Redis client doesn't support pub/sub operations.
+     */
+    enableNotifications(): Promise<void>;
+    /**
+     * Disables real-time notifications for job arrivals.
+     *
+     * Stops listening to notification channels.
+     */
+    disableNotifications(): Promise<void>;
+    /**
+     * Registers a notification listener for one or more queues.
+     *
+     * Uses Redis Pub/Sub to listen for job arrivals. When a job is pushed,
+     * the driver publishes a notification that triggers the callback.
+     *
+     * @param queues - Queue name(s) to listen for.
+     * @param callback - Function called with queue name when a job arrives.
+     */
+    onNotify(queues: string | string[], callback: (queue: string) => Promise<void>): Promise<void>;
+}

package/dist/drivers/SQSDriver.d.ts

@@ -0,0 +1,111 @@
+import type { SerializedJob } from '../types';
+import type { QueueDriver } from './QueueDriver';
+/**
+ * SQS driver configuration.
+ */
+export interface SQSDriverConfig {
+    /**
+     * SQS client instance (`@aws-sdk/client-sqs`).
+     */
+    client: {
+        send: (command: unknown) => Promise<{
+            MessageId?: string;
+            Messages?: Array<{
+                MessageId?: string;
+                ReceiptHandle?: string;
+                Body?: string;
+            }>;
+        }>;
+    };
+    /**
+     * Queue URL prefix (used to build full queue URLs).
+     */
+    queueUrlPrefix?: string;
+    /**
+     * Visibility timeout (seconds, default: 30).
+     */
+    visibilityTimeout?: number;
+    /**
+     * Long-polling duration (seconds, default: 20).
+     */
+    waitTimeSeconds?: number;
+}
+/**
+ * Amazon SQS queue driver.
+ *
+ * Wraps the AWS SDK for SQS. Supports standard and FIFO queues, long polling,
+ * and visibility timeouts.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * import { SQSClient } from '@aws-sdk/client-sqs';
+ * const sqs = new SQSClient({ region: 'us-east-1' });
+ * const driver = new SQSDriver({ client: sqs });
+ * ```
+ */
+export declare class SQSDriver implements QueueDriver {
+    private client;
+    private queueUrlPrefix;
+    private visibilityTimeout;
+    private waitTimeSeconds;
+    private queueUrls;
+    constructor(config: SQSDriverConfig);
+    /**
+     * Resolve the full queue URL.
+     */
+    private getQueueUrl;
+    /**
+     * Pushes a job to SQS.
+     *
+     * @param queue - The queue name (or URL).
+     * @param job - The serialized job.
+     */
+    push(queue: string, job: SerializedJob): Promise<void>;
+    /**
+     * Pops a job from SQS (using long polling).
+     *
+     * @param queue - The queue name (or URL).
+     */
+    pop(queue: string): Promise<SerializedJob | null>;
+    /**
+     * Pops multiple jobs (up to 10).
+     *
+     * @param queue - The queue name.
+     * @param count - Max jobs (capped at 10 by SQS).
+     */
+    popMany(queue: string, count: number): Promise<SerializedJob[]>;
+    /**
+     * Returns the approximate number of messages in the queue.
+     *
+     * @param queue - The queue name.
+     */
+    size(queue: string): Promise<number>;
+    /**
+     * Clears the queue by continuously receiving and deleting messages.
+     *
+     * SQS does not have a "purge" command in the client data plane easily accessible here,
+     * so we drain the queue.
+     *
+     * @param queue - The queue name.
+     */
+    clear(queue: string): Promise<void>;
+    /**
+     * Pushes multiple jobs using SQS batch API.
+     *
+     * @param queue - The queue name.
+     * @param jobs - Array of jobs.
+     */
+    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
+    /**
+     * Throws error as SQS requires ReceiptHandle, not just MessageId.
+     */
+    acknowledge(_messageId: string): Promise<void>;
+    /**
+     * Deletes a message using its ReceiptHandle (ACK).
+     *
+     * @param queue - The queue name.
+     * @param receiptHandle - The SQS receipt handle.
+     */
+    deleteMessage(queue: string, receiptHandle: string): Promise<void>;
+}

package/dist/drivers/kafka/BackpressureController.d.ts

@@ -0,0 +1,60 @@
+import type { BackpressureConfig } from './types';
+/**
+ * Monitors buffer watermarks and signals pause/resume to the Kafka consumer.
+ *
+ * Decoupled from KafkaDriver to enable independent testing and reuse.
+ * Uses callback pattern for minimal overhead.
+ *
+ * @public
+ */
+export declare class BackpressureController {
+    private readonly bufferSize;
+    private readonly highWatermark;
+    private readonly lowWatermark;
+    private readonly maxInFlight;
+    private paused;
+    private inFlightCount;
+    private onPause;
+    private onResume;
+    constructor(bufferSize: number, config?: BackpressureConfig);
+    /**
+     * Register pause/resume callbacks.
+     */
+    onBackpressure(callbacks: {
+        pause: (topics: string[]) => void;
+        resume: (topics: string[]) => void;
+    }): void;
+    /**
+     * Check if a topic should be paused based on buffer level.
+     */
+    shouldPause(bufferLength: number): boolean;
+    /**
+     * Check if a topic should be resumed based on buffer level.
+     */
+    shouldResume(bufferLength: number): boolean;
+    /**
+     * Acquire a slot for in-flight callback processing.
+     * Returns false if max in-flight limit reached.
+     */
+    acquireSlot(): boolean;
+    /**
+     * Release a slot after callback processing completes.
+     */
+    releaseSlot(): void;
+    /**
+     * Evaluate backpressure for a topic and trigger callbacks if needed.
+     */
+    evaluate(bufferLength: number, topics: string[]): void;
+    /**
+     * Check if consumer is currently paused.
+     */
+    isPaused(): boolean;
+    /**
+     * Get current in-flight callback count.
+     */
+    getInFlightCount(): number;
+    /**
+     * Reset state (for cleanup/testing).
+     */
+    reset(): void;
+}

package/dist/drivers/kafka/BatchProcessor.d.ts

@@ -0,0 +1,50 @@
+import type { SerializedJob } from '../../types';
+import type { MessageBuffer } from './MessageBuffer';
+import type { BatchConfig, BatchResult, BufferedMessage, KafkaProducerClient } from './types';
+/**
+ * 批次處理引擎，管理 Kafka 的批次發送和消費。
+ *
+ * 功能：
+ * - 並行管線推送（pushBatches）：最多 concurrency 個並行批次發送
+ * - 批次收集（collectBatch）：等待 linger 時間收集滿批次
+ * - 並行回調處理（processBatch）：控制並行度 + 逾時 + 部分失敗追蹤
+ *
+ * @public
+ */
+export declare class BatchProcessor {
+    private readonly maxBatchSize;
+    private readonly batchLingerMs;
+    private readonly concurrency;
+    private readonly enablePipeline;
+    constructor(config?: BatchConfig);
+    /**
+     * 並行管線推送多個批次到 Kafka producer。
+     *
+     * 將 jobs 分割成 maxBatchSize 的批次，
+     * 最多 concurrency 個並行發送。
+     */
+    pushBatches(producer: KafkaProducerClient, topic: string, jobs: SerializedJob[], serialize: (job: SerializedJob) => string | Buffer): Promise<void>;
+    /**
+     * 從 buffer 收集一個批次的訊息。
+     *
+     * 立即取出現有訊息，如果未滿則等待 linger 時間收集更多。
+     */
+    collectBatch(buffer: MessageBuffer, topic: string, maxWaitMs: number): Promise<BufferedMessage[]>;
+    /**
+     * 並行執行批次回調並追蹤結果。
+     *
+     * 使用信號量控制並行度，支援逾時和部分失敗。
+     */
+    processBatch(messages: BufferedMessage[], callback: (job: SerializedJob) => Promise<void>, options: {
+        concurrency: number;
+        timeoutMs: number;
+    }): Promise<BatchResult>;
+    /**
+     * 取得當前配置快照。
+     */
+    getConfig(): Required<BatchConfig>;
+    /**
+     * 內部：發送單一批次到 Kafka。
+     */
+    private sendBatch;
+}

package/dist/drivers/kafka/ConsumerLifecycleManager.d.ts

@@ -0,0 +1,80 @@
+import { EventEmitter } from 'node:events';
+import type { ConsumerLifecycleState, LifecycleEvent } from './types';
+/**
+ * Manages consumer lifecycle state transitions and events.
+ *
+ * State transitions:
+ * - idle → starting → running → stopping → stopped
+ * - running → restarting → running
+ * - any state → error
+ *
+ * Emits lifecycle events on state changes for coordination with other components.
+ *
+ * @public
+ */
+export declare class ConsumerLifecycleManager extends EventEmitter {
+    private currentState;
+    private previousState;
+    private lastTransitionTime;
+    private stateHistory;
+    /**
+     * Register a listener for lifecycle events.
+     * Event data: { state, previousState, timestamp, error? }
+     */
+    onStateChange(listener: (event: LifecycleEvent) => void): void;
+    /**
+     * Start the consumer (transition idle → starting → running).
+     */
+    start(): Promise<void>;
+    /**
+     * Restart the consumer (running → restarting → running).
+     */
+    restart(): Promise<void>;
+    /**
+     * Stop the consumer (running/starting → stopping → stopped).
+     */
+    stop(): Promise<void>;
+    /**
+     * Reset to idle state (for testing/reuse).
+     */
+    reset(): void;
+    /**
+     * Get current lifecycle state.
+     */
+    getState(): ConsumerLifecycleState;
+    /**
+     * Get previous lifecycle state.
+     */
+    getPreviousState(): ConsumerLifecycleState;
+    /**
+     * Get last state transition timestamp.
+     */
+    getLastTransitionTime(): number;
+    /**
+     * Get state transition history.
+     */
+    getStateHistory(): Array<{
+        state: ConsumerLifecycleState;
+        timestamp: number;
+    }>;
+    /**
+     * Check if consumer is in running state.
+     */
+    isRunning(): boolean;
+    /**
+     * Check if consumer is in error state.
+     */
+    isError(): boolean;
+    /**
+     * Check if consumer is stopped.
+     */
+    isStopped(): boolean;
+    /**
+     * Internal: transition to new state and emit event.
+     */
+    private transition;
+    /**
+     * Internal: transition to error state with error details.
+     */
+    private transitionError;
+}

package/dist/drivers/kafka/ErrorCategorizer.d.ts

@@ -0,0 +1,39 @@
+import type { ErrorCategory } from './types';
+/**
+ * 根據錯誤訊息與型別分類 Kafka 操作錯誤。
+ *
+ * 錯誤分類：
+ * - `transient`（影響電路斷路器）：連線失敗、逾時、網路錯誤
+ * - `serialization`（不影響電路）：JSON parse 失敗、序列化錯誤
+ * - `business_logic`（不影響電路）：回調拋出的一般錯誤
+ * - `permanent`（不影響電路）：其他永久性失敗
+ *
+ * 使用者應根據錯誤分類決定是否影響電路斷路器狀態。
+ * 只有 `transient` 錯誤才應遞增電路的失敗計數。
+ *
+ * @public
+ */
+export declare class ErrorCategorizer {
+    /**
+     * 將錯誤分類為標準分類之一。
+     *
+     * 分類規則（優先順序）：
+     * 1. `transient` - ECONNREFUSED, ETIMEDOUT, ECONNRESET, 網路相關
+     * 2. `serialization` - SyntaxError, JSON.parse 相關
+     * 3. `business_logic` - 回調拋出的一般 Error（isBusinessLogic 為 true）
+     * 4. `permanent` - 其他所有錯誤
+     */
+    categorize(error: Error, isBusinessLogic?: boolean): ErrorCategory;
+    /**
+     * 檢查某個分類的錯誤是否應影響電路斷路器。
+     *
+     * 只有 `transient` 分類應遞增電路的失敗計數。
+     * 其他分類（`serialization`, `business_logic`, `permanent`）
+     * 應直接跳過電路斷路器邏輯。
+     */
+    shouldAffectCircuit(category: ErrorCategory): boolean;
+    /**
+     * 根據錯誤直接判斷是否應影響電路（便利方法）。
+     */
+    shouldRecordInCircuit(error: Error, isBusinessLogic?: boolean): boolean;
+}