@gravito/stream 2.1.2 → 3.0.0

package/dist/consumer/JobSourceGenerator.d.ts

@@ -1,31 +0,0 @@
-import type { QueueManager } from '../QueueManager';
-import type { FetchResult, JobSourceOptions, StopSignal } from './types';
-/**
- * Async Generator，用於從佇列持續抓取 Job。
- *
- * 封裝所有抓取策略（blocking/batch/sequential），自適應 backoff，
- * 以及 rate limit 過濾邏輯。每次 yield 一個 FetchResult，
- * 包含本輪抓取的 job 列表和是否使用阻塞模式的資訊。
- *
- * 設計原則：
- * - blocking pop 已在 driver 層等待，不再額外 backoff
- * - 自適應退避：空佇列時指數增長，有 job 時重設
- * - rate limit：每輪抓取前過濾掉受限的佇列
- * - 中止信號：每次 yield 前檢查 signal.aborted
- *
- * @param queueManager - QueueManager 實例
- * @param options - 抓取策略選項
- * @param signal - 中止信號（AbortController.signal）
- *
- * @example
- * ```typescript
- * const source = jobSourceGenerator(queueManager, options, controller.signal)
- *
- * for await (const result of source) {
- *   for (const job of result.jobs) {
- *     await processJob(job)
- *   }
- * }
- * ```
- */
-export declare function jobSourceGenerator(queueManager: QueueManager, options: JobSourceOptions, signal: StopSignal): AsyncGenerator<FetchResult>;

package/dist/consumer/PollingStrategy.d.ts

@@ -1,42 +0,0 @@
-import type { Job } from '../Job';
-import type { QueueManager } from '../QueueManager';
-import type { ConsumerStrategy } from './ConsumerStrategy';
-/**
- * PollingStrategy implements pull-based job consumption.
- *
- * Continuously polls queues at adaptive intervals, backing off when empty
- * and resetting when jobs are found.
- *
- * Characteristics:
- * - Simple and reliable
- * - Predictable polling intervals
- * - Higher latency and resource usage vs reactive
- * - No external dependencies (works with any driver)
- *
- * @public
- */
-export declare class PollingStrategy implements ConsumerStrategy {
-    private queueManager;
-    private queues;
-    private connectionName;
-    private options;
-    private running;
-    constructor(queueManager: QueueManager, queues: string[], connectionName: string, options: {
-        minPollInterval: number;
-        maxPollInterval: number;
-        backoffMultiplier: number;
-        batchSize: number;
-        useBlocking: boolean;
-        blockingTimeout: number;
-        concurrency: number;
-        stats: {
-            active: number;
-        };
-        debug: boolean;
-        log: (message: string, data?: unknown) => void;
-    });
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    isRunning(): boolean;
-    fetchJobs(): Promise<Job[]>;
-}

package/dist/consumer/ReactiveStrategy.d.ts

@@ -1,41 +0,0 @@
-import type { Job } from '../Job';
-import type { QueueManager } from '../QueueManager';
-import type { ConsumerStrategy } from './ConsumerStrategy';
-/**
- * ReactiveStrategy implements push-based job consumption.
- *
- * Listens for queue notifications and pulls jobs reactively when they arrive.
- * Implements drain-loop pattern with optional polling fallback.
- *
- * Characteristics:
- * - Low latency (immediate job pickup)
- * - Efficient resource usage (no continuous polling)
- * - Requires driver support for onNotify()
- * - Includes polling fallback to prevent starvation
- *
- * @public
- */
-export declare class ReactiveStrategy implements ConsumerStrategy {
-    private queueManager;
-    private queues;
-    private connectionName;
-    private options;
-    private running;
-    private lastNotificationTime;
-    private fallbackPollingTimer;
-    constructor(queueManager: QueueManager, queues: string[], connectionName: string, options: {
-        concurrency: number;
-        batchSize: number;
-        stats: {
-            active: number;
-        };
-        reactivePollingFallback: number;
-        debug: boolean;
-        log: (message: string, data?: unknown) => void;
-    });
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    isRunning(): boolean;
-    fetchJobs(): Promise<Job[]>;
-    private startFallbackPolling;
-}

package/dist/consumer/StreamingConsumer.d.ts

@@ -1,90 +0,0 @@
-import { EventEmitter } from 'node:events';
-import type { ConsumerOptions } from '../Consumer';
-import type { QueueManager } from '../QueueManager';
-import type { ConsumerStats } from './types';
-/**
- * StreamingConsumer 是 Consumer 管線的核心實作。
- *
- * 組合以下元件，提供高效能、可測試的 job 消費管線：
- * - JobSourceGenerator：抓取 job 的 Async Generator
- * - ConcurrencyGate：Promise-based Semaphore，取代 busy-wait
- * - GroupSequencer：確保相同 groupId 的 job 依序執行
- * - JobExecutor：完整的 job 執行生命週期
- * - HeartbeatManager：定時心跳與監控日誌
- *
- * 主管線採用 `for await...of` 模式，
- * 完全消除原始 Consumer 中的 `setTimeout(50)` busy-wait。
- *
- * @example
- * ```typescript
- * const streaming = new StreamingConsumer(queueManager, options)
- * streaming.on('job:processed', (payload) => console.log('Done:', payload.job.id))
- *
- * await streaming.start()
- * ```
- */
-export declare class StreamingConsumer extends EventEmitter {
-    private readonly queueManager;
-    private readonly options;
-    private running;
-    private stopRequested;
-    private readonly workerId;
-    private readonly stats;
-    private gate;
-    private sequencer;
-    private heartbeat;
-    private abortController;
-    private sharedWorker;
-    constructor(queueManager: QueueManager, options: ConsumerOptions);
-    /**
-     * 啟動主消費管線。
-     *
-     * 初始化所有元件，然後進入 `for await...of` 管線迴圈：
-     * 1. 從 jobSourceGenerator 取得 FetchResult
-     * 2. 對每個 job，acquire ConcurrencyGate
-     * 3. 透過 GroupSequencer 確保 group 序列
-     * 4. 由 JobExecutor 執行 job
-     * 5. release ConcurrencyGate
-     *
-     * @throws {Error} 如果 consumer 已在執行中
-     */
-    start(): Promise<void>;
-    /**
-     * 請求優雅停止。
-     *
-     * 設定 stopRequested 旗標，並中止 generator，
-     * 等待目前執行中的 job 完成後停止。
-     *
-     * @returns 等待完全停止的 Promise
-     */
-    requestStop(): Promise<void>;
-    /**
-     * 檢查是否仍有 job 在執行中。
-     */
-    isActive(): boolean;
-    /**
-     * 取得目前的統計資料快照。
-     */
-    getStats(): ConsumerStats;
-    /**
-     * 重置統計計數器。
-     */
-    resetStats(): void;
-    /**
-     * Consumer 是否正在執行中。
-     */
-    isRunning(): boolean;
-    /**
-     * 等待所有正在執行的 job 完成。
-     */
-    private waitForAllJobsToComplete;
-    /**
-     * 清理所有資源。
-     */
-    private cleanup;
-    /**
-     * 輸出 debug 日誌（僅在 debug=true 時）。
-     */
-    private log;
-    private createWorkerForExecution;
-}

package/dist/consumer/index.d.ts

@@ -1,13 +0,0 @@
-/**
- * Consumer 管線模組 - 重新導出所有公開類型與類別。
- *
- * 供測試及進階使用者直接引用各個子元件。
- */
-export { ConcurrencyGate } from './ConcurrencyGate';
-export { GroupSequencer } from './GroupSequencer';
-export type { HeartbeatManagerOptions } from './HeartbeatManager';
-export { HeartbeatManager } from './HeartbeatManager';
-export { JobExecutor } from './JobExecutor';
-export { jobSourceGenerator } from './JobSourceGenerator';
-export { StreamingConsumer } from './StreamingConsumer';
-export type { ConsumerStats, ExecutionResult, ExecutorOptions, FetchResult, JobSourceOptions, StopSignal, } from './types';

package/dist/consumer/types.d.ts

@@ -1,102 +0,0 @@
-import type { Job } from '../Job';
-/**
- * 從佇列抓取 job 的結果。
- *
- * JobSourceGenerator 每次 yield 一個 FetchResult，
- * 包含本輪抓取的所有 job，以及是否使用阻塞式等待的資訊。
- */
-export interface FetchResult {
-    /** 本次抓取到的 job 列表（可能為空） */
-    jobs: Job[];
-    /**
-     * 是否使用了阻塞式 pop（如 BLPOP）。
-     * 若為 true，代表已在 driver 層等待，不需額外 backoff。
-     */
-    didBlock: boolean;
-}
-/**
- * JobSourceGenerator 的選項設定。
- */
-export interface JobSourceOptions {
-    /** 要消費的佇列名稱列表 */
-    queues: string[];
-    /** 連線名稱，不指定時使用預設連線 */
-    connection?: string;
-    /** 每次嘗試抓取的 job 數量，大於 1 時使用批次抓取 */
-    batchSize: number;
-    /** 是否使用阻塞式 pop */
-    useBlocking: boolean;
-    /** 阻塞式 pop 的逾時秒數 */
-    blockingTimeout: number;
-    /** 佇列為空時是否持續存活 */
-    keepAlive: boolean;
-    /** 最小輪詢間隔（毫秒） */
-    minPollInterval: number;
-    /** 最大輪詢間隔（毫秒） */
-    maxPollInterval: number;
-    /** 自適應退避乘數 */
-    backoffMultiplier: number;
-    /** Rate limit 設定，key 為佇列名稱 */
-    rateLimits?: Record<string, {
-        max: number;
-        duration: number;
-    }>;
-    /** 並發容量 getter，讓 generator 知道可以抓多少 job */
-    getCapacity: () => number;
-}
-/**
- * JobExecutor 執行選項。
- */
-export interface ExecutorOptions {
-    /** debug 模式，會輸出詳細 log */
-    debug: boolean;
-    /** Worker ID，用於 log 識別 */
-    workerId: string;
-    /** Worker 選項（maxAttempts 等） */
-    workerOptions?: {
-        maxAttempts?: number;
-    };
-    /** 監控設定 */
-    monitor?: boolean | {
-        interval?: number;
-        extraInfo?: Record<string, unknown>;
-        prefix?: string;
-    };
-    /** 最大請求數量，達到後自動停止 */
-    maxRequests?: number;
-    /** 外部事件回呼 */
-    onEvent?: (event: string, payload: unknown) => void;
-}
-/**
- * JobExecutor 執行一個 job 後的結果。
- */
-export interface ExecutionResult {
-    /** job 是否成功完成 */
-    success: boolean;
-    /** 執行耗時（毫秒） */
-    duration: number;
-    /** 是否因達到 maxRequests 而需要停止 consumer */
-    shouldStop: boolean;
-    /** 如果失敗，包含錯誤資訊 */
-    error?: Error;
-}
-/**
- * Consumer 執行統計資訊。
- */
-export interface ConsumerStats {
-    /** 已成功處理的 job 數量 */
-    processed: number;
-    /** 失敗（含重試失敗）的 job 數量 */
-    failed: number;
-    /** 重試的 job 數量 */
-    retried: number;
-    /** 目前正在執行的 job 數量 */
-    active: number;
-}
-/**
- * 停止訊號，用於 AbortController 整合。
- */
-export interface StopSignal {
-    /** 是否已發出停止訊號 */
-    readonly aborted: boolean;
-}

package/dist/drivers/BinaryJobFrame.d.ts

@@ -1,78 +0,0 @@
-import type { SerializedJob } from '../types';
-/**
- * BinaryJobFrame - Gravito Job 的二進制幀格式
- *
- * 幀結構（Frame Format）：
- *   Magic(2B) + Version(1B) + Flags(1B) + MetaLen(2B Big-Endian) + Metadata(CBOR) + Payload
- *
- * - Magic：固定為 [0x47, 0x4A]（"GJ" = Gravito Job）
- * - Version：目前為 0x01
- * - Flags：位元旗標，目前定義：
- *     Bit 0 (HAS_CLASS_NAME)：是否包含 className 欄位
- *     Bit 1 (HAS_GROUP_ID)：是否包含 groupId 欄位
- *     Bit 2 (HAS_DELAY)：是否包含 delaySeconds 欄位
- *     Bit 3 (HAS_PRIORITY)：是否包含 priority 欄位
- *     Bit 4 (HAS_ERROR)：是否包含 error / failedAt 欄位
- * - MetaLen：Metadata CBOR 資料的長度（2 bytes Big-Endian，最大 65535 bytes）
- * - Metadata：以 CBOR 或 JSON 編碼的 metadata 物件（id, createdAt, attempts 等）
- * - Payload：原始 Uint8Array 資料（job.data）
- *
- * @module BinaryJobFrame
- */
-/** Magic bytes，標識 Gravito Job 二進制幀 */
-export declare const MAGIC: readonly [71, 74];
-/** 幀格式版本號 */
-export declare const VERSION = 1;
-/** 幀 header 的固定大小（Magic 2B + Version 1B + Flags 1B + MetaLen 2B = 6 bytes） */
-export declare const HEADER_SIZE = 6;
-/** Flags 位元定義 */
-export declare const Flags: {
-    /** 包含 className 欄位 */
-    readonly HAS_CLASS_NAME: 1;
-    /** 包含 groupId 欄位 */
-    readonly HAS_GROUP_ID: 2;
-    /** 包含 delaySeconds 欄位 */
-    readonly HAS_DELAY: 4;
-    /** 包含 priority 欄位 */
-    readonly HAS_PRIORITY: 8;
-    /** 包含 error / failedAt 欄位 */
-    readonly HAS_ERROR: 16;
-};
-/**
- * Metadata 短鍵映射（節省序列化空間）
- * 完整鍵名 → 短鍵名
- */
-export declare const META_KEY_MAP: Record<string, string>;
-/**
- * 短鍵反向映射（反序列化用）
- * 短鍵名 → 完整鍵名
- */
-export declare const META_KEY_REVERSE_MAP: Record<string, string>;
-/**
- * 判斷給定的資料是否為 Gravito Job 二進制幀
- *
- * 透過 Magic byte 嗅探識別：檢查前 2 bytes 是否為 [0x47, 0x4A]
- *
- * @param data - 要檢查的資料（Uint8Array 或 Buffer）
- * @returns 如果是 Gravito Job 幀則回傳 true
- */
-export declare function isGravitoJobFrame(data: Uint8Array | Buffer): boolean;
-/**
- * 將 SerializedJob 編碼為二進制幀格式
- *
- * 僅適用於 type === 'binary' 且 data 為 Uint8Array 的 Job。
- *
- * @param job - 要編碼的 SerializedJob（必須是 binary type）
- * @returns 編碼後的 Uint8Array
- * @throws {TypeError} 如果 job.data 不是 Uint8Array
- * @throws {Error} 如果 metadata 長度超過 65535 bytes
- */
-export declare function encodeBinaryJobFrame(job: SerializedJob): Uint8Array;
-/**
- * 將二進制幀解碼為 SerializedJob
- *
- * @param data - 要解碼的二進制幀（Uint8Array 或 Buffer）
- * @returns 解碼後的 SerializedJob，data 為 Uint8Array
- * @throws {Error} 如果資料格式無效
- */
-export declare function decodeBinaryJobFrame(data: Uint8Array | Buffer): SerializedJob;

package/dist/drivers/BullMQDriver.d.ts

@@ -1,237 +0,0 @@
-import type { JobPushOptions, QueueStats, SerializedJob, TopicOptions } from '../types';
-import type { QueueDriver } from './QueueDriver';
-/**
- * Bull Queue client interface (compatible with bullmq package).
- */
-export interface BullQueueClient {
-    add(name: string, data: BullJobData, options?: BullJobOptions): Promise<unknown>;
-    getJob(id: string): Promise<BullStoredJob | null>;
-    count(): Promise<number>;
-    process(handler: (job: BullStoredJob) => Promise<void>): void;
-    on(event: string, handler: (...args: unknown[]) => void): void;
-    off(event: string, handler: (...args: unknown[]) => void): void;
-    pause(): Promise<void>;
-    resume(): Promise<void>;
-    clean(grace: number, limit?: number, type?: string): Promise<number>;
-    close(): Promise<void>;
-    getJobCounts(types?: string[]): Promise<Record<string, number>>;
-    getDelayedCount(): Promise<number>;
-    getFailedCount(): Promise<number>;
-    getActiveCount(): Promise<number>;
-    [key: string]: unknown;
-}
-export interface BullWorkerClient {
-    [key: string]: unknown;
-}
-interface BullJobOptions {
-    priority?: number;
-    delay?: number;
-    attempts?: number;
-    backoff?: {
-        type: 'exponential';
-        delay: number;
-    };
-    group?: {
-        id: string;
-    };
-}
-interface BullJobData {
-    id: string;
-    type: SerializedJob['type'];
-    data: SerializedJob['data'];
-    className?: string;
-    createdAt: number;
-    delaySeconds?: number;
-    attempts: number;
-    maxAttempts: number;
-    groupId?: string;
-    retryAfterSeconds?: number;
-    retryMultiplier?: number;
-    error?: string;
-    failedAt?: number;
-    priority?: string | number;
-}
-interface BullStoredJob {
-    moveToFailed?(error: Error, token: boolean): Promise<void>;
-}
-interface WorkerHeartbeatPayload {
-    id: string;
-    status: string;
-    hostname: string;
-    pid: number;
-    uptime: number;
-    last_ping: string;
-    queues: string[];
-    metrics?: Record<string, unknown>;
-    [key: string]: unknown;
-}
-interface WorkerLogPayload {
-    level: string;
-    message: string;
-    workerId: string;
-    jobId?: string;
-    timestamp: string;
-    [key: string]: unknown;
-}
-/**
- * Bull Queue driver configuration.
- */
-export interface BullMQDriverConfig {
-    /**
-     * Bull Queue instance (from bullmq package).
-     */
-    queue: BullQueueClient;
-    /**
-     * Optional Bull Worker instance for processing jobs.
-     */
-    worker?: BullWorkerClient;
-    /**
-     * Connection options (host, port, etc. for Redis).
-     */
-    connection?: {
-        host?: string;
-        port?: number;
-        password?: string;
-        db?: number;
-        [key: string]: unknown;
-    };
-    /**
-     * Default number of concurrent workers (default: 1).
-     */
-    concurrency?: number;
-    /**
-     * Key prefix for namespacing queues (default: 'gravito:').
-     */
-    prefix?: string;
-    /**
-     * Enable debug logging.
-     */
-    debug?: boolean;
-}
-/**
- * Bull Queue driver implementation.
- *
- * Provides high-performance, persistent job queuing using Bull Queue (backed by Redis).
- * Supports priority, delays, retries, and group-based FIFO processing.
- *
- * @public
- * @example
- * ```typescript
- * import { Queue } from 'bullmq'
- * import Redis from 'ioredis'
- *
- * const redis = new Redis()
- * const queue = new Queue('gravito-events', { connection: redis })
- * const driver = new BullMQDriver({ queue })
- * ```
- */
-export declare class BullMQDriver implements QueueDriver {
-    private queue;
-    private prefix;
-    private debug;
-    private queueMap;
-    constructor(config: BullMQDriverConfig);
-    /**
-     * Get or create a queue for the given queue name.
-     */
-    private getQueue;
-    /**
-     * Build Job Options from JobPushOptions.
-     */
-    private buildJobOptions;
-    /**
-     * Create Bull job data from SerializedJob.
-     */
-    private createBullJobData;
-    /**
-     * Pushes a job to Bull Queue.
-     */
-    push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
-    /**
-     * Pops a job from Bull Queue.
-     * Note: Bull Queue typically uses Workers, not manual pop.
-     * This is a fallback implementation.
-     */
-    pop(queue: string): Promise<SerializedJob | null>;
-    /**
-     * Returns the size of the queue.
-     */
-    size(queue: string): Promise<number>;
-    /**
-     * Clears the queue.
-     */
-    clear(queue: string): Promise<void>;
-    /**
-     * Marks a job as failed (moves to failed list).
-     */
-    fail(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Returns detailed statistics for the queue.
-     */
-    stats(queue: string): Promise<QueueStats>;
-    /**
-     * Retrieves failed jobs from the Dead Letter Queue.
-     */
-    getFailed(queue: string, _start?: number, _end?: number): Promise<SerializedJob[]>;
-    /**
-     * Retries failed jobs.
-     */
-    retryFailed(queue: string, _count?: number): Promise<number>;
-    /**
-     * Clears the Dead Letter Queue.
-     */
-    clearFailed(queue: string): Promise<void>;
-    /**
-     * Creates a new queue/topic.
-     */
-    createTopic(_topic: string, _options?: TopicOptions): Promise<void>;
-    /**
-     * Deletes a queue/topic.
-     */
-    deleteTopic(topic: string): Promise<void>;
-    /**
-     * Pushes multiple jobs in batch.
-     */
-    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
-    /**
-     * Pops multiple jobs in batch.
-     */
-    popMany(_queue: string, _count: number): Promise<SerializedJob[]>;
-    /**
-     * Reports worker heartbeat.
-     */
-    reportHeartbeat(workerInfo: {
-        id: string;
-        status: string;
-        hostname: string;
-        pid: number;
-        uptime: number;
-        last_ping: string;
-        queues: string[];
-        metrics?: Record<string, unknown>;
-        [key: string]: unknown;
-    } & WorkerHeartbeatPayload, _prefix?: string): Promise<void>;
-    /**
-     * Publishes a log message.
-     */
-    publishLog(logPayload: {
-        level: string;
-        message: string;
-        workerId: string;
-        jobId?: string;
-        timestamp: string;
-        [key: string]: unknown;
-    } & WorkerLogPayload, _prefix?: string): Promise<void>;
-    /**
-     * Checks rate limit for a queue.
-     */
-    checkRateLimit(_queue: string, _config: {
-        max: number;
-        duration: number;
-    }): Promise<boolean>;
-    /**
-     * Retrieves all queue names.
-     */
-    getQueues(): Promise<string[]>;
-}
-export {};