@gravito/stream 2.0.1 → 2.1.0

package/dist/StreamEventBackend.d.ts

@@ -0,0 +1,114 @@
+import type { EventBackend, EventTask } from '@gravito/core';
+import type { QueueManager } from './QueueManager';
+/**
+ * Retry strategy for events.
+ *
+ * - 'bull': Use Bull Queue's built-in retry mechanism
+ * - 'core': Use EventPriorityQueue's retry logic (not implemented in Stream backend)
+ * - 'hybrid': Bull retries + Core DLQ fallback (future phase)
+ */
+export type RetryStrategy = 'bull' | 'core' | 'hybrid';
+/**
+ * Configuration for StreamEventBackend.
+ */
+export interface StreamEventBackendConfig {
+    /**
+     * Retry strategy to use.
+     * @default 'bull'
+     */
+    retryStrategy?: RetryStrategy;
+    /**
+     * Whether to integrate with CircuitBreaker.
+     * @default false
+     */
+    circuitBreakerIntegration?: boolean;
+    /**
+     * Optional DLQ (Dead Letter Queue) handler for failed events.
+     */
+    dlqHandler?: {
+        handle(event: EventTask, error: Error, attempt: number): Promise<void>;
+    };
+    /**
+     * CircuitBreaker getter (injected from core)
+     */
+    getCircuitBreaker?: (hook: string) => any;
+}
+/**
+ * Event backend implementation using Gravito Stream (Bull Queue).
+ *
+ * Provides persistent, distributed event processing with:
+ * - Bull Queue persistence (Redis-backed)
+ * - Configurable retry strategies
+ * - Optional CircuitBreaker integration
+ * - DLQ support for failed events
+ *
+ * @example
+ * ```typescript
+ * const queueManager = new QueueManager({
+ *   default: 'bullmq',
+ *   connections: {
+ *     bullmq: {
+ *       driver: 'bullmq',
+ *       queue: new Queue('gravito-events', { connection: redis })
+ *     }
+ *   }
+ * })
+ *
+ * const backend = new StreamEventBackend(queueManager, {
+ *   retryStrategy: 'bull',
+ *   circuitBreakerIntegration: true
+ * })
+ * ```
+ */
+export declare class StreamEventBackend implements EventBackend {
+    private queueManager;
+    private config;
+    constructor(queueManager: QueueManager, config?: StreamEventBackendConfig);
+    /**
+     * Build Job Push Options from EventOptions.
+     *
+     * Maps EventOptions to Bull Queue JobPushOptions with retry strategy applied.
+     */
+    private buildJobOptions;
+    /**
+     * Enqueue an event task to the stream queue.
+     *
+     * Applies retry strategy and CircuitBreaker checks based on configuration.
+     * Supports DLQ routing for failed events.
+     */
+    enqueue(task: EventTask): Promise<void>;
+    /**
+     * Apply retry strategy to the job based on configuration.
+     */
+    private applyRetryStrategy;
+    /**
+     * Handle job failure and route to DLQ if configured.
+     *
+     * Called when a job exhausts all retry attempts.
+     */
+    handleJobFailure(task: EventTask, error: Error, attempt: number): Promise<void>;
+    /**
+     * Record a job failure for CircuitBreaker state management.
+     *
+     * Called when a job fails, regardless of retry status.
+     */
+    recordJobFailure(task: EventTask, error: Error): void;
+    /**
+     * Record a job success for CircuitBreaker state management.
+     *
+     * Called when a job completes successfully.
+     */
+    recordJobSuccess(task: EventTask): void;
+    /**
+     * Get the retry strategy configuration.
+     */
+    getRetryStrategy(): RetryStrategy;
+    /**
+     * Check if CircuitBreaker integration is enabled.
+     */
+    isCircuitBreakerEnabled(): boolean;
+    /**
+     * Get the DLQ handler, if configured.
+     */
+    getDLQHandler(): StreamEventBackendConfig['dlqHandler'] | undefined;
+}

package/dist/SystemEventJob.d.ts

@@ -0,0 +1,33 @@
+import { Job } from './Job';
+/**
+ * SystemEventJob - Internal job for processing Gravito async hooks.
+ *
+ * @internal
+ */
+export declare class SystemEventJob extends Job {
+    readonly hook: string;
+    readonly args: unknown;
+    readonly options: Record<string, any>;
+    /**
+     * Optional failure callback for DLQ handling.
+     */
+    private onFailedCallback?;
+    constructor(hook: string, args: unknown, options?: Record<string, any>);
+    /**
+     * Set failure callback for DLQ handling.
+     *
+     * @param callback - Called when job fails permanently
+     * @returns Self for chaining
+     */
+    onFailed(callback: (error: Error, attempt: number) => Promise<void>): this;
+    /**
+     * Execute the hook listeners in the worker process.
+     */
+    handle(): Promise<void>;
+    /**
+     * Called when job fails permanently after all retries.
+     *
+     * This method is invoked by the worker when job exhausts all retry attempts.
+     */
+    failed(error: Error, attempt?: number): Promise<void>;
+}

package/dist/Worker.d.ts

@@ -0,0 +1,139 @@
+import type { Job } from './Job';
+import { type SandboxedWorkerConfig } from './workers/SandboxedWorker';
+/**
+ * Configuration options for the Worker.
+ *
+ * Controls the execution behavior of jobs, including retry limits and timeouts.
+ *
+ * @example
+ * ```typescript
+ * const options: WorkerOptions = {
+ *   maxAttempts: 3,
+ *   timeout: 30
+ * };
+ * ```
+ */
+export interface WorkerOptions {
+    /**
+     * The maximum number of attempts for a job before it is marked as failed.
+     *
+     * This value serves as a default fallback if the job itself does not specify `maxAttempts`.
+     */
+    maxAttempts?: number;
+    /**
+     * The maximum execution time for a job in seconds.
+     *
+     * If the job exceeds this duration, it will be timed out and marked as failed.
+     */
+    timeout?: number;
+    /**
+     * Callback function triggered when a job permanently fails.
+     *
+     * This allows for custom error reporting or cleanup logic outside of the job class.
+     */
+    onFailed?: (job: Job, error: Error) => Promise<void>;
+    /**
+     * Enable sandboxed execution using Worker Threads.
+     *
+     * When enabled, jobs are executed in isolated Worker Threads, providing:
+     * - Context isolation: Each job runs in a separate execution environment
+     * - Crash protection: Worker crashes don't affect the main thread
+     * - Memory limits: Prevent memory leaks from affecting the main process
+     * - Timeout enforcement: Jobs exceeding the timeout are forcefully terminated
+     *
+     * @default false
+     */
+    sandboxed?: boolean;
+    /**
+     * Sandboxed worker configuration options.
+     *
+     * Only used when `sandboxed` is true.
+     */
+    sandboxConfig?: SandboxedWorkerConfig;
+}
+/**
+ * Executes background jobs.
+ *
+ * The Worker is responsible for running the `handle()` method of a job, managing its lifecycle,
+ * enforcing timeouts, and handling retries or failures.
+ *
+ * Supports two execution modes:
+ * - **Standard Mode** (default): Executes jobs directly in the current process
+ * - **Sandboxed Mode**: Executes jobs in isolated Worker Threads for enhanced security and stability
+ *
+ * @example
+ * ```typescript
+ * // Standard mode
+ * const worker = new Worker({
+ *   maxAttempts: 3,
+ *   timeout: 60
+ * });
+ *
+ * // Sandboxed mode
+ * const sandboxedWorker = new Worker({
+ *   maxAttempts: 3,
+ *   timeout: 60,
+ *   sandboxed: true,
+ *   sandboxConfig: {
+ *     maxExecutionTime: 30000,
+ *     maxMemory: 512,
+ *     isolateContexts: true
+ *   }
+ * });
+ *
+ * await worker.process(job);
+ * ```
+ */
+export declare class Worker {
+    private options;
+    private sandboxedWorker?;
+    constructor(options?: WorkerOptions);
+    /**
+     * Processes a single job instance.
+     *
+     * 1. Checks attempt counts.
+     * 2. Enforces execution timeout (if configured).
+     * 3. Runs `job.handle()` (either directly or in a sandboxed Worker Thread).
+     * 4. Catches errors and invokes failure handlers if max attempts are reached.
+     *
+     * @param job - The job to process.
+     * @throws {Error} If the job execution fails (to trigger retry logic in the consumer).
+     */
+    process(job: Job): Promise<void>;
+    /**
+     * Processes a job in standard mode (directly in current process).
+     *
+     * @param job - The job to process.
+     * @param timeout - Optional timeout in seconds.
+     */
+    private processStandard;
+    /**
+     * Processes a job in sandboxed mode (in Worker Thread).
+     *
+     * @param job - The job to process.
+     */
+    private processSandboxed;
+    /**
+     * Serializes a Job instance for Worker Thread execution.
+     *
+     * @param job - The job to serialize.
+     * @returns Serialized job data.
+     */
+    private serializeJob;
+    /**
+     * Handles the permanent failure of a job.
+     *
+     * Invokes the job's `failed()` method and any global `onFailed` callback.
+     *
+     * @param job - The failed job.
+     * @param error - The error that caused the failure.
+     */
+    private handleFailure;
+    /**
+     * Terminates the sandboxed worker and releases resources.
+     *
+     * Should be called when the worker is no longer needed.
+     * Only applicable when running in sandboxed mode.
+     */
+    terminate(): Promise<void>;
+}

package/dist/benchmarks/PerformanceReporter.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Performance Reporter
+ *
+ * Generates comprehensive performance reports comparing BunWorker and SandboxedWorker
+ */
+export interface BenchmarkResult {
+    name: string;
+    runtime: 'bun' | 'node';
+    metric: string;
+    value: number;
+    unit: string;
+    timestamp: number;
+}
+export interface PerformanceReport {
+    title: string;
+    timestamp: string;
+    environment: {
+        runtime: string;
+        platform: string;
+        nodeVersion?: string;
+        bunVersion?: string;
+    };
+    results: BenchmarkResult[];
+    comparisons: PerformanceComparison[];
+    summary: string;
+}
+export interface PerformanceComparison {
+    metric: string;
+    bun: {
+        value: number;
+        unit: string;
+    };
+    node: {
+        value: number;
+        unit: string;
+    };
+    improvement: {
+        percentage: number;
+        direction: 'faster' | 'slower' | 'equal';
+    };
+}
+/**
+ * Performance Reporter for generating benchmark reports
+ */
+export declare class PerformanceReporter {
+    private results;
+    /**
+     * Record a benchmark result
+     */
+    recordResult(result: BenchmarkResult): void;
+    /**
+     * Record multiple results
+     */
+    recordResults(results: BenchmarkResult[]): void;
+    /**
+     * Generate a performance comparison report
+     */
+    generateReport(): PerformanceReport;
+    /**
+     * Compare results between Bun and Node.js
+     */
+    private compareResults;
+    /**
+     * Calculate improvement percentage
+     */
+    private calculateImprovement;
+    /**
+     * Get environment information
+     */
+    private getEnvironmentInfo;
+    /**
+     * Generate summary text
+     */
+    private generateSummary;
+    /**
+     * Generate markdown report
+     */
+    generateMarkdown(): string;
+    /**
+     * Generate JSON report
+     */
+    generateJSON(): string;
+    /**
+     * Export results to CSV format
+     */
+    generateCSV(): string;
+    /**
+     * Clear recorded results
+     */
+    clear(): void;
+    /**
+     * Get recorded results
+     */
+    getResults(): BenchmarkResult[];
+}
+/**
+ * Helper function to create a performance reporter with sample data
+ */
+export declare function createSampleReport(): PerformanceReport;

package/dist/consumer/ConcurrencyGate.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Promise-based Semaphore，用於控制並發執行數量。
+ *
+ * 完全取代原始 Consumer 中的 `stats.active` + `setTimeout(50)` busy-wait 模式。
+ * 使用 Promise 等待機制，當並發數達到上限時，後續的 acquire() 會被掛起，
+ * 直到有其他任務 release() 後才會被喚醒，不會浪費 CPU 資源。
+ *
+ * @example
+ * ```typescript
+ * const gate = new ConcurrencyGate(5)
+ *
+ * await gate.acquire()
+ * try {
+ *   await doWork()
+ * } finally {
+ *   gate.release()
+ * }
+ * ```
+ */
+export declare class ConcurrencyGate {
+    private readonly limit;
+    private active;
+    /** 等待中的 resolve 函數佇列（FIFO） */
+    private waiters;
+    constructor(limit: number);
+    /**
+     * 取得並發許可。
+     *
+     * 若目前 active < limit，立即遞增 active 並回傳。
+     * 否則等待直到有 release() 釋放許可後再獲取。
+     */
+    acquire(): Promise<void>;
+    /**
+     * 釋放並發許可。
+     *
+     * 遞減 active 計數，並喚醒下一個等待者（FIFO 順序）。
+     */
+    release(): void;
+    /**
+     * 目前可立即取得的許可數量。
+     */
+    get available(): number;
+    /**
+     * 目前正在執行的任務數量。
+     */
+    get activeCount(): number;
+    /**
+     * 目前等待中的任務數量。
+     */
+    get waitingCount(): number;
+    /**
+     * 最大並發數量上限。
+     */
+    get concurrencyLimit(): number;
+}

package/dist/consumer/ConsumerStrategy.d.ts

@@ -0,0 +1,41 @@
+import type { Job } from '../Job';
+/**
+ * ConsumerStrategy defines the contract for different job consumption modes.
+ *
+ * Implementations handle the scheduling and fetching of jobs from the queue.
+ * This allows pluggable consumption strategies: polling, reactive, hybrid, etc.
+ *
+ * @public
+ */
+export interface ConsumerStrategy {
+    /**
+     * Starts the consumption strategy.
+     *
+     * Initiates the job fetching loop based on the specific strategy implementation.
+     *
+     * @returns A promise that resolves when the strategy loop has started.
+     */
+    start(): Promise<void>;
+    /**
+     * Stops the consumption strategy gracefully.
+     *
+     * Signals the strategy to stop accepting new jobs and clean up resources.
+     *
+     * @returns A promise that resolves when the strategy has fully stopped.
+     */
+    stop(): Promise<void>;
+    /**
+     * Checks if the strategy is currently active.
+     *
+     * @returns True if the strategy loop is running.
+     */
+    isRunning(): boolean;
+    /**
+     * Called by the consumer loop to fetch the next batch of jobs.
+     *
+     * The strategy determines how to fetch jobs (blocking, non-blocking, reactive, etc.).
+     *
+     * @returns An array of jobs to process, or empty array if no jobs are available.
+     */
+    fetchJobs(): Promise<Job[]>;
+}

package/dist/consumer/GroupSequencer.d.ts

@@ -0,0 +1,57 @@
+/**
+ * 管理每個 groupId 的 FIFO 序列執行器。
+ *
+ * 確保相同 groupId 的 job 按照 FIFO 順序依序執行，
+ * 不同 groupId 的 job 則可以並行執行。
+ *
+ * 使用 pLimit(1) 為每個 group 建立容量為 1 的限制器，
+ * 確保同一 group 同時只有一個 job 在執行。
+ *
+ * @example
+ * ```typescript
+ * const sequencer = new GroupSequencer()
+ *
+ * // group1 的 job 依序執行
+ * sequencer.run('group1', async () => await processJob(jobA))
+ * sequencer.run('group1', async () => await processJob(jobB))
+ *
+ * // group2 與 group1 並行
+ * sequencer.run('group2', async () => await processJob(jobC))
+ *
+ * // 無 group 的 job 直接執行
+ * sequencer.run(undefined, async () => await processJob(jobD))
+ * ```
+ */
+export declare class GroupSequencer {
+    /** Group ID 到 pLimit limiter 的映射 */
+    private limiters;
+    /** Group ID 到最後使用時間（ms）的映射，用於 TTL 清理 */
+    private lastUsed;
+    /** 定期清理計時器 */
+    private cleanupTimer;
+    /** Group limiter 存活時間（毫秒），超過且無任務時清理 */
+    private static readonly TTL_MS;
+    constructor();
+    /**
+     * 執行一個任務，確保相同 group 內的任務依序執行。
+     *
+     * @param groupId - 群組 ID，undefined 表示無群組（直接執行）
+     * @param fn - 要執行的非同步任務
+     * @returns 任務的 Promise
+     */
+    run<T>(groupId: string | undefined, fn: () => Promise<T>): Promise<T>;
+    /**
+     * 清理超過 TTL 且沒有 active/pending 任務的 group limiters。
+     *
+     * 定期呼叫以避免記憶體洩漏。
+     */
+    cleanup(): void;
+    /**
+     * 停止時完全清理所有 limiters 和計時器。
+     */
+    destroy(): void;
+    /**
+     * 目前管理中的 group 數量（用於診斷）。
+     */
+    get groupCount(): number;
+}

package/dist/consumer/HeartbeatManager.d.ts

@@ -0,0 +1,65 @@
+import type { QueueManager } from '../QueueManager';
+import type { ConsumerStats } from './types';
+/**
+ * HeartbeatManager 設定選項。
+ */
+export interface HeartbeatManagerOptions {
+    /** Worker ID */
+    workerId: string;
+    /** 要監控的佇列列表 */
+    queues: string[];
+    /** 連線名稱 */
+    connectionName: string;
+    /** 心跳間隔（毫秒），預設 5000 */
+    interval?: number;
+    /** 額外資訊，包含在心跳 payload 中 */
+    extraInfo?: Record<string, unknown>;
+    /** 監控 key prefix */
+    prefix?: string;
+}
+/**
+ * 管理 Consumer 的心跳與監控日誌。
+ *
+ * 定時向 driver 發送心跳信號（reportHeartbeat），
+ * 並在支援時發送監控日誌（publishLog）。
+ * 所有異常均被靜默捕獲，不影響主流程。
+ *
+ * @example
+ * ```typescript
+ * const heartbeat = new HeartbeatManager(queueManager, options, () => stats)
+ * heartbeat.start()
+ * // ... consumer running ...
+ * heartbeat.stop()
+ * ```
+ */
+export declare class HeartbeatManager {
+    private readonly queueManager;
+    private readonly options;
+    /** getter 函數，每次心跳時取得最新統計 */
+    private readonly getStats;
+    private timer;
+    private readonly intervalMs;
+    constructor(queueManager: QueueManager, options: HeartbeatManagerOptions, 
+    /** getter 函數，每次心跳時取得最新統計 */
+    getStats: () => ConsumerStats);
+    /**
+     * 啟動心跳定時器。
+     */
+    start(): void;
+    /**
+     * 停止心跳定時器。
+     */
+    stop(): void;
+    /**
+     * 發送一次心跳。
+     */
+    sendHeartbeat(): Promise<void>;
+    /**
+     * 發送監控日誌。
+     *
+     * @param level - 日誌等級（info, success, warning, error）
+     * @param message - 日誌訊息
+     * @param jobId - 相關的 job ID（可選）
+     */
+    publishLog(level: string, message: string, jobId?: string): Promise<void>;
+}

package/dist/consumer/JobExecutor.d.ts

@@ -0,0 +1,61 @@
+import type { EventEmitter } from 'node:events';
+import type { Job } from '../Job';
+import type { QueueManager } from '../QueueManager';
+import type { Worker } from '../Worker';
+import type { HeartbeatManager } from './HeartbeatManager';
+import type { ConsumerStats, ExecutionResult, ExecutorOptions } from './types';
+/**
+ * JobExecutor 負責完整的 Job 執行生命週期。
+ *
+ * 從原始 Consumer.handleJob() 提取，封裝以下邏輯：
+ * - 發射生命週期事件（job:started, job:processed, job:failed 等）
+ * - 呼叫 worker.process(job)
+ * - 成功後呼叫 queueManager.complete()
+ * - 失敗後執行重試邏輯（指數退避 + re-push）
+ * - 超出 maxAttempts 後發射 job:failed_permanently 並移入 DLQ
+ * - maxRequests 達到時設定 shouldStop = true
+ *
+ * 所有異常均被捕獲，不拋出（確保管線繼續執行）。
+ *
+ * @example
+ * ```typescript
+ * const executor = new JobExecutor(queueManager, emitter, stats, heartbeat, options)
+ * const result = await executor.execute(job, worker)
+ *
+ * if (result.shouldStop) {
+ *   // 停止 consumer
+ * }
+ * ```
+ */
+export declare class JobExecutor {
+    private readonly queueManager;
+    private readonly emitter;
+    private readonly stats;
+    private readonly heartbeat;
+    private readonly options;
+    constructor(queueManager: QueueManager, emitter: EventEmitter, stats: ConsumerStats, heartbeat: HeartbeatManager | null, options: ExecutorOptions);
+    /**
+     * 執行單一 Job 的完整生命週期。
+     *
+     * @param job - 要執行的 Job
+     * @param worker - Worker 實例
+     * @returns ExecutionResult，包含成功/失敗狀態、耗時、shouldStop 旗標
+     */
+    execute(job: Job, worker: Worker): Promise<ExecutionResult>;
+    /**
+     * 安排 job 重試，使用指數退避策略。
+     */
+    private scheduleRetry;
+    /**
+     * 處理永久失敗的 job，移入 DLQ 並發射事件。
+     */
+    private handlePermanentFailure;
+    /**
+     * 檢查是否達到 maxRequests，若達到則發射事件並回傳 shouldStop=true。
+     */
+    private checkMaxRequests;
+    /**
+     * 輸出 debug 日誌（僅在 debug=true 時）。
+     */
+    private log;
+}

package/dist/consumer/JobSourceGenerator.d.ts

@@ -0,0 +1,31 @@
+import type { QueueManager } from '../QueueManager';
+import type { FetchResult, JobSourceOptions, StopSignal } from './types';
+/**
+ * Async Generator，用於從佇列持續抓取 Job。
+ *
+ * 封裝所有抓取策略（blocking/batch/sequential），自適應 backoff，
+ * 以及 rate limit 過濾邏輯。每次 yield 一個 FetchResult，
+ * 包含本輪抓取的 job 列表和是否使用阻塞模式的資訊。
+ *
+ * 設計原則：
+ * - blocking pop 已在 driver 層等待，不再額外 backoff
+ * - 自適應退避：空佇列時指數增長，有 job 時重設
+ * - rate limit：每輪抓取前過濾掉受限的佇列
+ * - 中止信號：每次 yield 前檢查 signal.aborted
+ *
+ * @param queueManager - QueueManager 實例
+ * @param options - 抓取策略選項
+ * @param signal - 中止信號（AbortController.signal）
+ *
+ * @example
+ * ```typescript
+ * const source = jobSourceGenerator(queueManager, options, controller.signal)
+ *
+ * for await (const result of source) {
+ *   for (const job of result.jobs) {
+ *     await processJob(job)
+ *   }
+ * }
+ * ```
+ */
+export declare function jobSourceGenerator(queueManager: QueueManager, options: JobSourceOptions, signal: StopSignal): AsyncGenerator<FetchResult>;