@gravito/stream 2.0.2 → 2.1.0

package/README.md

@@ -11,15 +11,33 @@
 
 ## ✨ Features
 
-- 🪐 **Orbit Integration** - Native integration with PlanetCore micro-kernel and dependency injection.
+- 🪐 **Galaxy-Ready Orbit** - Native integration with PlanetCore micro-kernel and dependency injection.
 - 🔌 **Multi-Broker Support** - Built-in drivers for **Redis**, **SQS**, **Kafka**, **RabbitMQ**, **Database** (SQL), and **Memory**.
 - 🛠️ **Job-Based API** - Clean, class-based job definitions with built-in serialization and failure handling.
 - 🚀 **High Throughput** - Optimized for **Bun**, supporting batch consumption, concurrent processing, and adaptive polling.
+- 📡 **Cross-Satellite Event Streaming** - Enable loosely coupled communication between isolated Satellites.
 - 🛡️ **Reliability** - Built-in exponential backoff retries, Dead Letter Queues (DLQ), and sequential job grouping.
 - 📝 **Audit & Persistence** - Optional SQL-based persistence layer for archiving job history and providing complete audit trails.
 - 🕒 **Scheduler** - Built-in CRON-based task scheduling for recurring jobs.
 - 🏢 **Worker Modes** - Run embedded workers during development or standalone worker processes in production.
 
+## 🌌 Role in Galaxy Architecture
+
+In the **Gravito Galaxy Architecture**, Stream acts as the **Async Engine (Background Processing)**.
+
+- **Satellite Decoupling**: Instead of Satellite A calling Satellite B synchronously and waiting for a response, Satellite A fires an event/job into Stream, and Satellite B consumes it asynchronously.
+- **Resilience Backbone**: Works hand-in-hand with `@gravito/resilience` to ensure that failed web requests or heavy tasks are retried in the background without affecting the user experience.
+- **Distributed State**: Facilitates event-driven architecture (EDA), allowing multiple Satellites to react to the same domain event reliably.
+
+```mermaid
+graph TD
+    SatA[Satellite: Order] -- "Push Job" --> Stream{Stream Orbit}
+    Stream -- "Queue: emails" --> Worker1[Worker Pool]
+    Worker1 --> SatB[Satellite: Notification]
+    Stream -- "Queue: payments" --> Worker2[Worker Pool]
+    Worker2 --> SatC[Satellite: Finance]
+```
+
 ## 📦 Installation
 
 ```bash
@@ -148,6 +166,14 @@ Accessed via `c.get('queue')` or `core.container.make('queue')`.
 - **`backoff(seconds, multiplier?)`**: Configure retry strategy.
 - **`withPriority(priority)`**: Set job priority.
 
+## 📚 Documentation
+
+Detailed guides and references for the Galaxy Architecture:
+
+- [🏗️ **Architecture Overview**](./README.md) — Multi-broker queue and job system.
+- [📡 **Event-Driven Architecture**](./doc/EVENT_DRIVEN_ARCHITECTURE.md) — **NEW**: Cross-Satellite communication and pub/sub.
+- [⚙️ **Worker Configuration**](#-worker-modes) — Embedded vs standalone background workers.
+
 ## 🔌 Supported Drivers
 
 - **Redis** - Feature-rich (DLQ, Rate limiting, Priorities).

package/dist/BatchConsumer.d.ts

@@ -0,0 +1,81 @@
+import type { Job } from './Job';
+import type { QueueManager } from './QueueManager';
+/**
+ * Configuration options for the BatchConsumer.
+ *
+ * @example
+ * ```typescript
+ * const options: BatchConsumerOptions = {
+ *   batchSize: 50,
+ *   autoAck: false
+ * };
+ * ```
+ */
+export interface BatchConsumerOptions {
+    /**
+     * The name of the queue to consume from.
+     * @default 'default'
+     */
+    queue?: string;
+    /**
+     * The connection name to use.
+     * @default The default connection of QueueManager
+     */
+    connection?: string;
+    /**
+     * The number of jobs to try to retrieve in each batch.
+     * @default 10
+     */
+    batchSize?: number;
+    /**
+     * The polling interval in milliseconds when the queue is empty.
+     * @default 1000
+     */
+    pollInterval?: number;
+    /**
+     * Whether to automatically complete jobs after the handler returns successfully.
+     *
+     * If set to `false`, the handler function is responsible for calling `manager.complete()`
+     * or `manager.fail()` for each job.
+     *
+     * @default true
+     */
+    autoAck?: boolean;
+}
+/**
+ * Specialized consumer for processing jobs in bulk.
+ *
+ * Unlike the standard `Consumer` which processes jobs individually (even if fetched in batches),
+ * the `BatchConsumer` passes an array of jobs to a single handler function. This is ideal for
+ * operations that benefit from bulk processing, such as database inserts or API calls that support batching.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * const consumer = new BatchConsumer(manager, async (jobs) => {
+ *   // Process 100 jobs at once
+ *   await elasticsearch.bulkIndex(jobs.map(j => j.data));
+ * }, { batchSize: 100 });
+ *
+ * consumer.start();
+ * ```
+ */
+export declare class BatchConsumer {
+    private manager;
+    private handler;
+    private running;
+    private options;
+    constructor(manager: QueueManager, handler: (jobs: Job[]) => Promise<void>, options?: BatchConsumerOptions);
+    /**
+     * Starts the batch consuming loop.
+     *
+     * Continuously polls for batches of jobs and passes them to the handler.
+     */
+    start(): Promise<void>;
+    /**
+     * Stops the consumer loop.
+     *
+     * Sets the running flag to false. The loop will exit after the current iteration finishes.
+     */
+    stop(): void;
+}

package/dist/Consumer.d.ts

@@ -0,0 +1,215 @@
+import { EventEmitter } from 'node:events';
+import type { ConsumerStats } from './consumer/types';
+import type { QueueManager } from './QueueManager';
+import type { WorkerOptions } from './Worker';
+/**
+ * Configuration options for the Consumer.
+ *
+ * Defines which queues to listen to, connection settings, concurrency levels,
+ * and advanced behavior like rate limiting and batch processing.
+ *
+ * @example
+ * ```typescript
+ * const options: ConsumerOptions = {
+ *   queues: ['emails', 'notifications'],
+ *   concurrency: 5,
+ *   pollInterval: 2000
+ * };
+ * ```
+ */
+export interface ConsumerOptions {
+    /**
+     * List of queue names to consume jobs from.
+     *
+     * The consumer will poll these queues in the order provided or based on driver logic.
+     */
+    queues: string[];
+    /**
+     * The connection name to use (e.g., 'redis', 'sqs').
+     *
+     * If not provided, uses the default connection from QueueManager.
+     */
+    connection?: string;
+    /**
+     * Configuration options passed to the underlying Worker.
+     */
+    workerOptions?: WorkerOptions;
+    /**
+     * The interval in milliseconds to wait before polling again when the queue is empty.
+     */
+    pollInterval?: number;
+    /**
+     * Whether to keep the process alive when queues are empty.
+     *
+     * If false, the consumer will exit the loop when no jobs are found (useful for one-off scripts).
+     */
+    keepAlive?: boolean;
+    /**
+     * Monitoring configuration.
+     *
+     * Can be a boolean to enable default monitoring, or an object for advanced configuration.
+     */
+    monitor?: boolean | {
+        /**
+         * The interval in milliseconds for sending heartbeat updates.
+         * @default 5000
+         */
+        interval?: number;
+        /**
+         * Additional metadata to include in heartbeat payloads.
+         */
+        extraInfo?: Record<string, unknown>;
+        /**
+         * Key prefix for monitoring events (e.g. for Redis Pub/Sub).
+         */
+        prefix?: string;
+    };
+    /**
+     * Rate limiting configuration per queue.
+     *
+     * Defines the maximum number of jobs to process within a given duration.
+     *
+     * @example
+     * ```typescript
+     * { 'emails': { max: 10, duration: 1000 } } // 10 emails per second
+     * ```
+     */
+    rateLimits?: Record<string, {
+        max: number;
+        duration: number;
+    }>;
+    /**
+     * The maximum number of jobs to process concurrently.
+     *
+     * @default 1
+     */
+    concurrency?: number;
+    /**
+     * Whether to enforce sequential processing for jobs with the same `groupId`.
+     *
+     * If true, jobs sharing a `groupId` will be processed one after another,
+     * even if global concurrency is high.
+     *
+     * @default true
+     */
+    groupJobsSequential?: boolean;
+    /**
+     * The minimum polling interval in milliseconds for adaptive polling.
+     *
+     * @default 100
+     */
+    minPollInterval?: number;
+    /**
+     * The maximum polling interval in milliseconds for adaptive polling.
+     *
+     * @default 5000
+     */
+    maxPollInterval?: number;
+    /**
+     * The multiplier used to increase the polling interval when the queue is empty.
+     *
+     * @default 1.5
+     */
+    backoffMultiplier?: number;
+    /**
+     * The number of jobs to try to fetch in a single request.
+     *
+     * If supported by the driver, fetching multiple jobs reduces network round-trips.
+     *
+     * @default 1
+     */
+    batchSize?: number;
+    /**
+     * Whether to use blocking operations (like BLPOP in Redis) when polling.
+     *
+     * Significant optimization for low-latency job pickup. Only applies when `batchSize` is 1.
+     *
+     * @default true
+     */
+    useBlocking?: boolean;
+    /**
+     * The timeout in seconds for blocking operations.
+     *
+     * @default 5
+     */
+    blockingTimeout?: number;
+    /**
+     * Enable verbose debug logging for consumer activities.
+     *
+     * @default false
+     */
+    debug?: boolean;
+    /**
+     * 最大處理請求數量。
+     *
+     * 當 consumer 處理完這個數量的 job 後會自動停止（觸發 max_requests_reached 事件）。
+     * 適用於需要定期重啟 worker 的場景（避免記憶體累積、載入最新程式碼等）。
+     *
+     * @default undefined (無限制)
+     */
+    maxRequests?: number;
+    /**
+     * Optional event callback for external monitoring systems.
+     *
+     * Called whenever a job lifecycle event occurs (started, processed, failed, etc.).
+     */
+    onEvent?: (event: string, payload: unknown) => void;
+    /**
+     * Enable reactive (push-based) consumption mode.
+     *
+     * When enabled, the consumer listens for notifications and pulls jobs reactively
+     * instead of polling continuously. This reduces latency and resource usage.
+     *
+     * @default false
+     */
+    reactive?: boolean;
+    /**
+     * Fallback polling interval (ms) when no notifications are received.
+     *
+     * In reactive mode, if no notifications arrive for this duration,
+     * the consumer will fallback to polling to ensure jobs aren't missed.
+     * Prevents starvation if the notification system fails.
+     *
+     * @default 30000 (30 seconds)
+     */
+    reactivePollingFallback?: number;
+}
+/**
+ * Consumer 門面類別（Facade），提供向後相容的公開 API。
+ *
+ * 內部委派給 StreamingConsumer 實作，所有事件都通過 passthrough。
+ * 公開 API 與原始 Consumer 完全一致，不破壞現有使用者程式碼。
+ *
+ * @public
+ */
+export declare class Consumer extends EventEmitter {
+    readonly queueManager: QueueManager;
+    readonly options: ConsumerOptions;
+    /** 內部 StreamingConsumer 實例 */
+    private streaming;
+    constructor(queueManager: QueueManager, options: ConsumerOptions);
+    /**
+     * 將 StreamingConsumer 的所有事件轉發給 Consumer。
+     */
+    private forwardEvents;
+    /**
+     * Starts the consumer loop.
+     */
+    start(): Promise<void>;
+    /**
+     * Gracefully stops the consumer.
+     */
+    stop(): Promise<void>;
+    /**
+     * Checks if the consumer is currently active.
+     */
+    isRunning(): boolean;
+    /**
+     * Retrieves current operational statistics.
+     */
+    getStats(): ConsumerStats;
+    /**
+     * Resets the internal statistics counters.
+     */
+    resetStats(): void;
+}

package/dist/DashboardProvider.d.ts

@@ -0,0 +1,20 @@
+import type { QueueManager } from './QueueManager';
+/**
+ * Provides API routes for the Stream Monitoring Dashboard.
+ *
+ * This class encapsulates the logic for querying queue statistics,
+ * browsing archived jobs, and performing management actions like retrying jobs.
+ *
+ * @internal
+ */
+export declare class DashboardProvider {
+    private manager;
+    constructor(manager: QueueManager);
+    /**
+     * Registers dashboard API routes on the provided core adapter.
+     *
+     * @param core - The PlanetCore instance.
+     * @param basePath - The base path for API routes (default: '/_flux').
+     */
+    registerRoutes(core: any, basePath?: string): void;
+}

package/dist/Job.d.ts

@@ -0,0 +1,183 @@
+import type { Queueable } from './Queueable';
+/**
+ * Abstract base class for all background jobs.
+ *
+ * This class serves as the foundation for creating queueable tasks. It implements the `Queueable`
+ * interface for fluent configuration and provides the core structure for defining execution logic (`handle`)
+ * and failure handling (`failed`).
+ *
+ * Subclasses must implement the `handle` method.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * export class SendEmailJob extends Job {
+ *   constructor(private email: string, private subject: string) {
+ *     super();
+ *   }
+ *
+ *   async handle(): Promise<void> {
+ *     await emailService.send(this.email, this.subject);
+ *   }
+ * }
+ *
+ * // Usage
+ * await queue.push(new SendEmailJob('user@example.com', 'Welcome'))
+ *   .onQueue('emails')
+ *   .delay(60);
+ * ```
+ */
+export declare abstract class Job implements Queueable {
+    /**
+     * Unique identifier for the job instance.
+     *
+     * Assigned automatically when the job is pushed to the queue.
+     */
+    id?: string;
+    /**
+     * The name of the queue where this job will be processed.
+     */
+    queueName?: string;
+    /**
+     * The name of the connection used to transport this job.
+     */
+    connectionName?: string;
+    /**
+     * Delay in seconds before the job becomes available for processing.
+     */
+    delaySeconds?: number;
+    /**
+     * The current attempt number (starts at 1).
+     */
+    attempts?: number;
+    /**
+     * The maximum number of retry attempts allowed.
+     *
+     * Can be overridden by the worker configuration or per-job using `maxAttempts`.
+     */
+    maxAttempts?: number;
+    /**
+     * Group ID for sequential processing.
+     *
+     * Jobs with the same `groupId` will be processed in strict order (FIFO)
+     * if the consumer supports it.
+     */
+    groupId?: string;
+    /**
+     * Priority level of the job.
+     */
+    priority?: string | number;
+    /**
+     * Initial delay in seconds before the first retry attempt.
+     *
+     * Used for exponential backoff calculation.
+     */
+    retryAfterSeconds?: number;
+    /**
+     * Multiplier applied to the retry delay for each subsequent attempt.
+     *
+     * Used for exponential backoff calculation.
+     */
+    retryMultiplier?: number;
+    /**
+     * Sets the target queue for the job.
+     *
+     * @param queue - The name of the target queue.
+     * @returns The job instance for chaining.
+     *
+     * @example
+     * ```typescript
+     * job.onQueue('billing');
+     * ```
+     */
+    onQueue(queue: string): this;
+    /**
+     * Sets the target connection for the job.
+     *
+     * @param connection - The name of the connection (e.g., 'redis').
+     * @returns The job instance for chaining.
+     *
+     * @example
+     * ```typescript
+     * job.onConnection('sqs-primary');
+     * ```
+     */
+    onConnection(connection: string): this;
+    /**
+     * Sets the priority of the job.
+     *
+     * @param priority - The priority level (e.g., 'high', 10).
+     * @returns The job instance for chaining.
+     *
+     * @example
+     * ```typescript
+     * job.withPriority('high');
+     * ```
+     */
+    withPriority(priority: string | number): this;
+    /**
+     * Delays the job execution.
+     *
+     * @param delay - Delay in seconds.
+     * @returns The job instance for chaining.
+     *
+     * @example
+     * ```typescript
+     * job.delay(60); // Run after 1 minute
+     * ```
+     */
+    delay(delay: number): this;
+    /**
+     * Configures the exponential backoff strategy for retries.
+     *
+     * @param seconds - Initial delay in seconds before the first retry.
+     * @param multiplier - Factor by which the delay increases for each subsequent attempt (default: 2).
+     * @returns The job instance for chaining.
+     *
+     * @example
+     * ```typescript
+     * // Wait 5s, then 10s, then 20s...
+     * job.backoff(5, 2);
+     * ```
+     */
+    backoff(seconds: number, multiplier?: number): this;
+    /**
+     * Calculates the delay for the next retry attempt based on the backoff strategy.
+     *
+     * Uses the formula: `initialDelay * multiplier^(attempt - 1)`, capped at 1 hour.
+     *
+     * @param attempt - The current attempt number (1-based).
+     * @returns The calculated delay in milliseconds.
+     *
+     * @example
+     * ```typescript
+     * const nextDelay = job.getRetryDelay(2);
+     * ```
+     */
+    getRetryDelay(attempt: number): number;
+    /**
+     * Contains the main business logic of the job.
+     *
+     * This method is called by the worker to process the job.
+     * Implementations should be idempotent if possible.
+     *
+     * @throws {Error} If the job fails and should be retried.
+     */
+    abstract handle(): Promise<void>;
+    /**
+     * Optional handler for when the job has permanently failed.
+     *
+     * Called when the job has exhausted all retry attempts.
+     * Useful for cleaning up resources, sending alerts, or logging.
+     *
+     * @param _error - The error that caused the final failure.
+     *
+     * @example
+     * ```typescript
+     * async failed(error: Error) {
+     *   await notifyAdmin(`Job failed: ${error.message}`);
+     * }
+     * ```
+     */
+    failed(_error: Error): Promise<void>;
+}

package/dist/OrbitStream.d.ts

@@ -0,0 +1,151 @@
+import type { GravitoOrbit, PlanetCore } from '@gravito/core';
+import type { ConsumerOptions } from './Consumer';
+import { QueueManager } from './QueueManager';
+import type { QueueConfig } from './types';
+/**
+ * Configuration options for the OrbitStream extension.
+ *
+ * Extends the standard `QueueConfig` with specific options for integration into
+ * the Gravito lifecycle, such as auto-starting workers in development environments.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * const options: OrbitStreamOptions = {
+ *   default: 'redis',
+ *   autoStartWorker: true
+ * };
+ * ```
+ */
+export interface OrbitStreamOptions extends QueueConfig {
+    /**
+     * Automatically start an embedded worker process.
+     *
+     * If set to `true`, a background worker will be spawned within the main process.
+     * This is recommended for development or simple deployments but should be
+     * avoided in high-scale production to keep web servers stateless.
+     */
+    autoStartWorker?: boolean;
+    /**
+     * Configuration options for the embedded worker.
+     *
+     * Only used if `autoStartWorker` is true. Defines concurrency, polling intervals, etc.
+     */
+    workerOptions?: ConsumerOptions;
+    /**
+     * Configuration for the Stream Monitoring Dashboard API.
+     *
+     * If enabled, registers API routes for monitoring queue health and job history.
+     *
+     * @default false
+     */
+    dashboard?: boolean | {
+        /**
+         * Base path for the dashboard API routes.
+         * @default '/_flux'
+         */
+        path?: string;
+    };
+}
+/**
+ * The Queue Orbit (Plugin) for Gravito Framework.
+ *
+ * This class acts as the integration layer between the `@gravito/stream` package
+ * and the Gravito core system (`PlanetCore`). It registers the `QueueManager`
+ * service, injects it into the request context, and manages the lifecycle of
+ * embedded workers.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * const stream = OrbitStream.configure({
+ *   default: 'redis',
+ *   connections: {
+ *     redis: { driver: 'redis', client: redis }
+ *   }
+ * });
+ *
+ * await PlanetCore.boot({ orbits: [stream] });
+ * ```
+ */
+export declare class OrbitStream implements GravitoOrbit {
+    private options;
+    private queueManager?;
+    private consumer?;
+    private core?;
+    constructor(options?: OrbitStreamOptions);
+    /**
+     * Factory method for creating and configuring an OrbitStream instance.
+     *
+     * Provides a fluent way to instantiate the orbit during application bootstrap.
+     *
+     * @param options - Configuration options.
+     * @returns A new OrbitStream instance.
+     *
+     * @example
+     * ```typescript
+     * const orbit = OrbitStream.configure({ default: 'memory' });
+     * ```
+     */
+    static configure(options: OrbitStreamOptions): OrbitStream;
+    /**
+     * Installs the Queue system into the Gravito PlanetCore.
+     *
+     * This lifecycle method:
+     * 1. Initializes the `QueueManager`.
+     * 2. Registers the `queue` service in the dependency injection container.
+     * 3. Sets up a global middleware to inject `QueueManager` into the request context (`c.get('queue')`).
+     * 4. Automatically detects and registers database connections if available in the context.
+     * 5. Starts the embedded worker if configured.
+     *
+     * @param core - The PlanetCore instance.
+     */
+    install(core: PlanetCore): void;
+    /**
+     * Starts the embedded worker process.
+     *
+     * Launches a `Consumer` instance to process jobs in the background.
+     * Throws an error if `QueueManager` is not initialized or if a worker is already running.
+     *
+     * @param options - Consumer configuration options.
+     * @throws {Error} If QueueManager is missing or worker is already active.
+     *
+     * @example
+     * ```typescript
+     * orbit.startWorker({ queues: ['default'] });
+     * ```
+     */
+    startWorker(options: ConsumerOptions): void;
+    /**
+     * Stops the embedded worker process.
+     *
+     * Gracefully shuts down the consumer, waiting for active jobs to complete.
+     *
+     * @returns A promise that resolves when the worker has stopped.
+     *
+     * @example
+     * ```typescript
+     * await orbit.stopWorker();
+     * ```
+     */
+    stopWorker(): Promise<void>;
+    /**
+     * Retrieves the underlying QueueManager instance.
+     *
+     * @returns The active QueueManager, or undefined if not installed.
+     *
+     * @example
+     * ```typescript
+     * const manager = orbit.getQueueManager();
+     * ```
+     */
+    getQueueManager(): QueueManager | undefined;
+}
+declare module '@gravito/core' {
+    interface GravitoVariables {
+        /** Queue manager for job processing */
+        queue?: QueueManager;
+        /** Database service (from orbit-db) */
+        db?: unknown;
+    }
+}