@gravito/stream 2.0.1 → 2.1.0

package/dist/QueueManager.d.ts

@@ -0,0 +1,319 @@
+import type { QueueDriver } from './drivers/QueueDriver';
+import type { Job } from './Job';
+import type { Queueable } from './Queueable';
+import type { Scheduler } from './Scheduler';
+import type { JobSerializer } from './serializers/JobSerializer';
+import type { JobPushOptions, PersistenceAdapter, QueueConfig, QueueConnectionConfig, QueueStats, SerializedJob } from './types';
+/**
+ * The central manager for queue operations.
+ *
+ * This class manages multiple queue connections and drivers, exposing a unified API for pushing,
+ * popping, and managing jobs. It handles connection pooling, serialization, persistence,
+ * and driver lazy-loading.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * const manager = new QueueManager({
+ *   default: 'redis',
+ *   connections: {
+ *     redis: { driver: 'redis', client: redisClient }
+ *   }
+ * });
+ *
+ * await manager.push(new SendEmailJob('hello@example.com'));
+ * ```
+ */
+export declare class QueueManager {
+    private drivers;
+    private serializers;
+    private defaultConnection;
+    private defaultSerializer;
+    private persistence?;
+    private scheduler?;
+    private debug;
+    constructor(config?: QueueConfig);
+    /**
+     * Log debug message.
+     */
+    private log;
+    /**
+     * Registers a new queue connection with the manager.
+     *
+     * Dynamically loads the required driver implementation based on the configuration.
+     *
+     * @param name - The name of the connection (e.g., 'primary').
+     * @param config - The configuration object for the driver.
+     * @throws {Error} If the driver type is missing required dependencies or unsupported.
+     *
+     * @example
+     * ```typescript
+     * manager.registerConnection('analytics', { driver: 'sqs', client: sqs });
+     * ```
+     */
+    registerConnection(name: string, config: QueueConnectionConfig): void;
+    /**
+     * Retrieves the driver instance for a specific connection.
+     *
+     * @param connection - The name of the connection.
+     * @returns The configured QueueDriver instance.
+     * @throws {Error} If the connection has not been registered.
+     *
+     * @example
+     * ```typescript
+     * const driver = manager.getDriver('redis');
+     * ```
+     */
+    getDriver(connection: string): QueueDriver;
+    /**
+     * Gets the name of the default connection.
+     *
+     * @returns The default connection name.
+     */
+    getDefaultConnection(): string;
+    /**
+     * Retrieves a serializer instance by type.
+     *
+     * @param type - The serializer type (e.g., 'json', 'class'). If omitted, returns the default serializer.
+     * @returns The JobSerializer instance.
+     * @throws {Error} If the requested serializer type is not found.
+     */
+    getSerializer(type?: string): JobSerializer;
+    /**
+     * Registers Job classes for the `ClassNameSerializer`.
+     *
+     * This is required when using 'class' serialization to allow proper hydration of job instances
+     * upon deserialization.
+     *
+     * @param jobClasses - An array of Job class constructors.
+     *
+     * @example
+     * ```typescript
+     * manager.registerJobClasses([SendEmailJob, ProcessOrderJob]);
+     * ```
+     */
+    registerJobClasses(jobClasses: Array<new (...args: unknown[]) => Job>): void;
+    /**
+     * Pushes a single job to the queue.
+     *
+     * Serializes the job, selects the appropriate driver based on job configuration,
+     * and dispatches it. Also handles audit logging if persistence is enabled.
+     *
+     * @template T - The type of the job (extends Job).
+     * @param job - The job instance to enqueue.
+     * @param options - Optional overrides for push behavior (priority, delay, etc.).
+     * @returns The same job instance (for chaining).
+     *
+     * @example
+     * ```typescript
+     * await manager.push(new SendEmailJob('user@example.com'));
+     * ```
+     */
+    push<T extends Job & Queueable>(job: T, options?: JobPushOptions): Promise<T>;
+    /**
+     * Pushes multiple jobs to the queue in a batch.
+     *
+     * Optimizes network requests by batching jobs where possible. Groups jobs by connection
+     * and queue to maximize throughput.
+     *
+     * @template T - The type of the jobs.
+     * @param jobs - An array of job instances to enqueue.
+     * @param options - Configuration for batch size and concurrency.
+     * @returns A promise that resolves when all jobs have been pushed.
+     *
+     * @example
+     * ```typescript
+     * await manager.pushMany(jobs, { batchSize: 500, concurrency: 5 });
+     * ```
+     */
+    pushMany<T extends Job & Queueable>(jobs: T[], options?: {
+        batchSize?: number;
+        concurrency?: number;
+    }): Promise<void>;
+    /**
+     * Pops a single job from the queue.
+     *
+     * Retrieves the next available job from the specified queue.
+     *
+     * @param queue - The queue name (default: 'default').
+     * @param connection - The connection name (defaults to default connection).
+     * @returns A Job instance if found, or `null` if the queue is empty.
+     *
+     * @example
+     * ```typescript
+     * const job = await manager.pop('priority-queue');
+     * if (job) await job.handle();
+     * ```
+     */
+    pop(queue?: string, connection?: string): Promise<Job | null>;
+    /**
+     * Pops multiple jobs from the queue efficiently.
+     *
+     * Attempts to retrieve a batch of jobs from the driver. If the driver does not support
+     * batching, it falls back to sequential popping.
+     *
+     * @param queue - The queue name (default: 'default').
+     * @param count - The maximum number of jobs to retrieve (default: 10).
+     * @param connection - The connection name.
+     * @returns An array of Job instances.
+     *
+     * @example
+     * ```typescript
+     * const jobs = await manager.popMany('default', 50);
+     * ```
+     */
+    popMany(queue?: string, count?: number, connection?: string): Promise<Job[]>;
+    /**
+     * Retrieves the current size of a queue.
+     *
+     * @param queue - The queue name (default: 'default').
+     * @param connection - The connection name.
+     * @returns The number of waiting jobs.
+     *
+     * @example
+     * ```typescript
+     * const count = await manager.size('emails');
+     * ```
+     */
+    size(queue?: string, connection?: string): Promise<number>;
+    /**
+     * Pops a job from the queue with blocking (wait) behavior.
+     *
+     * Waits for a job to become available for the specified timeout duration.
+     * Useful for reducing polling loop frequency.
+     *
+     * @param queues - A queue name or array of queue names to listen to.
+     * @param timeout - Timeout in seconds (0 = block indefinitely).
+     * @param connection - The connection name.
+     * @returns A Job instance if found, or `null` if timed out.
+     *
+     * @example
+     * ```typescript
+     * // Wait up to 30 seconds for a job
+     * const job = await manager.popBlocking('default', 30);
+     * ```
+     */
+    popBlocking(queues?: string | string[], timeout?: number, connection?: string): Promise<Job | null>;
+    /**
+     * Removes all jobs from a specific queue.
+     *
+     * @param queue - The queue name to purge.
+     * @param connection - The connection name.
+     *
+     * @example
+     * ```typescript
+     * await manager.clear('test-queue');
+     * ```
+     */
+    clear(queue?: string, connection?: string): Promise<void>;
+    /**
+     * Retrieves comprehensive statistics for a queue.
+     *
+     * Includes counts for pending, processing, delayed, and failed jobs.
+     *
+     * @param queue - The queue name.
+     * @param connection - The connection name.
+     * @returns A QueueStats object.
+     *
+     * @example
+     * ```typescript
+     * const stats = await manager.stats('default');
+     * console.log(stats.size, stats.failed);
+     * ```
+     */
+    stats(queue?: string, connection?: string): Promise<QueueStats>;
+    /**
+     * Marks a job as successfully completed.
+     *
+     * Removes the job from the processing state and optionally archives it.
+     *
+     * @param job - The job instance that finished.
+     *
+     * @example
+     * ```typescript
+     * await manager.complete(job);
+     * ```
+     */
+    complete<T extends Job & Queueable>(job: T): Promise<void>;
+    /**
+     * Marks a job as failed.
+     *
+     * Moves the job to the failed state (Dead Letter Queue) and optionally archives it.
+     * This is typically called after max retry attempts are exhausted.
+     *
+     * @param job - The job instance that failed.
+     * @param error - The error that caused the failure.
+     *
+     * @example
+     * ```typescript
+     * await manager.fail(job, new Error('Something went wrong'));
+     * ```
+     */
+    fail<T extends Job & Queueable>(job: T, error: Error): Promise<void>;
+    /**
+     * Retrieves the configured persistence adapter.
+     *
+     * @returns The PersistenceAdapter instance, or undefined if not configured.
+     */
+    getPersistence(): PersistenceAdapter | undefined;
+    /**
+     * Gets the Scheduler instance associated with this manager.
+     *
+     * The Scheduler handles delayed jobs and periodic tasks.
+     *
+     * @returns The Scheduler instance.
+     */
+    getScheduler(): Scheduler;
+    /**
+     * Retrieves failed jobs from the Dead Letter Queue.
+     *
+     * @param queue - The queue name.
+     * @param start - The starting index (pagination).
+     * @param end - The ending index (pagination).
+     * @param connection - The connection name.
+     * @returns An array of serialized jobs.
+     *
+     * @example
+     * ```typescript
+     * const failedJobs = await manager.getFailed('default', 0, 10);
+     * ```
+     */
+    getFailed(queue: string, start?: number, end?: number, connection?: string): Promise<SerializedJob[]>;
+    /**
+     * Retries failed jobs from the Dead Letter Queue.
+     *
+     * Moves jobs from the failed state back to the active queue for re-processing.
+     *
+     * @param queue - The queue name.
+     * @param count - The number of jobs to retry.
+     * @param connection - The connection name.
+     * @returns The number of jobs successfully retried.
+     *
+     * @example
+     * ```typescript
+     * await manager.retryFailed('default', 5);
+     * ```
+     */
+    retryFailed(queue: string, count?: number, connection?: string): Promise<number>;
+    /**
+     * Clears all failed jobs from the Dead Letter Queue.
+     *
+     * @param queue - The queue name.
+     * @param connection - The connection name.
+     *
+     * @example
+     * ```typescript
+     * await manager.clearFailed('default');
+     * ```
+     */
+    clearFailed(queue: string, connection?: string): Promise<void>;
+    /**
+     * Retrieves high-level statistics across all registered connections and queues.
+     *
+     * Iterates through all drivers and collects metadata to provide a comprehensive
+     * snapshot of the entire queue system's health.
+     *
+     * @returns A promise resolving to a GlobalStats object.
+     */
+    getGlobalStats(): Promise<import('./types').GlobalStats>;
+}

package/dist/Queueable.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Defines the contract for queueable items (jobs) with fluent configuration.
+ *
+ * This interface enables method chaining for configuring job properties such as
+ * target queue, connection, delay, and priority before dispatching. It ensures
+ * consistent API surface across different job implementations.
+ *
+ * @public
+ * @example
+ * ```typescript
+ * class MyJob implements Queueable {
+ *   queueName?: string
+ *   // ... implementation
+ *   onQueue(queue: string): this {
+ *     this.queueName = queue
+ *     return this
+ *   }
+ *   // ...
+ * }
+ * ```
+ */
+export interface Queueable {
+    /**
+     * The specific queue name where the job should be processed.
+     *
+     * If not set, the default queue for the connection will be used.
+     */
+    queueName?: string;
+    /**
+     * The connection name (e.g., 'redis', 'sqs') to use for this job.
+     *
+     * If not set, the default connection configured in QueueManager will be used.
+     */
+    connectionName?: string;
+    /**
+     * The number of seconds to delay the job execution.
+     */
+    delaySeconds?: number;
+    /**
+     * The priority level of the job.
+     */
+    priority?: string | number;
+    /**
+     * Sets the target queue for the job.
+     *
+     * @param queue - The name of the queue to push the job to.
+     * @returns The instance for method chaining.
+     *
+     * @example
+     * ```typescript
+     * job.onQueue('notifications');
+     * ```
+     */
+    onQueue(queue: string): this;
+    /**
+     * Sets the target connection for the job.
+     *
+     * @param connection - The name of the connection to use.
+     * @returns The instance for method chaining.
+     *
+     * @example
+     * ```typescript
+     * job.onConnection('sqs');
+     * ```
+     */
+    onConnection(connection: string): this;
+    /**
+     * Sets the priority of the job.
+     *
+     * @param priority - The priority level (e.g., 'high', 'low', 10).
+     * @returns The instance for method chaining.
+     *
+     * @example
+     * ```typescript
+     * job.withPriority('critical');
+     * ```
+     */
+    withPriority(priority: string | number): this;
+    /**
+     * Sets a delay before the job is available for processing.
+     *
+     * @param delay - The delay in seconds.
+     * @returns The instance for method chaining.
+     *
+     * @example
+     * ```typescript
+     * job.delay(300); // 5 minutes
+     * ```
+     */
+    delay(delay: number): this;
+}

package/dist/Scheduler.d.ts

@@ -0,0 +1,214 @@
+import type { QueueManager } from './QueueManager';
+import type { SerializedJob } from './types';
+/**
+ * Configuration for a recurring scheduled job.
+ *
+ * Defines the schedule (CRON), the job to execute, and metadata tracking execution times.
+ *
+ * @public
+ * @since 3.0.0
+ * @example
+ * ```typescript
+ * const config: ScheduledJobConfig = {
+ *   id: 'daily-report',
+ *   cron: '0 0 * * *',
+ *   queue: 'reports',
+ *   job: serializedJob,
+ *   enabled: true
+ * };
+ * ```
+ */
+export interface ScheduledJobConfig {
+    /** Unique identifier for the scheduled task. */
+    id: string;
+    /** Cron expression defining the schedule (e.g., '* * * * *'). */
+    cron: string;
+    /** The target queue name where the job should be pushed. */
+    queue: string;
+    /** The serialized job data. */
+    job: SerializedJob;
+    /** Timestamp of the last successful execution in milliseconds. */
+    lastRun?: number;
+    /** Timestamp of the next scheduled execution in milliseconds. */
+    nextRun?: number;
+    /** Whether the scheduled job is active. */
+    enabled: boolean;
+}
+/**
+ * Configuration options for the Scheduler.
+ *
+ * Defines behavior for scheduling tasks, including distributed lock settings.
+ *
+ * @public
+ * @since 3.1.0
+ * @example
+ * ```typescript
+ * const options: SchedulerOptions = {
+ *   prefix: 'myapp:queue:',
+ *   lockTtl: 60000,        // Lock held for 60 seconds
+ *   lockRefreshInterval: 20000  // Auto-renew every 20 seconds
+ * };
+ * ```
+ */
+export interface SchedulerOptions {
+    /**
+     * Prefix for Redis keys.
+     *
+     * @default 'queue:'
+     */
+    prefix?: string;
+    /**
+     * Time-to-live for the distributed lock in milliseconds.
+     *
+     * Setting a longer TTL ensures long-running tasks are not executed repeatedly
+     * due to lock expiration. Recommended to be 2-3 times the expected execution time.
+     *
+     * @default 60000 (60 seconds)
+     */
+    lockTtl?: number;
+    /**
+     * Interval for automatic lock renewal in milliseconds.
+     *
+     * If set, the lock will be automatically extended every `lockRefreshInterval`.
+     * Recommended to be 1/3 of `lockTtl`.
+     *
+     * @default 20000 (20 seconds)
+     */
+    lockRefreshInterval?: number;
+    /**
+     * Number of retries when acquiring a lock fails.
+     *
+     * @default 0
+     */
+    lockRetryCount?: number;
+    /**
+     * Delay between lock acquisition retries in milliseconds.
+     *
+     * @default 100
+     */
+    lockRetryDelay?: number;
+    /**
+     * The interval in milliseconds at which the scheduler checks for due tasks.
+     *
+     * @default 60000 (1 minute)
+     */
+    tickInterval?: number;
+    /**
+     * The time-to-live for the leader lock in milliseconds.
+     *
+     * Ensures that only one node acts as the scheduler leader.
+     * @default 30000 (30 seconds)
+     */
+    leaderTtl?: number;
+}
+/**
+ * Manages recurring tasks and cron jobs.
+ *
+ * The Scheduler allows you to register jobs to run at specific intervals using CRON syntax.
+ * It uses Redis (or a compatible driver) to coordinate distributed execution, ensuring that
+ * a scheduled job runs only once per interval across multiple scheduler instances.
+ *
+ * @public
+ * @since 3.0.0
+ * @example
+ * ```typescript
+ * const scheduler = manager.getScheduler();
+ * await scheduler.register({
+ *   id: 'cleanup',
+ *   cron: '0 * * * *', // Every hour
+ *   job: new CleanupJob()
+ * });
+ *
+ * // Automatically start the scheduler loop
+ * await scheduler.start();
+ * ```
+ */
+export declare class Scheduler {
+    private manager;
+    private prefix;
+    private lockTtl;
+    private lockRefreshInterval?;
+    private lockRetryCount;
+    private lockRetryDelay;
+    private tickInterval;
+    private leaderTtl;
+    private distributedLock?;
+    private running;
+    private timer;
+    private isLeader;
+    constructor(manager: QueueManager, options?: SchedulerOptions);
+    private get client();
+    /**
+     * Gets or creates the distributed lock instance.
+     *
+     * @private
+     */
+    private getDistributedLock;
+    /**
+     * Registers a new scheduled job or updates an existing one.
+     *
+     * Calculates the next run time based on the CRON expression and stores the configuration in Redis.
+     *
+     * @param config - The job configuration (excluding nextRun and enabled status which are auto-set).
+     * @throws {Error} If Redis client does not support pipelining.
+     */
+    register(config: Omit<ScheduledJobConfig, 'nextRun' | 'enabled'>): Promise<void>;
+    /**
+     * Removes a scheduled job.
+     *
+     * Deletes the job metadata and schedule entry from Redis.
+     *
+     * @param id - The unique identifier of the scheduled job.
+     */
+    remove(id: string): Promise<void>;
+    /**
+     * Lists all registered scheduled jobs.
+     *
+     * @returns An array of all scheduled job configurations.
+     */
+    list(): Promise<ScheduledJobConfig[]>;
+    /**
+     * Starts the automatic scheduler loop.
+     *
+     * Periodically triggers `tick()` to process due jobs. Uses leader election
+     * to ensure that only one node performs the scanning in a multi-node environment.
+     */
+    start(): Promise<void>;
+    /**
+     * Stops the automatic scheduler loop.
+     */
+    stop(): Promise<void>;
+    /**
+     * Acquires the leader lock and performs a tick.
+     *
+     * @private
+     */
+    private performTickWithLeaderElection;
+    /**
+     * Releases the leader lock.
+     *
+     * @private
+     */
+    private releaseLeader;
+    /**
+     * Manually triggers a scheduled job immediately.
+     *
+     * Forces execution of the job regardless of its schedule, without affecting the next scheduled run time.
+     *
+     * @param id - The unique identifier of the scheduled job.
+     */
+    runNow(id: string): Promise<void>;
+    /**
+     * Checks for and triggers tasks that are due for execution.
+     *
+     * This method should be called periodically (e.g., via a system cron or a dedicated tick loop).
+     * It scans the schedule for tasks with `nextRun <= now`, acquires a distributed lock for each,
+     * pushes them to their queue, and updates the `nextRun` time.
+     *
+     * The distributed lock ensures that in a multi-node environment, each scheduled job is executed
+     * only once per interval, even if multiple scheduler instances are running.
+     *
+     * @returns The number of jobs triggered in this tick.
+     */
+    tick(): Promise<number>;
+}