@gravito/stream 2.1.2 → 3.0.0

package/dist/drivers/DatabaseDriver.d.ts

@@ -1,131 +0,0 @@
-import type { QueueStats, SerializedJob } from '../types';
-import type { QueueDriver } from './QueueDriver';
-/**
- * Generic database service interface.
- *
- * Adapts any SQL database client (e.g., pg, mysql2, sqlite3) for use with the DatabaseDriver.
- * Users must provide an implementation of this interface that wraps their specific DB library.
- */
-export interface DatabaseService {
-    /**
-     * Execute a raw SQL query.
-     *
-     * @param sql - The SQL query string with placeholders (e.g., $1, ?).
-     * @param bindings - The values to bind to the placeholders.
-     * @returns The query result (rows or metadata).
-     */
-    execute<T = unknown>(sql: string, bindings?: unknown[]): Promise<T[] | T>;
-    /**
-     * Execute multiple queries within a single transaction.
-     *
-     * @param callback - A function that receives a transaction-scoped service instance.
-     * @returns The result of the callback.
-     */
-    transaction<T>(callback: (tx: DatabaseService) => Promise<T>): Promise<T>;
-}
-/**
- * Configuration options for the DatabaseDriver.
- */
-export interface DatabaseDriverConfig {
-    /**
-     * The name of the table used to store jobs.
-     * @default 'jobs'
-     */
-    table?: string;
-    /**
-     * The database service adapter instance.
-     */
-    dbService?: DatabaseService;
-}
-/**
- * Database-backed queue driver.
- *
- * Persists jobs in a SQL database table. Supports delayed jobs, reservation (locking),
- * and reliable delivery. Compatible with PostgreSQL (SKIP LOCKED), MySQL, and SQLite.
- *
- * @public
- * @example
- * ```typescript
- * const driver = new DatabaseDriver({
- *   dbService: myDbAdapter,
- *   table: 'queue_jobs'
- * });
- * ```
- */
-export declare class DatabaseDriver implements QueueDriver {
-    private tableName;
-    private dbService;
-    constructor(config: DatabaseDriverConfig);
-    /**
-     * Pushes a job to the database queue.
-     *
-     * Inserts a new row into the jobs table.
-     *
-     * @param queue - The queue name.
-     * @param job - The serialized job.
-     */
-    push(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Pops the next available job from the queue.
-     *
-     * Uses transactional locking (SELECT ... FOR UPDATE SKIP LOCKED if supported) to ensure
-     * atomic reservation of jobs by workers.
-     *
-     * @param queue - The queue name.
-     * @returns The job or `null`.
-     */
-    pop(queue: string): Promise<SerializedJob | null>;
-    /**
-     * Pops multiple jobs from the queue in a single transaction.
-     *
-     * @param queue - The queue name.
-     * @param count - Max jobs to pop.
-     */
-    popMany(queue: string, count: number): Promise<SerializedJob[]>;
-    /**
-     * Retrieves queue statistics by querying the table.
-     *
-     * @param queue - The queue name.
-     */
-    stats(queue: string): Promise<QueueStats>;
-    /**
-     * Returns the count of pending jobs.
-     *
-     * @param queue - The queue name.
-     */
-    size(queue: string): Promise<number>;
-    /**
-     * Clears the queue by deleting all rows for the queue.
-     *
-     * @param queue - The queue name.
-     */
-    clear(queue: string): Promise<void>;
-    /**
-     * Pops a job using a polling loop (Blocking simulation).
-     *
-     * @param queue - The queue name.
-     * @param timeout - Timeout in seconds.
-     */
-    popBlocking(queue: string, timeout: number): Promise<SerializedJob | null>;
-    /**
-     * Pushes multiple jobs using a transaction.
-     *
-     * @param queue - The queue name.
-     * @param jobs - Array of jobs.
-     */
-    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
-    /**
-     * Marks a job as permanently failed by moving it to the DLQ (separate logical queue in DB).
-     *
-     * @param queue - The queue name.
-     * @param job - The failed job.
-     */
-    fail(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Deletes a job row from the database (completion).
-     *
-     * @param _queue - The queue name (unused).
-     * @param job - The job to complete.
-     */
-    complete(_queue: string, job: SerializedJob): Promise<void>;
-}

package/dist/drivers/GrpcDriver.d.ts

@@ -1,16 +0,0 @@
-import type { GrpcDriverConfig, JobPushOptions, QueueStats, SerializedJob } from '../types';
-import type { QueueDriver } from './QueueDriver';
-export type { GrpcDriverConfig } from '../types';
-export declare class GrpcDriver implements QueueDriver {
-    private client;
-    constructor(config: GrpcDriverConfig);
-    private getCredentials;
-    push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
-    pop(queue: string): Promise<SerializedJob | null>;
-    size(queue: string): Promise<number>;
-    clear(queue: string): Promise<void>;
-    acknowledge(messageId: string): Promise<void>;
-    stats(queue: string): Promise<QueueStats>;
-    private toProtoJob;
-    private fromProtoJob;
-}

package/dist/drivers/KafkaDriver.d.ts

@@ -1,161 +0,0 @@
-import type { SerializedJob, TopicOptions } from '../types';
-import type { QueueDriver } from './QueueDriver';
-/**
- * Kafka driver configuration.
- */
-export interface KafkaDriverConfig {
-    /**
-     * Kafka client instance (kafkajs).
-     *
-     * Must provide producer, admin, and consumer factories compatible with KafkaJS.
-     */
-    client: {
-        producer: () => {
-            connect: () => Promise<void>;
-            send: (args: {
-                topic: string;
-                messages: Array<{
-                    key?: string;
-                    value: string;
-                }>;
-            }) => Promise<void>;
-            disconnect: () => Promise<void>;
-        };
-        admin: () => {
-            connect: () => Promise<void>;
-            createTopics: (args: {
-                topics: Array<{
-                    topic: string;
-                    numPartitions?: number;
-                    replicationFactor?: number;
-                }>;
-            }) => Promise<void>;
-            deleteTopics: (args: {
-                topics: string[];
-            }) => Promise<void>;
-            disconnect: () => Promise<void>;
-        };
-        consumer: (args: {
-            groupId: string;
-        }) => {
-            connect: () => Promise<void>;
-            subscribe: (args: {
-                topics: string[];
-            }) => Promise<void>;
-            run: (args: KafkaConsumerRunOptions) => Promise<void>;
-            disconnect: () => Promise<void>;
-        };
-    };
-    /**
-     * Consumer group ID used for reading messages.
-     * @default 'gravito-workers'
-     */
-    consumerGroupId?: string;
-}
-interface KafkaMessage {
-    key?: Buffer;
-    value: Buffer | null;
-    offset: string;
-}
-interface KafkaBatch {
-    messages: KafkaMessage[];
-}
-interface KafkaEachBatchPayload {
-    batch: KafkaBatch;
-    resolveOffset(offset: string): void;
-    heartbeat(): Promise<void>;
-    isRunning(): boolean;
-}
-interface KafkaConsumerRunOptions {
-    eachMessage?: (args: {
-        topic: string;
-        partition: number;
-        message: KafkaMessage;
-    }) => Promise<void>;
-    eachBatch?: (args: KafkaEachBatchPayload) => Promise<void>;
-}
-/**
- * Kafka-backed queue driver.
- *
- * Uses Apache Kafka topics as queues. Designed for high-throughput streaming
- * rather than traditional job queue semantics (pop/delete).
- * Supports push-based consumption via `subscribe()`.
- *
- * @public
- * @example
- * ```typescript
- * const driver = new KafkaDriver({ client: kafka, consumerGroupId: 'my-app' });
- * ```
- */
-export declare class KafkaDriver implements QueueDriver {
-    private client;
-    private consumerGroupId;
-    private producer?;
-    private admin?;
-    constructor(config: KafkaDriverConfig);
-    /**
-     * Ensure the producer is connected.
-     */
-    private ensureProducer;
-    /**
-     * Ensure the admin client is connected.
-     */
-    private ensureAdmin;
-    /**
-     * Pushes a job to a Kafka topic.
-     *
-     * @param queue - The topic name.
-     * @param job - The job to publish.
-     */
-    push(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Pop is not supported for Kafka (Push-based).
-     *
-     * Kafka consumers typically stream messages. Use `subscribe()` instead.
-     *
-     * @throws {Error} Always throws as Kafka does not support polling individual messages in this manner.
-     */
-    pop(_queue: string): Promise<SerializedJob | null>;
-    /**
-     * Returns 0 as Kafka does not expose a simple "queue size".
-     *
-     * Monitoring lag requires external tools or Admin API checks not implemented here.
-     */
-    size(_queue: string): Promise<number>;
-    /**
-     * Clears a queue by deleting the topic.
-     *
-     * @param queue - The topic name.
-     */
-    clear(queue: string): Promise<void>;
-    /**
-     * Pushes multiple jobs to a Kafka topic.
-     *
-     * @param queue - The topic name.
-     * @param jobs - Array of jobs.
-     */
-    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
-    /**
-     * Creates a new Kafka topic.
-     *
-     * @param topic - The topic name.
-     * @param options - Config for partitions/replication.
-     */
-    createTopic(topic: string, options?: TopicOptions): Promise<void>;
-    /**
-     * Deletes a Kafka topic.
-     *
-     * @param topic - The topic name.
-     */
-    deleteTopic(topic: string): Promise<void>;
-    /**
-     * Subscribes to a topic for streaming jobs.
-     *
-     * Starts a Kafka consumer group and processes messages as they arrive.
-     *
-     * @param queue - The topic name.
-     * @param callback - Function to handle the job.
-     */
-    subscribe(queue: string, callback: (job: SerializedJob) => Promise<void>): Promise<void>;
-}
-export {};

package/dist/drivers/MemoryDriver.d.ts

@@ -1,119 +0,0 @@
-import type { QueueStats, SerializedJob } from '../types';
-import type { QueueDriver } from './QueueDriver';
-/**
- * Configuration options for the MemoryDriver.
- */
-export interface MemoryDriverConfig {
-    /**
-     * The maximum number of jobs allowed in a single queue.
-     *
-     * @default Infinity
-     */
-    maxSize?: number;
-}
-/**
- * In-memory queue driver.
- *
- * Stores jobs in a local JavaScript Map. Ideal for development, testing, and simple
- * use cases where persistence across restarts is not required.
- * It supports basic delay handling but data is volatile.
- *
- * @public
- * @example
- * ```typescript
- * const driver = new MemoryDriver({ maxSize: 1000 });
- * await driver.push('default', job);
- * ```
- */
-export declare class MemoryDriver implements QueueDriver {
-    private queues;
-    private maxSize;
-    private readonly callbacks;
-    private notificationsEnabled;
-    constructor(config?: MemoryDriverConfig);
-    /**
-     * Pushes a job to the in-memory queue.
-     *
-     * @param queue - The queue name.
-     * @param job - The serialized job.
-     * @throws {Error} If the queue has reached `maxSize`.
-     */
-    push(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Pops the next available job from the queue.
-     *
-     * Respects `delaySeconds` by checking the job's `createdAt` timestamp.
-     *
-     * @param queue - The queue name.
-     * @returns The job or `null`.
-     */
-    pop(queue: string): Promise<SerializedJob | null>;
-    /**
-     * Returns the number of jobs in the queue.
-     *
-     * @param queue - The queue name.
-     */
-    size(queue: string): Promise<number>;
-    /**
-     * Clears all jobs from the queue.
-     *
-     * @param queue - The queue name.
-     */
-    clear(queue: string): Promise<void>;
-    /**
-     * Moves a job to the failed (DLQ) list.
-     *
-     * In MemoryDriver, this simply pushes to a `failed:{queue}` list.
-     *
-     * @param queue - The original queue name.
-     * @param job - The failed job.
-     */
-    fail(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Retrieves statistics for the queue.
-     *
-     * Calculates pending, delayed, and failed counts by iterating through the list.
-     *
-     * @param queue - The queue name.
-     */
-    stats(queue: string): Promise<QueueStats>;
-    /**
-     * Pushes multiple jobs to the queue.
-     *
-     * @param queue - The queue name.
-     * @param jobs - Array of jobs.
-     */
-    pushMany(queue: string, jobs: SerializedJob[]): Promise<void>;
-    /**
-     * Pops multiple jobs from the queue.
-     *
-     * @param queue - The queue name.
-     * @param count - Max jobs to pop.
-     */
-    popMany(queue: string, count: number): Promise<SerializedJob[]>;
-    /**
-     * Lists all active queues in memory.
-     */
-    getQueues(): Promise<string[]>;
-    /**
-     * Enable notifications for this driver.
-     */
-    enableNotifications(): Promise<void>;
-    /**
-     * Disable notifications for this driver.
-     */
-    disableNotifications(): Promise<void>;
-    /**
-     * Register a notification callback for specific queues.
-     *
-     * @param queues - The queue names.
-     * @param callback - The callback function.
-     */
-    onNotify(queues: string | string[], callback: (queue: string) => Promise<void>): Promise<void>;
-    /**
-     * Internal notify helper.
-     *
-     * @param queue - The queue name.
-     */
-    private notify;
-}

package/dist/drivers/QueueDriver.d.ts

@@ -1,250 +0,0 @@
-import type { JobPushOptions, QueueStats, SerializedJob, TopicOptions } from '../types';
-/**
- * Queue driver interface.
- *
- * Defines the contract that all storage backends (Redis, Database, SQS, etc.) must implement
- * to be compatible with the QueueManager. It covers the basic operations for a job queue:
- * push, pop, size, and clear.
- *
- * Advanced capabilities like rate limiting, reliable delivery (acknowledgements), and
- * batch operations are optional but recommended for high-performance drivers.
- *
- * @public
- * @example
- * ```typescript
- * class MyCustomDriver implements QueueDriver {
- *   async push(queue: string, job: SerializedJob) {
- *     // ... write to storage
- *   }
- *   async pop(queue: string) {
- *     // ... read from storage
- *     return job;
- *   }
- *   async size(queue: string) { return 0; }
- *   async clear(queue: string) {}
- * }
- * ```
- */
-export interface QueueDriver {
-    /**
-     * Pushes a job onto the specified queue.
-     *
-     * @param queue - The name of the queue.
-     * @param job - The serialized job data.
-     * @param options - Optional parameters like priority or group ID.
-     * @returns A promise that resolves when the job is successfully stored.
-     */
-    push(queue: string, job: SerializedJob, options?: JobPushOptions): Promise<void>;
-    /**
-     * Retrieves and removes the next job from the queue (FIFO).
-     *
-     * @param queue - The name of the queue.
-     * @returns The serialized job if available, or `null` if the queue is empty.
-     */
-    pop(queue: string): Promise<SerializedJob | null>;
-    /**
-     * Blocking version of `pop`. Waits for a job to arrive if the queue is empty.
-     *
-     * @param queues - A single queue name or an array of queues to listen to.
-     * @param timeout - The maximum time to wait in seconds (0 means indefinite).
-     * @returns The serialized job if one arrives within the timeout, or `null`.
-     */
-    popBlocking?(queues: string | string[], timeout: number): Promise<SerializedJob | null>;
-    /**
-     * Marks a job as completed.
-     *
-     * Used primarily by drivers that support advanced flow control (like FIFO groups)
-     * or explicit acknowledgement (like SQS/RabbitMQ).
-     *
-     * @param queue - The name of the queue.
-     * @param job - The job that was completed.
-     */
-    complete?(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Returns the number of jobs currently waiting in the queue.
-     *
-     * @param queue - The name of the queue.
-     * @returns The count of pending jobs.
-     */
-    size(queue: string): Promise<number>;
-    /**
-     * Removes all jobs from the specified queue.
-     *
-     * @param queue - The name of the queue to purge.
-     */
-    clear(queue: string): Promise<void>;
-    /**
-     * Moves a job to the Dead Letter Queue (DLQ) after repeated failures.
-     *
-     * @param queue - The original queue name.
-     * @param job - The job data (usually including error information).
-     */
-    fail?(queue: string, job: SerializedJob): Promise<void>;
-    /**
-     * Retrieves detailed statistics for a queue.
-     *
-     * @param queue - The name of the queue.
-     * @returns An object containing counts for pending, delayed, failed, etc.
-     */
-    stats?(queue: string): Promise<QueueStats>;
-    /**
-     * Pushes multiple jobs to the queue in a single batch operation.
-     *
-     * @param queue - The name of the queue.
-     * @param jobs - An array of serialized jobs.
-     */
-    pushMany?(queue: string, jobs: SerializedJob[]): Promise<void>;
-    /**
-     * Retrieves multiple jobs from the queue in a single batch operation.
-     *
-     * @param queue - The name of the queue.
-     * @param count - The maximum number of jobs to retrieve.
-     * @returns An array of serialized jobs.
-     */
-    popMany?(queue: string, count: number): Promise<SerializedJob[]>;
-    /**
-     * Acknowledges a specific message by its ID.
-     *
-     * Used for drivers that require explicit acknowledgement (e.g., SQS, Kafka).
-     *
-     * @param messageId - The ID of the message to acknowledge.
-     */
-    acknowledge?(messageId: string): Promise<void>;
-    /**
-     * Subscribes to a queue for real-time job processing (Push model).
-     *
-     * Alternative to polling (`pop`). The driver pushes jobs to the callback as they arrive.
-     *
-     * @param queue - The name of the queue.
-     * @param callback - The function to call when a job is received.
-     */
-    subscribe?(queue: string, callback: (job: SerializedJob) => Promise<void>): Promise<void>;
-    /**
-     * Creates a new topic or queue (for drivers like Kafka/RabbitMQ).
-     *
-     * @param topic - The name of the topic/queue.
-     * @param options - Configuration options (partitions, replication, etc.).
-     */
-    createTopic?(topic: string, options?: TopicOptions): Promise<void>;
-    /**
-     * Deletes a topic or queue.
-     *
-     * @param topic - The name of the topic/queue to delete.
-     */
-    deleteTopic?(topic: string): Promise<void>;
-    /**
-     * Sends a heartbeat signal for a worker instance.
-     *
-     * Used for monitoring worker health and presence.
-     *
-     * @param workerInfo - Metadata about the worker (ID, status, resources).
-     * @param prefix - Optional key prefix for storage.
-     */
-    reportHeartbeat?(workerInfo: {
-        id: string;
-        status: string;
-        hostname: string;
-        pid: number;
-        uptime: number;
-        last_ping: string;
-        queues: string[];
-        metrics?: Record<string, unknown>;
-        [key: string]: unknown;
-    }, prefix?: string): Promise<void>;
-    /**
-     * Publishes a log entry to the monitoring system.
-     *
-     * @param logPayload - The log data (level, message, context).
-     * @param prefix - Optional key prefix.
-     */
-    publishLog?(logPayload: {
-        level: string;
-        message: string;
-        workerId: string;
-        jobId?: string;
-        timestamp: string;
-        [key: string]: unknown;
-    }, prefix?: string): Promise<void>;
-    /**
-     * Checks if a specific queue has exceeded its rate limit.
-     *
-     * @param queue - The name of the queue.
-     * @param config - The rate limit rules (max jobs per duration).
-     * @returns `true` if the job is allowed to proceed, `false` if limited.
-     */
-    checkRateLimit?(queue: string, config: {
-        max: number;
-        duration: number;
-    }): Promise<boolean>;
-    /**
-     * Retries failed jobs from the Dead Letter Queue.
-     *
-     * Moves jobs from the DLQ back to the active queue.
-     *
-     * @param queue - The name of the queue.
-     * @param count - The number of jobs to retry (optional).
-     * @returns The number of jobs actually moved.
-     */
-    retryFailed?(queue: string, count?: number): Promise<number>;
-    /**
-     * Retrieves a list of failed jobs from the Dead Letter Queue.
-     *
-     * @param queue - The name of the queue.
-     * @param start - Pagination start index.
-     * @param end - Pagination end index.
-     * @returns An array of failed jobs.
-     */
-    getFailed?(queue: string, start?: number, end?: number): Promise<SerializedJob[]>;
-    /**
-     * Clears the Dead Letter Queue for a specific queue.
-     *
-     * @param queue - The name of the queue.
-     */
-    clearFailed?(queue: string): Promise<void>;
-    /**
-     * Lists all queues managed by this driver.
-     *
-     * Useful for monitoring dashboards to discover active queues dynamically.
-     *
-     * @returns A list of queue names.
-     */
-    getQueues?(): Promise<string[]>;
-    /**
-     * Registers a notification listener for when jobs arrive in a queue.
-     *
-     * Only sends the queue name as notification, not the job data itself.
-     * The consumer is responsible for pulling jobs from the queue.
-     * This enables reactive pull-based consumption with low latency.
-     *
-     * @param queues - One or more queue names to listen for notifications.
-     * @param callback - Called when a job arrives in any of the queues, with the queue name.
-     * @returns A promise that resolves when the listener is registered.
-     *
-     * @example
-     * ```typescript
-     * driver.onNotify(['emails', 'sms'], (queue) => {
-     *   console.log(`Job arrived in ${queue}`);
-     *   // Consumer will pull from this queue reactively
-     * });
-     * ```
-     */
-    onNotify?(queues: string | string[], callback: (queue: string) => Promise<void>): Promise<void>;
-    /**
-     * Enables real-time notifications for job arrivals.
-     *
-     * Activates the notification mechanism so that `onNotify` callbacks are invoked.
-     * Called once during consumer startup.
-     *
-     * @returns A promise that resolves when notifications are enabled.
-     */
-    enableNotifications?(): Promise<void>;
-    /**
-     * Disables real-time notifications for job arrivals.
-     *
-     * Stops the notification mechanism to prevent further `onNotify` callbacks.
-     * Called during consumer shutdown.
-     *
-     * @returns A promise that resolves when notifications are disabled.
-     */
-    disableNotifications?(): Promise<void>;
-}