@gravito/stream 1.0.0-alpha.6 → 1.0.0-beta.2

package/README.md

@@ -7,10 +7,14 @@ Lightweight, high-performance queueing for Gravito. Supports multiple storage dr
 ## Features
 
 - **Zero runtime overhead**: Thin wrappers that delegate to drivers
-- **Multi-driver support**: Memory, Database, Redis, Kafka, SQS
+- **Multi-driver support**: Memory, Database, Redis, Kafka, SQS, RabbitMQ
 - **Modular**: Install only the driver you need (core < 50KB)
 - **Embedded or standalone workers**: Run in-process during development or standalone in production
 - **AI-friendly**: Strong typing, clear JSDoc, and predictable APIs
+- **Custom Retry Strategies**: Built-in exponential backoff with per-job overrides
+- **Dead Letter Queue (DLQ)**: Automatic handling of permanently failed jobs, with retry and clear operations
+- **Priority Queues**: Assign priority (critical, high, low) to any job
+- **Rate Limiting**: Control job consumption rate per queue (requires Redis)
 
 ## Installation
 
@@ -37,6 +41,24 @@ export class SendWelcomeEmail extends Job {
 }
 ```
 
+### 3. Rate Limit & Priority (Optional)
+
+```typescript
+const queue = c.get('queue')
+
+// High priority job
+await queue.push(new SendWelcomeEmail(user.id))
+  .onQueue('emails')
+  .withPriority('high') // 'critical' | 'high' | 'default' | 'low'
+
+// Configure rate limits in Consumer
+const consumer = new Consumer(manager, {
+  rateLimits: {
+    emails: { limit: 10, window: 60 } // Max 10 jobs per minute
+  }
+})
+```
+
 ### 2. Enqueue a job
 
 ```typescript
@@ -72,17 +94,47 @@ const core = await PlanetCore.boot({
 
 ```typescript
 import { OrbitStream } from '@gravito/stream'
-import { OrbitDB } from '@gravito/db'
+
+// Create a database service adapter that implements DatabaseService interface
+const dbService = {
+  execute: async (sql, bindings) => yourDbClient.query(sql, bindings),
+  transaction: async (callback) => yourDbClient.transaction(callback),
+}
 
 const core = await PlanetCore.boot({
   orbits: [
-    OrbitDB.configure({ db: drizzleClient }),
     OrbitStream.configure({
       default: 'database',
       connections: {
         database: {
           driver: 'database',
-          table: 'jobs'
+          table: 'jobs',
+          dbService: dbService // Pass your database service
+        }
+      }
+    })
+  ]
+})
+```
+
+## RabbitMQ Driver Example
+
+```typescript
+import { OrbitStream } from '@gravito/stream'
+import amqp from 'amqplib'
+
+const connection = await amqp.connect('amqp://localhost')
+
+const core = await PlanetCore.boot({
+  orbits: [
+    OrbitStream.configure({
+      default: 'rabbitmq',
+      connections: {
+        rabbitmq: {
+          driver: 'rabbitmq',
+          client: connection,
+          exchange: 'gravito.events',
+          exchangeType: 'fanout'
         }
       }
     })
@@ -103,10 +155,37 @@ CREATE TABLE jobs (
   created_at TIMESTAMP NOT NULL DEFAULT NOW()
 );
 
+CREATE INDEX idx_jobs_queue_available ON jobs(queue, available_at);
+CREATE INDEX idx_jobs_reserved ON jobs(reserved_at);
 CREATE INDEX idx_jobs_queue_available ON jobs(queue, available_at);
 CREATE INDEX idx_jobs_reserved ON jobs(reserved_at);
 ```
 
+## Persistence and Audit Mode
+
+The `@gravito/stream` package supports an optional persistence layer (using SQLite or MySQL) for archiving job history and providing an audit trail.
+
+### Configuration
+
+```typescript
+OrbitStream.configure({
+  // ... other config
+  persistence: {
+    adapter: new SQLitePersistence(DB), // or MySQLPersistence
+    archiveCompleted: true, // Archive jobs when they complete successfully
+    archiveFailed: true,    // Archive jobs when they fail permanently
+    archiveEnqueued: true   // (Audit Mode) Archive jobs immediately when pushed
+  }
+})
+```
+
+### Audit Mode (`archiveEnqueued: true`)
+
+When Audit Mode is enabled, every job pushed to the queue is immediately written to the SQL archive with a `waiting` status. This happens in parallel with the main queue operation (Fire-and-Forget).
+
+- **Benefit**: Provides a complete audit trail. Even if the queue driver (e.g., Redis) crashes and loses data, the SQL archive will contain the record of the job being enqueued.
+- **Performance**: Designed to be non-blocking. The SQL write happens asynchronously and does not delay the `push()` operation.
+
 ## Standalone Worker
 
 ```bash
@@ -128,6 +207,13 @@ abstract class Job implements Queueable {
   onQueue(queue: string): this
   onConnection(connection: string): this
   delay(seconds: number): this
+  
+  /**
+   * Set retry backoff strategy.
+   * @param seconds - Initial delay in seconds
+   * @param multiplier - Multiplier for each subsequent attempt (default: 2)
+   */
+  backoff(seconds: number, multiplier = 2): this
 }
 ```
 
@@ -140,6 +226,8 @@ class QueueManager {
   async pop(queue?: string, connection?: string): Promise<Job | null>
   async size(queue?: string, connection?: string): Promise<number>
   async clear(queue?: string, connection?: string): Promise<void>
+  async complete(job: Job): Promise<void>
+  async fail(job: Job, error: Error): Promise<void>
   registerJobClasses(jobClasses: Array<new (...args: unknown[]) => Job>): void
 }
 ```
@@ -148,9 +236,10 @@ class QueueManager {
 
 - **MemoryDriver** - in-memory (development)
 - **DatabaseDriver** - PostgreSQL/MySQL/SQLite
-- **RedisDriver** - delayed jobs supported
+- **RedisDriver** - delayed jobs, priority queues, rate limiting, and DLQ support
 - **KafkaDriver** - topics and consumer groups
 - **SQSDriver** - standard/FIFO queues and long polling
+- **RabbitMQDriver** - exchanges, queues, and advanced confirm mode
 
 ## License