@agendajs/redis-backend 1.0.0-alpha.0

package/LICENSE.md

@@ -0,0 +1,25 @@
+## License
+(The MIT License)
+
+Copyright (c) 2013 Ryan Schmukler <ryan@slingingcode.com>
+
+Copyright (c) 2022-2026 Simon Tretter <s.tretter@gmail.com>
+
+Contributors: Mikael Korpela, Alexis Tyler, Vasyl Boroviak, Haris Sulaiman, Loris Guignard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the 'Software'), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,186 @@
+# @agendajs/redis-backend
+
+Redis backend for [Agenda](https://github.com/agenda/agenda) job scheduler with Pub/Sub support for real-time job processing.
+
+## Features
+
+- Full Redis storage backend for Agenda jobs
+- Real-time job notifications using Redis Pub/Sub
+- Efficient job indexing with sorted sets
+- Atomic job locking with WATCH/MULTI/EXEC
+- Connection pooling via `ioredis` library
+- TypeScript support with full type definitions
+
+## Installation
+
+```bash
+npm install @agendajs/redis-backend
+# or
+pnpm add @agendajs/redis-backend
+# or
+yarn add @agendajs/redis-backend
+```
+
+## Usage
+
+### Basic Usage
+
+```typescript
+import { Agenda } from 'agenda';
+import { RedisBackend } from '@agendajs/redis-backend';
+
+// Create agenda with Redis backend
+const agenda = new Agenda({
+  backend: new RedisBackend({
+    connectionString: 'redis://localhost:6379'
+  })
+});
+
+// Define jobs
+agenda.define('send-email', async (job) => {
+  const { to, subject, body } = job.attrs.data;
+  await sendEmail(to, subject, body);
+});
+
+// Start processing
+await agenda.start();
+
+// Schedule jobs
+await agenda.every('5 minutes', 'send-email', { to: 'user@example.com', subject: 'Hello', body: 'World' });
+await agenda.schedule('in 10 minutes', 'send-email', { to: 'other@example.com', subject: 'Hi', body: 'There' });
+```
+
+### Configuration Options
+
+```typescript
+import { RedisBackend } from '@agendajs/redis-backend';
+
+const backend = new RedisBackend({
+  // Redis connection string (required unless redis/redisOptions is provided)
+  connectionString: 'redis://localhost:6379',
+
+  // Or use Redis client options (creates a new client)
+  redisOptions: {
+    host: 'localhost',
+    port: 6379,
+    password: 'secret',
+    db: 0
+  },
+
+  // Key prefix for all Redis keys (default: 'agenda:')
+  keyPrefix: 'agenda:',
+
+  // Channel name for Pub/Sub notifications (default: 'agenda:notifications')
+  channelName: 'agenda:notifications',
+
+  // Name to identify this Agenda instance (stored as lastModifiedBy)
+  name: 'worker-1',
+
+  // Sort order for job queries
+  sort: {
+    nextRunAt: 1,  // 1 for ASC, -1 for DESC
+    priority: -1
+  }
+});
+```
+
+### Using an Existing Redis Client
+
+If your application already has a Redis client, you can pass it directly. The client will **not** be closed when Agenda disconnects:
+
+```typescript
+import { Redis } from 'ioredis';
+import { RedisBackend } from '@agendajs/redis-backend';
+
+// Your app's existing Redis client
+const redis = new Redis('redis://localhost:6379');
+
+const backend = new RedisBackend({ redis });
+
+// After agenda.stop(), your Redis client is still usable
+```
+
+## How It Works
+
+### Storage
+
+The backend stores jobs using Redis data structures:
+
+- **Hashes** (`{prefix}job:{id}`) - Store job data as key-value pairs
+- **Sets** (`{prefix}jobs:all`) - Track all job IDs
+- **Sets** (`{prefix}jobs:by_name:{name}`) - Index jobs by name for fast lookup
+- **Sorted Sets** (`{prefix}jobs:by_next_run_at`) - Index jobs by nextRunAt for efficient scheduling
+- **Strings** (`{prefix}jobs:single:{name}`) - Track single-type jobs for upsert operations
+
+### Real-Time Notifications
+
+Redis has built-in pub/sub capabilities. When a job is saved, a notification is published to all subscribing Agenda instances, triggering immediate job processing without waiting for the next poll interval.
+
+This means:
+- Lower latency for job execution
+- Reduced polling overhead
+- Efficient cross-process coordination
+
+### Distributed Locking
+
+The backend uses Redis WATCH/MULTI/EXEC transactions to atomically lock jobs for processing, preventing duplicate execution across multiple Agenda instances.
+
+## Persistence
+
+By default, Redis keeps data in memory. For production use with Agenda, configure Redis persistence:
+
+```bash
+# In redis.conf
+appendonly yes           # Enable AOF persistence
+appendfsync everysec     # Sync to disk every second
+```
+
+Or use RDB snapshots:
+
+```bash
+save 900 1      # Save after 900 seconds if at least 1 key changed
+save 300 10     # Save after 300 seconds if at least 10 keys changed
+save 60 10000   # Save after 60 seconds if at least 10000 keys changed
+```
+
+For maximum durability, enable both AOF and RDB.
+
+## Testing
+
+Tests require a Redis server. The easiest way is to use Docker:
+
+```bash
+# Start Redis container and run tests
+pnpm test:docker
+
+# Or manually:
+pnpm docker:up      # Start Redis container
+pnpm test           # Run tests
+pnpm docker:down    # Stop container
+```
+
+You can also use an existing Redis server:
+
+```bash
+REDIS_TEST_URL=redis://localhost:6379 pnpm test
+```
+
+### Available Scripts
+
+| Script | Description |
+|--------|-------------|
+| `pnpm test` | Run tests (requires Redis or Docker) |
+| `pnpm test:docker` | Start container, run tests, stop container |
+| `pnpm docker:up` | Start Redis container |
+| `pnpm docker:down` | Stop Redis container |
+| `pnpm docker:logs` | View container logs |
+
+## Requirements
+
+- Node.js >= 18.0.0
+- Redis >= 6.0
+- Agenda >= 6.0.0
+
+## License
+
+MIT

package/dist/RedisBackend.d.ts

@@ -0,0 +1,73 @@
+import type { AgendaBackend, JobRepository, NotificationChannel } from 'agenda';
+import type { RedisBackendConfig } from './types.js';
+/**
+ * Redis backend for Agenda
+ *
+ * Provides both storage (via RedisJobRepository) and real-time notifications
+ * (via RedisNotificationChannel using Pub/Sub).
+ *
+ * @example
+ * ```typescript
+ * import { Agenda } from 'agenda';
+ * import { RedisBackend } from '@agendajs/redis-backend';
+ *
+ * // Using connection string
+ * const agenda = new Agenda({
+ *   backend: new RedisBackend({
+ *     connectionString: 'redis://localhost:6379'
+ *   })
+ * });
+ *
+ * // Or using an existing Redis client (e.g., shared with your app)
+ * // The client will NOT be closed when Agenda disconnects
+ * import Redis from 'ioredis';
+ * const redis = new Redis();
+ *
+ * const agenda = new Agenda({
+ *   backend: new RedisBackend({ redis })
+ * });
+ *
+ * // Your app can continue using the client after agenda.stop()
+ *
+ * agenda.define('myJob', async (job) => {
+ *   console.log('Running job:', job.attrs.name);
+ * });
+ *
+ * await agenda.start();
+ * await agenda.every('5 minutes', 'myJob');
+ * ```
+ */
+export declare class RedisBackend implements AgendaBackend {
+    private _repository;
+    private _notificationChannel;
+    private config;
+    private _ownsConnection;
+    constructor(config: RedisBackendConfig);
+    /**
+     * The job repository for storage operations
+     */
+    get repository(): JobRepository;
+    /**
+     * The notification channel for real-time notifications via Pub/Sub
+     */
+    get notificationChannel(): NotificationChannel;
+    /**
+     * Whether this backend owns its Redis connection.
+     * True if created from connectionString/redisOptions, false if redis client was passed in.
+     */
+    get ownsConnection(): boolean;
+    /**
+     * Connect to Redis
+     *
+     * - Establishes Redis connection
+     * - Sets up Pub/Sub for real-time notifications
+     */
+    connect(): Promise<void>;
+    /**
+     * Disconnect from Redis
+     *
+     * - Unsubscribes from Pub/Sub channel
+     * - Closes Redis connection
+     */
+    disconnect(): Promise<void>;
+}

package/dist/RedisBackend.js

@@ -0,0 +1,114 @@
+import debug from 'debug';
+import { RedisJobRepository } from './RedisJobRepository.js';
+import { RedisNotificationChannel } from './RedisNotificationChannel.js';
+const log = debug('agenda:redis:backend');
+/**
+ * Redis backend for Agenda
+ *
+ * Provides both storage (via RedisJobRepository) and real-time notifications
+ * (via RedisNotificationChannel using Pub/Sub).
+ *
+ * @example
+ * ```typescript
+ * import { Agenda } from 'agenda';
+ * import { RedisBackend } from '@agendajs/redis-backend';
+ *
+ * // Using connection string
+ * const agenda = new Agenda({
+ *   backend: new RedisBackend({
+ *     connectionString: 'redis://localhost:6379'
+ *   })
+ * });
+ *
+ * // Or using an existing Redis client (e.g., shared with your app)
+ * // The client will NOT be closed when Agenda disconnects
+ * import Redis from 'ioredis';
+ * const redis = new Redis();
+ *
+ * const agenda = new Agenda({
+ *   backend: new RedisBackend({ redis })
+ * });
+ *
+ * // Your app can continue using the client after agenda.stop()
+ *
+ * agenda.define('myJob', async (job) => {
+ *   console.log('Running job:', job.attrs.name);
+ * });
+ *
+ * await agenda.start();
+ * await agenda.every('5 minutes', 'myJob');
+ * ```
+ */
+export class RedisBackend {
+    _repository;
+    _notificationChannel;
+    config;
+    _ownsConnection;
+    constructor(config) {
+        this.config = config;
+        // Determine if we own the connection (not passed in by user)
+        this._ownsConnection = !config.redis;
+        // Create repository
+        this._repository = new RedisJobRepository(config);
+        // Create notification channel
+        this._notificationChannel = new RedisNotificationChannel({
+            channelName: config.channelName,
+            reconnect: {
+                enabled: true
+            }
+        });
+        log('RedisBackend created with config: %O', {
+            keyPrefix: config.keyPrefix || 'agenda:',
+            channelName: config.channelName || 'agenda:jobs'
+        });
+    }
+    /**
+     * The job repository for storage operations
+     */
+    get repository() {
+        return this._repository;
+    }
+    /**
+     * The notification channel for real-time notifications via Pub/Sub
+     */
+    get notificationChannel() {
+        return this._notificationChannel;
+    }
+    /**
+     * Whether this backend owns its Redis connection.
+     * True if created from connectionString/redisOptions, false if redis client was passed in.
+     */
+    get ownsConnection() {
+        return this._ownsConnection;
+    }
+    /**
+     * Connect to Redis
+     *
+     * - Establishes Redis connection
+     * - Sets up Pub/Sub for real-time notifications
+     */
+    async connect() {
+        log('connecting RedisBackend');
+        // Connect repository first
+        await this._repository.connect();
+        // Share the Redis client with notification channel
+        this._notificationChannel.setRedis(this._repository.getRedis());
+        // Note: notification channel is connected by Agenda.start()
+        // when it's needed, so we don't connect it here
+        log('RedisBackend connected');
+    }
+    /**
+     * Disconnect from Redis
+     *
+     * - Unsubscribes from Pub/Sub channel
+     * - Closes Redis connection
+     */
+    async disconnect() {
+        log('disconnecting RedisBackend');
+        // Notification channel is disconnected by Agenda.stop()
+        // Repository disconnect will close the shared client
+        await this._repository.disconnect();
+        log('RedisBackend disconnected');
+    }
+}
+//# sourceMappingURL=RedisBackend.js.map

package/dist/RedisBackend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RedisBackend.js","sourceRoot":"","sources":["../src/RedisBackend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAGzE,MAAM,GAAG,GAAG,KAAK,CAAC,sBAAsB,CAAC,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,OAAO,YAAY;IAChB,WAAW,CAAqB;IAChC,oBAAoB,CAA2B;IAC/C,MAAM,CAAqB;IAC3B,eAAe,CAAU;IAEjC,YAAY,MAA0B;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,6DAA6D;QAC7D,IAAI,CAAC,eAAe,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;QAErC,oBAAoB;QACpB,IAAI,CAAC,WAAW,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAElD,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,GAAG,IAAI,wBAAwB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,SAAS,EAAE;gBACV,OAAO,EAAE,IAAI;aACb;SACD,CAAC,CAAC;QAEH,GAAG,CAAC,sCAAsC,EAAE;YAC3C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;YACxC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,aAAa;SAChD,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,IAAI,cAAc;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACZ,GAAG,CAAC,yBAAyB,CAAC,CAAC;QAE/B,2BAA2B;QAC3B,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,mDAAmD;QACnD,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEhE,4DAA4D;QAC5D,gDAAgD;QAEhD,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACf,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAElC,wDAAwD;QACxD,qDAAqD;QACrD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAEpC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAClC,CAAC;CACD"}

package/dist/RedisJobRepository.d.ts

@@ -0,0 +1,75 @@
+import { Redis } from 'ioredis';
+import type { JobRepository, JobRepositoryOptions, RemoveJobsOptions, JobParameters, JobId, JobsQueryOptions, JobsResult, JobsOverview } from 'agenda';
+import type { RedisBackendConfig } from './types.js';
+/**
+ * Redis implementation of JobRepository
+ *
+ * Data structure:
+ * - `{prefix}job:{id}` - Hash containing job data
+ * - `{prefix}jobs:all` - Set of all job IDs
+ * - `{prefix}jobs:by_name:{name}` - Set of job IDs by name
+ * - `{prefix}jobs:by_next_run_at` - Sorted set of job IDs by nextRunAt timestamp
+ * - `{prefix}jobs:single:{name}` - String storing ID of single-type job for a name
+ */
+export declare class RedisJobRepository implements JobRepository {
+    private config;
+    private redis;
+    private ownClient;
+    private keyPrefix;
+    private sort;
+    constructor(config: RedisBackendConfig);
+    /**
+     * Get the underlying Redis client (for use by notification channel)
+     */
+    getRedis(): Redis;
+    /**
+     * Generate Redis key with prefix
+     */
+    private key;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    /**
+     * Convert Redis hash data to JobParameters
+     */
+    private hashToJob;
+    /**
+     * Convert JobParameters to Redis hash data
+     */
+    private jobToHash;
+    /**
+     * Get score for sorted set based on nextRunAt and priority
+     * Score format: timestamp.priority (e.g., 1234567890.005 for priority 5)
+     */
+    private getJobScore;
+    getJobById(id: string): Promise<JobParameters | null>;
+    queryJobs(options?: JobsQueryOptions): Promise<JobsResult>;
+    private sortJobs;
+    getJobsOverview(): Promise<JobsOverview[]>;
+    getDistinctJobNames(): Promise<string[]>;
+    getQueueSize(): Promise<number>;
+    removeJobs(options: RemoveJobsOptions): Promise<number>;
+    private deleteJob;
+    unlockJob(job: JobParameters): Promise<void>;
+    unlockJobs(jobIds: (JobId | string)[]): Promise<void>;
+    lockJob(job: JobParameters, options: JobRepositoryOptions | undefined): Promise<JobParameters | undefined>;
+    /**
+     * Lua script for atomic find-and-lock operation.
+     * This ensures that only one worker can lock a job, even under concurrent access.
+     *
+     * Arguments:
+     * KEYS[1] = jobs:by_name:{jobName} set
+     * KEYS[2] = jobs:by_next_run_at sorted set
+     * KEYS[3] = job:{id} hash prefix (without the id)
+     * ARGV[1] = nextScanAt timestamp (ms)
+     * ARGV[2] = lockDeadline timestamp (ms)
+     * ARGV[3] = lockTime ISO string
+     * ARGV[4] = lastModifiedBy
+     * ARGV[5] = sort direction ('asc' or 'desc')
+     *
+     * Returns: job ID if locked, nil if no job available
+     */
+    private static readonly FIND_AND_LOCK_SCRIPT;
+    getNextJobToRun(jobName: string, nextScanAt: Date, lockDeadline: Date, now: Date | undefined, options: JobRepositoryOptions | undefined): Promise<JobParameters | undefined>;
+    saveJobState(job: JobParameters, options: JobRepositoryOptions | undefined): Promise<void>;
+    saveJob<DATA = unknown>(job: JobParameters<DATA>, options: JobRepositoryOptions | undefined): Promise<JobParameters<DATA>>;
+}