@agendajs/postgres-backend 1.0.0-alpha.0

package/LICENSE.md

@@ -0,0 +1,25 @@
+## License
+(The MIT License)
+
+Copyright (c) 2013 Ryan Schmukler <ryan@slingingcode.com>
+
+Copyright (c) 2022-2026 Simon Tretter <s.tretter@gmail.com>
+
+Contributors: Mikael Korpela, Alexis Tyler, Vasyl Boroviak, Haris Sulaiman, Loris Guignard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the 'Software'), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,215 @@
+# @agendajs/postgres-backend
+
+PostgreSQL backend for [Agenda](https://github.com/agenda/agenda) job scheduler with LISTEN/NOTIFY support for real-time job processing.
+
+## Features
+
+- Full PostgreSQL storage backend for Agenda jobs
+- Real-time job notifications using PostgreSQL LISTEN/NOTIFY
+- Automatic schema creation with optimized indexes
+- Connection pooling via `pg` library
+- TypeScript support with full type definitions
+
+## Installation
+
+```bash
+npm install @agendajs/postgres-backend
+# or
+pnpm add @agendajs/postgres-backend
+# or
+yarn add @agendajs/postgres-backend
+```
+
+## Usage
+
+### Basic Usage
+
+```typescript
+import { Agenda } from 'agenda';
+import { PostgresBackend } from '@agendajs/postgres-backend';
+
+// Create agenda with PostgreSQL backend
+const agenda = new Agenda({
+  backend: new PostgresBackend({
+    connectionString: 'postgresql://user:pass@localhost:5432/mydb'
+  })
+});
+
+// Define jobs
+agenda.define('send-email', async (job) => {
+  const { to, subject, body } = job.attrs.data;
+  await sendEmail(to, subject, body);
+});
+
+// Start processing
+await agenda.start();
+
+// Schedule jobs
+await agenda.every('5 minutes', 'send-email', { to: 'user@example.com', subject: 'Hello', body: 'World' });
+await agenda.schedule('in 10 minutes', 'send-email', { to: 'other@example.com', subject: 'Hi', body: 'There' });
+```
+
+### Configuration Options
+
+```typescript
+import { PostgresBackend } from '@agendajs/postgres-backend';
+
+const backend = new PostgresBackend({
+  // PostgreSQL connection string (required unless pool/poolConfig is provided)
+  connectionString: 'postgresql://user:pass@localhost:5432/mydb',
+
+  // Or use pool configuration (creates a new pool)
+  poolConfig: {
+    host: 'localhost',
+    port: 5432,
+    database: 'mydb',
+    user: 'user',
+    password: 'pass',
+    max: 20 // max pool size
+  },
+
+  // Table name for jobs (default: 'agenda_jobs')
+  tableName: 'agenda_jobs',
+
+  // Channel name for LISTEN/NOTIFY (default: 'agenda_jobs')
+  channelName: 'agenda_jobs',
+
+  // Name to identify this Agenda instance (stored as lastModifiedBy)
+  name: 'worker-1',
+
+  // Whether to create the table and indexes on connect (default: true)
+  ensureSchema: true,
+
+  // Sort order for job queries
+  sort: {
+    nextRunAt: 1,  // 1 for ASC, -1 for DESC
+    priority: -1
+  }
+});
+```
+
+### Using an Existing Pool
+
+If your application already has a PostgreSQL connection pool, you can pass it directly. The pool will **not** be closed when Agenda disconnects:
+
+```typescript
+import { Pool } from 'pg';
+import { PostgresBackend } from '@agendajs/postgres-backend';
+
+// Your app's existing pool
+const pool = new Pool({ connectionString: 'postgresql://...' });
+
+const backend = new PostgresBackend({ pool });
+
+// After agenda.stop(), your pool is still usable
+```
+
+## How It Works
+
+### Storage
+
+The backend stores jobs in a PostgreSQL table with the following structure:
+
+- `id` - UUID primary key
+- `name` - Job name
+- `priority` - Job priority (higher = more urgent)
+- `next_run_at` - When the job should run next
+- `type` - Job type ('normal' or 'single')
+- `locked_at` - Lock timestamp for distributed execution
+- `data` - JSONB payload for job data
+- And other metadata fields...
+
+### Real-Time Notifications
+
+Unlike MongoDB, PostgreSQL has built-in pub/sub via LISTEN/NOTIFY. When a job is saved, a notification is published to all listening Agenda instances, triggering immediate job processing without waiting for the next poll interval.
+
+This means:
+- Lower latency for job execution
+- Reduced database polling overhead
+- Efficient cross-process coordination
+
+### Distributed Locking
+
+The backend uses PostgreSQL's `FOR UPDATE SKIP LOCKED` to atomically lock jobs for processing, preventing duplicate execution across multiple Agenda instances.
+
+## Database Indexes
+
+When `ensureSchema: true` (default), the following indexes are created:
+
+```sql
+-- Main index for finding and locking next job
+CREATE INDEX agenda_jobs_find_and_lock_idx
+  ON agenda_jobs (name, next_run_at, priority DESC, locked_at, disabled)
+  WHERE disabled = FALSE;
+
+-- Index for single jobs (upsert operations)
+CREATE UNIQUE INDEX agenda_jobs_single_job_idx
+  ON agenda_jobs (name) WHERE type = 'single';
+
+-- Index for locked_at (stale lock detection)
+CREATE INDEX agenda_jobs_locked_at_idx
+  ON agenda_jobs (locked_at) WHERE locked_at IS NOT NULL;
+
+-- Index for next_run_at queries
+CREATE INDEX agenda_jobs_next_run_at_idx
+  ON agenda_jobs (next_run_at) WHERE next_run_at IS NOT NULL;
+```
+
+## Manual Schema Management
+
+If you prefer to manage the schema yourself (set `ensureSchema: false`):
+
+```typescript
+import { getCreateTableSQL, getCreateIndexesSQL } from '@agendajs/postgres-backend';
+
+const tableName = 'agenda_jobs';
+
+// Create table
+await pool.query(getCreateTableSQL(tableName));
+
+// Create indexes
+for (const sql of getCreateIndexesSQL(tableName)) {
+  await pool.query(sql);
+}
+```
+
+## Testing
+
+Tests require a PostgreSQL database. The easiest way is to use Docker:
+
+```bash
+# Start PostgreSQL container and run tests
+pnpm test:docker
+
+# Or manually:
+pnpm docker:up        # Start PostgreSQL container
+pnpm test:postgres    # Run tests
+pnpm docker:down      # Stop container
+```
+
+You can also use an existing PostgreSQL database:
+
+```bash
+POSTGRES_TEST_URL=postgresql://user:pass@localhost:5432/agenda_test pnpm test
+```
+
+### Available Scripts
+
+| Script | Description |
+|--------|-------------|
+| `pnpm test` | Run tests (requires PostgreSQL or Docker) |
+| `pnpm test:postgres` | Run tests with local Docker PostgreSQL |
+| `pnpm test:docker` | Start container, run tests, stop container |
+| `pnpm docker:up` | Start PostgreSQL container |
+| `pnpm docker:down` | Stop PostgreSQL container |
+| `pnpm docker:logs` | View container logs |
+
+## Requirements
+
+- Node.js >= 18.0.0
+- PostgreSQL >= 12 (for `gen_random_uuid()` support)
+- Agenda >= 6.0.0
+
+## License
+
+MIT

package/dist/PostgresBackend.d.ts

@@ -0,0 +1,75 @@
+import type { AgendaBackend, JobRepository, NotificationChannel } from 'agenda';
+import type { PostgresBackendConfig } from './types.js';
+/**
+ * PostgreSQL backend for Agenda
+ *
+ * Provides both storage (via PostgresJobRepository) and real-time notifications
+ * (via PostgresNotificationChannel using LISTEN/NOTIFY).
+ *
+ * @example
+ * ```typescript
+ * import { Agenda } from 'agenda';
+ * import { PostgresBackend } from '@agendajs/postgres-backend';
+ *
+ * // Using connection string
+ * const agenda = new Agenda({
+ *   backend: new PostgresBackend({
+ *     connectionString: 'postgresql://user:pass@localhost:5432/mydb'
+ *   })
+ * });
+ *
+ * // Or using an existing pool (e.g., shared with your app)
+ * // The pool will NOT be closed when Agenda disconnects
+ * import { Pool } from 'pg';
+ * const pool = new Pool({ connectionString: '...' });
+ *
+ * const agenda = new Agenda({
+ *   backend: new PostgresBackend({ pool })
+ * });
+ *
+ * // Your app can continue using the pool after agenda.stop()
+ *
+ * agenda.define('myJob', async (job) => {
+ *   console.log('Running job:', job.attrs.name);
+ * });
+ *
+ * await agenda.start();
+ * await agenda.every('5 minutes', 'myJob');
+ * ```
+ */
+export declare class PostgresBackend implements AgendaBackend {
+    private _repository;
+    private _notificationChannel?;
+    private config;
+    private _ownsConnection;
+    constructor(config: PostgresBackendConfig);
+    /**
+     * The job repository for storage operations
+     */
+    get repository(): JobRepository;
+    /**
+     * Whether this backend owns its database connection.
+     * True if created from connectionString/poolConfig, false if pool was passed in.
+     */
+    get ownsConnection(): boolean;
+    /**
+     * The notification channel for real-time notifications via LISTEN/NOTIFY
+     * Returns undefined if notifications are disabled
+     */
+    get notificationChannel(): NotificationChannel | undefined;
+    /**
+     * Connect to PostgreSQL
+     *
+     * - Establishes database connection pool
+     * - Creates table and indexes if ensureSchema is true
+     * - Sets up LISTEN for real-time notifications
+     */
+    connect(): Promise<void>;
+    /**
+     * Disconnect from PostgreSQL
+     *
+     * - Stops LISTEN on notification channel
+     * - Closes database connection pool
+     */
+    disconnect(): Promise<void>;
+}

package/dist/PostgresBackend.js

@@ -0,0 +1,123 @@
+import debug from 'debug';
+import { PostgresJobRepository } from './PostgresJobRepository.js';
+import { PostgresNotificationChannel } from './PostgresNotificationChannel.js';
+const log = debug('agenda:postgres:backend');
+/**
+ * PostgreSQL backend for Agenda
+ *
+ * Provides both storage (via PostgresJobRepository) and real-time notifications
+ * (via PostgresNotificationChannel using LISTEN/NOTIFY).
+ *
+ * @example
+ * ```typescript
+ * import { Agenda } from 'agenda';
+ * import { PostgresBackend } from '@agendajs/postgres-backend';
+ *
+ * // Using connection string
+ * const agenda = new Agenda({
+ *   backend: new PostgresBackend({
+ *     connectionString: 'postgresql://user:pass@localhost:5432/mydb'
+ *   })
+ * });
+ *
+ * // Or using an existing pool (e.g., shared with your app)
+ * // The pool will NOT be closed when Agenda disconnects
+ * import { Pool } from 'pg';
+ * const pool = new Pool({ connectionString: '...' });
+ *
+ * const agenda = new Agenda({
+ *   backend: new PostgresBackend({ pool })
+ * });
+ *
+ * // Your app can continue using the pool after agenda.stop()
+ *
+ * agenda.define('myJob', async (job) => {
+ *   console.log('Running job:', job.attrs.name);
+ * });
+ *
+ * await agenda.start();
+ * await agenda.every('5 minutes', 'myJob');
+ * ```
+ */
+export class PostgresBackend {
+    _repository;
+    _notificationChannel;
+    config;
+    _ownsConnection;
+    constructor(config) {
+        this.config = config;
+        // Determine if we own the connection (not passed in by user)
+        this._ownsConnection = !config.pool;
+        // Create repository
+        this._repository = new PostgresJobRepository(config);
+        // Create notification channel (unless disabled)
+        if (!config.disableNotifications) {
+            this._notificationChannel = new PostgresNotificationChannel({
+                channelName: config.channelName,
+                reconnect: {
+                    enabled: true
+                }
+            });
+        }
+        log('PostgresBackend created with config: %O', {
+            tableName: config.tableName || 'agenda_jobs',
+            channelName: config.channelName || 'agenda_jobs',
+            ensureSchema: config.ensureSchema ?? true,
+            disableNotifications: config.disableNotifications ?? false,
+            ownsConnection: this._ownsConnection
+        });
+    }
+    /**
+     * The job repository for storage operations
+     */
+    get repository() {
+        return this._repository;
+    }
+    /**
+     * Whether this backend owns its database connection.
+     * True if created from connectionString/poolConfig, false if pool was passed in.
+     */
+    get ownsConnection() {
+        return this._ownsConnection;
+    }
+    /**
+     * The notification channel for real-time notifications via LISTEN/NOTIFY
+     * Returns undefined if notifications are disabled
+     */
+    get notificationChannel() {
+        return this._notificationChannel;
+    }
+    /**
+     * Connect to PostgreSQL
+     *
+     * - Establishes database connection pool
+     * - Creates table and indexes if ensureSchema is true
+     * - Sets up LISTEN for real-time notifications
+     */
+    async connect() {
+        log('connecting PostgresBackend');
+        // Connect repository first (creates pool and schema)
+        await this._repository.connect();
+        // Share the pool with notification channel (if enabled)
+        if (this._notificationChannel) {
+            this._notificationChannel.setPool(this._repository.getPool());
+        }
+        // Note: notification channel is connected by Agenda.start()
+        // when it's needed, so we don't connect it here
+        log('PostgresBackend connected');
+    }
+    /**
+     * Disconnect from PostgreSQL
+     *
+     * - Stops LISTEN on notification channel
+     * - Closes database connection pool
+     */
+    async disconnect() {
+        log('disconnecting PostgresBackend');
+        // Notification channel is disconnected by Agenda.stop()
+        // Repository disconnect will close the shared pool
+        await this._repository.disconnect();
+        log('PostgresBackend disconnected');
+    }
+}
+//# sourceMappingURL=PostgresBackend.js.map

package/dist/PostgresBackend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PostgresBackend.js","sourceRoot":"","sources":["../src/PostgresBackend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAG/E,MAAM,GAAG,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;AAE7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,OAAO,eAAe;IACnB,WAAW,CAAwB;IACnC,oBAAoB,CAA+B;IACnD,MAAM,CAAwB;IAC9B,eAAe,CAAU;IAEjC,YAAY,MAA6B;QACxC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,6DAA6D;QAC7D,IAAI,CAAC,eAAe,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAEpC,oBAAoB;QACpB,IAAI,CAAC,WAAW,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAErD,gDAAgD;QAChD,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,oBAAoB,GAAG,IAAI,2BAA2B,CAAC;gBAC3D,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,SAAS,EAAE;oBACV,OAAO,EAAE,IAAI;iBACb;aACD,CAAC,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,yCAAyC,EAAE;YAC9C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,aAAa;YAC5C,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,aAAa;YAChD,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;YACzC,oBAAoB,EAAE,MAAM,CAAC,oBAAoB,IAAI,KAAK;YAC1D,cAAc,EAAE,IAAI,CAAC,eAAe;SACpC,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,IAAI,cAAc;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IAClC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO;QACZ,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAElC,qDAAqD;QACrD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAEjC,wDAAwD;QACxD,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,4DAA4D;QAC5D,gDAAgD;QAEhD,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACf,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAErC,wDAAwD;QACxD,mDAAmD;QACnD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAEpC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IACrC,CAAC;CACD"}

package/dist/PostgresJobRepository.d.ts

@@ -0,0 +1,53 @@
+import { Pool } from 'pg';
+import type { JobRepository, JobRepositoryOptions, RemoveJobsOptions, JobParameters, JobId, JobsQueryOptions, JobsResult, JobsOverview } from 'agenda';
+import type { PostgresBackendConfig } from './types.js';
+/**
+ * PostgreSQL implementation of JobRepository
+ */
+export declare class PostgresJobRepository implements JobRepository {
+    private config;
+    private pool;
+    private ownPool;
+    private tableName;
+    private ensureSchema;
+    private sort;
+    private disconnected;
+    constructor(config: PostgresBackendConfig);
+    private createPool;
+    /**
+     * Get the underlying pool (for use by notification channel)
+     */
+    getPool(): Pool;
+    connect(): Promise<void>;
+    private createSchema;
+    disconnect(): Promise<void>;
+    /**
+     * Force close the pool - should only be called on application shutdown
+     * or when you're sure the backend won't be reused
+     */
+    close(): Promise<void>;
+    /**
+     * Convert PostgreSQL row to JobParameters
+     */
+    private rowToJob;
+    /**
+     * Convert SortDirection to SQL ASC/DESC
+     */
+    private toSqlDirection;
+    /**
+     * Convert sort options to PostgreSQL ORDER BY clause
+     */
+    private toOrderByClause;
+    getJobById(id: string): Promise<JobParameters | null>;
+    queryJobs(options?: JobsQueryOptions): Promise<JobsResult>;
+    getJobsOverview(): Promise<JobsOverview[]>;
+    getDistinctJobNames(): Promise<string[]>;
+    getQueueSize(): Promise<number>;
+    removeJobs(options: RemoveJobsOptions): Promise<number>;
+    unlockJob(job: JobParameters): Promise<void>;
+    unlockJobs(jobIds: (JobId | string)[]): Promise<void>;
+    lockJob(job: JobParameters, options: JobRepositoryOptions | undefined): Promise<JobParameters | undefined>;
+    getNextJobToRun(jobName: string, nextScanAt: Date, lockDeadline: Date, now: Date | undefined, options: JobRepositoryOptions | undefined): Promise<JobParameters | undefined>;
+    saveJobState(job: JobParameters, options: JobRepositoryOptions | undefined): Promise<void>;
+    saveJob<DATA = unknown>(job: JobParameters<DATA>, options: JobRepositoryOptions | undefined): Promise<JobParameters<DATA>>;
+}