@nicnocquee/dataqueue 1.24.0 → 1.25.0

package/migrations/1751131910825_add_timeout_seconds_to_job_queue.sql

@@ -1,6 +1,6 @@
 -- Up Migration: Add timeout_ms and failure_reason to job_queue
-ALTER TABLE job_queue ADD COLUMN timeout_ms INT;
-ALTER TABLE job_queue ADD COLUMN failure_reason VARCHAR;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS timeout_ms INT;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS failure_reason VARCHAR;
 
 -- Down Migration: Remove timeout_ms and failure_reason from job_queue
 ALTER TABLE job_queue DROP COLUMN IF EXISTS timeout_ms;

package/migrations/1751186053000_add_job_events_table.sql

@@ -7,15 +7,19 @@ CREATE TABLE IF NOT EXISTS job_events (
   metadata JSONB
 );
 
-ALTER TABLE job_events ADD CONSTRAINT fk_job_events_job_queue FOREIGN KEY (job_id) REFERENCES job_queue(id) ON DELETE CASCADE;
-CREATE INDEX idx_job_events_job_id ON job_events(job_id);
-CREATE INDEX idx_job_events_event_type ON job_events(event_type);
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_job_events_job_queue') THEN
+    ALTER TABLE job_events ADD CONSTRAINT fk_job_events_job_queue FOREIGN KEY (job_id) REFERENCES job_queue(id) ON DELETE CASCADE;
+  END IF;
+END $$;
+CREATE INDEX IF NOT EXISTS idx_job_events_job_id ON job_events(job_id);
+CREATE INDEX IF NOT EXISTS idx_job_events_event_type ON job_events(event_type);
 
-ALTER TABLE job_queue ADD COLUMN completed_at TIMESTAMPTZ;
-ALTER TABLE job_queue ADD COLUMN started_at TIMESTAMPTZ;
-ALTER TABLE job_queue ADD COLUMN last_retried_at TIMESTAMPTZ;
-ALTER TABLE job_queue ADD COLUMN last_failed_at TIMESTAMPTZ;
-ALTER TABLE job_queue ADD COLUMN last_cancelled_at TIMESTAMPTZ;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS last_retried_at TIMESTAMPTZ;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS last_failed_at TIMESTAMPTZ;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS last_cancelled_at TIMESTAMPTZ;
 
 -- Down Migration
 DROP INDEX IF EXISTS idx_job_events_event_type;

package/migrations/1751984773000_add_tags_to_job_queue.sql

@@ -1,5 +1,5 @@
 -- Up Migration
-ALTER TABLE job_queue ADD COLUMN tags TEXT[];
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS tags TEXT[];
 CREATE INDEX IF NOT EXISTS idx_job_queue_tags ON job_queue USING GIN (tags);
 
 -- Down Migration

package/migrations/1765809419000_add_force_kill_on_timeout_to_job_queue.sql

@@ -1,5 +1,5 @@
 -- Up Migration: Add force_kill_on_timeout to job_queue
-ALTER TABLE job_queue ADD COLUMN force_kill_on_timeout BOOLEAN DEFAULT FALSE;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS force_kill_on_timeout BOOLEAN DEFAULT FALSE;
 
 -- Down Migration: Remove force_kill_on_timeout from job_queue
 ALTER TABLE job_queue DROP COLUMN IF EXISTS force_kill_on_timeout;

package/migrations/1771100000000_add_idempotency_key_to_job_queue.sql

@@ -0,0 +1,7 @@
+-- Up Migration: Add idempotency_key to job_queue
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS idempotency_key VARCHAR(255);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_job_queue_idempotency_key ON job_queue (idempotency_key) WHERE idempotency_key IS NOT NULL;
+
+-- Down Migration: Remove idempotency_key from job_queue
+DROP INDEX IF EXISTS idx_job_queue_idempotency_key;
+ALTER TABLE job_queue DROP COLUMN IF EXISTS idempotency_key;

package/migrations/1781200000000_add_wait_support.sql

@@ -0,0 +1,12 @@
+-- Up Migration: Add wait support columns to job_queue
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS wait_until TIMESTAMPTZ DEFAULT NULL;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS wait_token_id VARCHAR(255) DEFAULT NULL;
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS step_data JSONB DEFAULT '{}';
+
+CREATE INDEX IF NOT EXISTS idx_job_queue_wait_until ON job_queue (wait_until) WHERE status = 'waiting' AND wait_until IS NOT NULL;
+
+-- Down Migration: Remove wait support columns from job_queue
+DROP INDEX IF EXISTS idx_job_queue_wait_until;
+ALTER TABLE job_queue DROP COLUMN IF EXISTS wait_until;
+ALTER TABLE job_queue DROP COLUMN IF EXISTS wait_token_id;
+ALTER TABLE job_queue DROP COLUMN IF EXISTS step_data;

package/migrations/1781200000001_create_waitpoints_table.sql

@@ -0,0 +1,18 @@
+-- Up Migration: Create waitpoints table for token-based waits
+CREATE TABLE IF NOT EXISTS waitpoints (
+  id VARCHAR(255) PRIMARY KEY,
+  job_id INT REFERENCES job_queue(id) ON DELETE CASCADE,
+  status VARCHAR(20) NOT NULL DEFAULT 'waiting',
+  output JSONB DEFAULT NULL,
+  timeout_at TIMESTAMPTZ DEFAULT NULL,
+  created_at TIMESTAMPTZ DEFAULT NOW(),
+  completed_at TIMESTAMPTZ DEFAULT NULL,
+  tags TEXT[] DEFAULT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_waitpoints_job_id ON waitpoints(job_id);
+CREATE INDEX IF NOT EXISTS idx_waitpoints_status ON waitpoints(status);
+CREATE INDEX IF NOT EXISTS idx_waitpoints_timeout ON waitpoints(timeout_at) WHERE status = 'waiting';
+
+-- Down Migration: Drop waitpoints table
+DROP TABLE IF EXISTS waitpoints;

package/migrations/1781200000002_add_performance_indexes.sql

@@ -0,0 +1,34 @@
+-- Up Migration: Add composite and partial indexes for performance at scale
+
+-- Composite partial index for the getNextBatch claim query.
+-- Covers pending jobs ordered by priority then age.
+-- The run_at <= NOW() filter is applied at query time (NOW() is not IMMUTABLE).
+CREATE INDEX IF NOT EXISTS idx_job_queue_claimable
+  ON job_queue (priority DESC, created_at ASC)
+  WHERE status = 'pending';
+
+-- Partial index for failed jobs eligible for retry (used in getNextBatch).
+CREATE INDEX IF NOT EXISTS idx_job_queue_failed_retry
+  ON job_queue (next_attempt_at ASC)
+  WHERE status = 'failed' AND next_attempt_at IS NOT NULL;
+
+-- Index for reclaimStuckJobs: processing jobs ordered by lock age.
+CREATE INDEX IF NOT EXISTS idx_job_queue_stuck
+  ON job_queue (locked_at ASC)
+  WHERE status = 'processing';
+
+-- Index for cleanupOldJobs: completed jobs by update time.
+CREATE INDEX IF NOT EXISTS idx_job_queue_cleanup
+  ON job_queue (updated_at ASC)
+  WHERE status = 'completed';
+
+-- Index for job_events cleanup and time-based queries.
+CREATE INDEX IF NOT EXISTS idx_job_events_created_at
+  ON job_events (created_at ASC);
+
+-- Down Migration
+DROP INDEX IF EXISTS idx_job_queue_claimable;
+DROP INDEX IF EXISTS idx_job_queue_failed_retry;
+DROP INDEX IF EXISTS idx_job_queue_stuck;
+DROP INDEX IF EXISTS idx_job_queue_cleanup;
+DROP INDEX IF EXISTS idx_job_events_created_at;

package/migrations/1781200000003_add_progress_to_job_queue.sql

@@ -0,0 +1,7 @@
+-- Add progress column to job_queue for tracking job completion percentage (0-100).
+
+-- Up Migration
+ALTER TABLE job_queue ADD COLUMN IF NOT EXISTS progress INTEGER DEFAULT NULL;
+
+-- Down Migration
+ALTER TABLE job_queue DROP COLUMN IF EXISTS progress;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@nicnocquee/dataqueue",
-  "version": "1.24.0",
-  "description": "PostgreSQL-based job queue for Node.js applications with support for serverless environments",
+  "version": "1.25.0",
+  "description": "PostgreSQL or Redis-backed job queue for Node.js applications with support for serverless environments",
   "type": "module",
   "main": "dist/index.js",
   "exports": {
@@ -19,26 +19,31 @@
   "keywords": [
     "nextjs",
     "postgresql",
+    "redis",
     "job-queue",
     "background-jobs",
     "vercel"
   ],
   "author": "Nico Prananta",
   "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nicnocquee/dataqueue.git",
+    "directory": "packages/dataqueue"
+  },
   "dependencies": {
     "pg": "^8.0.0",
-    "pg-connection-string": "^2.9.1",
-    "ts-node": "^10.9.2"
+    "pg-connection-string": "^2.9.1"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.18.2",
     "@changesets/cli": "^2.29.5",
     "@types/node": "^24.0.4",
     "@types/pg": "^8.15.4",
-    "@vitejs/plugin-react": "^4.6.0",
-    "jsdom": "^26.1.0",
+    "ioredis": "^5.9.3",
     "node-pg-migrate": "^8.0.3",
     "pnpm": "^9.0.0",
+    "ts-node": "^10.9.2",
     "prettier": "^3.6.2",
     "tsup": "^8.5.0",
     "turbo": "^1.13.0",
@@ -47,9 +52,18 @@
     "vitest": "^3.2.4"
   },
   "peerDependencies": {
+    "ioredis": "^5.0.0",
     "node-pg-migrate": "^8.0.3",
     "pg": "^8.0.0"
   },
+  "peerDependenciesMeta": {
+    "ioredis": {
+      "optional": true
+    },
+    "node-pg-migrate": {
+      "optional": true
+    }
+  },
   "bin": {
     "dataqueue-cli": "./cli.cjs"
   },

package/src/backend.ts

@@ -0,0 +1,163 @@
+import {
+  JobOptions,
+  JobRecord,
+  JobEvent,
+  JobEventType,
+  FailureReason,
+  TagQueryMode,
+  JobType,
+} from './types.js';
+
+/**
+ * Filter options used by getJobs, cancelAllUpcomingJobs, editAllPendingJobs
+ */
+export interface JobFilters {
+  jobType?: string;
+  priority?: number;
+  runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
+  tags?: { values: string[]; mode?: TagQueryMode };
+  /**
+   * Cursor for keyset pagination. When provided, only return jobs with id < cursor.
+   * This is more efficient than OFFSET for large datasets.
+   * Cannot be used together with offset.
+   */
+  cursor?: number;
+}
+
+/**
+ * Fields that can be updated on a job
+ */
+export interface JobUpdates {
+  payload?: any;
+  maxAttempts?: number;
+  priority?: number;
+  runAt?: Date | null;
+  timeoutMs?: number | null;
+  tags?: string[] | null;
+}
+
+/**
+ * Abstract backend interface that both PostgreSQL and Redis implement.
+ * All storage operations go through this interface so the processor
+ * and public API are backend-agnostic.
+ */
+export interface QueueBackend {
+  // ── Job CRUD ──────────────────────────────────────────────────────────
+
+  /** Add a job and return its numeric ID. */
+  addJob<PayloadMap, T extends JobType<PayloadMap>>(
+    job: JobOptions<PayloadMap, T>,
+  ): Promise<number>;
+
+  /** Get a single job by ID, or null if not found. */
+  getJob<PayloadMap, T extends JobType<PayloadMap>>(
+    id: number,
+  ): Promise<JobRecord<PayloadMap, T> | null>;
+
+  /** Get jobs filtered by status, ordered by createdAt DESC. */
+  getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(
+    status: string,
+    limit?: number,
+    offset?: number,
+  ): Promise<JobRecord<PayloadMap, T>[]>;
+
+  /** Get all jobs, ordered by createdAt DESC. */
+  getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(
+    limit?: number,
+    offset?: number,
+  ): Promise<JobRecord<PayloadMap, T>[]>;
+
+  /** Get jobs matching arbitrary filters, ordered by createdAt DESC. */
+  getJobs<PayloadMap, T extends JobType<PayloadMap>>(
+    filters?: JobFilters,
+    limit?: number,
+    offset?: number,
+  ): Promise<JobRecord<PayloadMap, T>[]>;
+
+  /** Get jobs by tag(s) with query mode. */
+  getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(
+    tags: string[],
+    mode?: TagQueryMode,
+    limit?: number,
+    offset?: number,
+  ): Promise<JobRecord<PayloadMap, T>[]>;
+
+  // ── Processing lifecycle ──────────────────────────────────────────────
+
+  /**
+   * Atomically claim a batch of ready jobs for the given worker.
+   * Equivalent to SELECT … FOR UPDATE SKIP LOCKED in Postgres.
+   */
+  getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(
+    workerId: string,
+    batchSize?: number,
+    jobType?: string | string[],
+  ): Promise<JobRecord<PayloadMap, T>[]>;
+
+  /** Mark a job as completed. */
+  completeJob(jobId: number): Promise<void>;
+
+  /** Mark a job as failed with error info and schedule retry. */
+  failJob(
+    jobId: number,
+    error: Error,
+    failureReason?: FailureReason,
+  ): Promise<void>;
+
+  /** Update locked_at to keep the job alive (heartbeat). */
+  prolongJob(jobId: number): Promise<void>;
+
+  // ── Job management ────────────────────────────────────────────────────
+
+  /** Retry a failed/cancelled job immediately. */
+  retryJob(jobId: number): Promise<void>;
+
+  /** Cancel a pending job. */
+  cancelJob(jobId: number): Promise<void>;
+
+  /** Cancel all pending jobs matching optional filters. Returns count. */
+  cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
+
+  /** Edit a single pending job. */
+  editJob(jobId: number, updates: JobUpdates): Promise<void>;
+
+  /** Edit all pending jobs matching filters. Returns count. */
+  editAllPendingJobs(
+    filters: JobFilters | undefined,
+    updates: JobUpdates,
+  ): Promise<number>;
+
+  /** Delete completed jobs older than N days. Returns count deleted. */
+  cleanupOldJobs(daysToKeep?: number): Promise<number>;
+
+  /** Delete job events older than N days. Returns count deleted. */
+  cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+
+  /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
+  reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
+
+  // ── Progress ──────────────────────────────────────────────────────────
+
+  /** Update the progress percentage (0-100) for a job. */
+  updateProgress(jobId: number, progress: number): Promise<void>;
+
+  // ── Events ────────────────────────────────────────────────────────────
+
+  /** Record a job event. Should not throw. */
+  recordJobEvent(
+    jobId: number,
+    eventType: JobEventType,
+    metadata?: any,
+  ): Promise<void>;
+
+  /** Get all events for a job, ordered by createdAt ASC. */
+  getJobEvents(jobId: number): Promise<JobEvent[]>;
+
+  // ── Internal helpers ──────────────────────────────────────────────────
+
+  /** Set a pending reason for unpicked jobs of a given type. */
+  setPendingReasonForUnpickedJobs(
+    reason: string,
+    jobType?: string | string[],
+  ): Promise<void>;
+}