@nicnocquee/dataqueue 1.30.0 → 1.32.0

package/migrations/1781200000004_create_cron_schedules_table.sql

@@ -0,0 +1,33 @@
+-- Create cron_schedules table for recurring job definitions
+
+-- Up Migration
+CREATE TABLE IF NOT EXISTS cron_schedules (
+  id SERIAL PRIMARY KEY,
+  schedule_name VARCHAR(255) NOT NULL UNIQUE,
+  cron_expression VARCHAR(255) NOT NULL,
+  job_type VARCHAR(255) NOT NULL,
+  payload JSONB NOT NULL DEFAULT '{}',
+  max_attempts INT DEFAULT 3,
+  priority INT DEFAULT 0,
+  timeout_ms INT,
+  force_kill_on_timeout BOOLEAN DEFAULT FALSE,
+  tags TEXT[],
+  timezone VARCHAR(100) DEFAULT 'UTC',
+  allow_overlap BOOLEAN DEFAULT FALSE,
+  status VARCHAR(50) DEFAULT 'active',
+  last_enqueued_at TIMESTAMPTZ,
+  last_job_id INT,
+  next_run_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_cron_schedules_status ON cron_schedules(status);
+CREATE INDEX IF NOT EXISTS idx_cron_schedules_next_run_at ON cron_schedules(next_run_at);
+CREATE INDEX IF NOT EXISTS idx_cron_schedules_name ON cron_schedules(schedule_name);
+
+-- Down Migration
+DROP INDEX IF EXISTS idx_cron_schedules_name;
+DROP INDEX IF EXISTS idx_cron_schedules_next_run_at;
+DROP INDEX IF EXISTS idx_cron_schedules_status;
+DROP TABLE IF EXISTS cron_schedules;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nicnocquee/dataqueue",
-  "version": "1.30.0",
+  "version": "1.32.0",
   "description": "PostgreSQL or Redis-backed job queue for Node.js applications with support for serverless environments",
   "type": "module",
   "main": "dist/index.js",
@@ -43,6 +43,7 @@
     "directory": "packages/dataqueue"
   },
   "dependencies": {
+    "croner": "^10.0.1",
     "pg": "^8.0.0",
     "pg-connection-string": "^2.9.1"
   },
@@ -53,8 +54,8 @@
     "ioredis": "^5.9.3",
     "node-pg-migrate": "^8.0.3",
     "pnpm": "^9.0.0",
-    "ts-node": "^10.9.2",
     "prettier": "^3.6.2",
+    "ts-node": "^10.9.2",
     "tsup": "^8.5.0",
     "turbo": "^1.13.0",
     "typescript": "^5.8.3",

package/src/backend.ts

@@ -6,6 +6,11 @@ import {
   FailureReason,
   TagQueryMode,
   JobType,
+  CronScheduleRecord,
+  CronScheduleStatus,
+  EditCronScheduleOptions,
+  WaitpointRecord,
+  CreateTokenOptions,
 } from './types.js';
 
 /**
@@ -36,6 +41,25 @@ export interface JobUpdates {
   tags?: string[] | null;
 }
 
+/**
+ * Input shape for creating a cron schedule in the backend.
+ * This is the backend-level version of CronScheduleOptions.
+ */
+export interface CronScheduleInput {
+  scheduleName: string;
+  cronExpression: string;
+  jobType: string;
+  payload: any;
+  maxAttempts: number;
+  priority: number;
+  timeoutMs: number | null;
+  forceKillOnTimeout: boolean;
+  tags: string[] | undefined;
+  timezone: string;
+  allowOverlap: boolean;
+  nextRunAt: Date | null;
+}
+
 /**
  * Abstract backend interface that both PostgreSQL and Redis implement.
  * All storage operations go through this interface so the processor
@@ -127,11 +151,11 @@ export interface QueueBackend {
     updates: JobUpdates,
   ): Promise<number>;
 
-  /** Delete completed jobs older than N days. Returns count deleted. */
-  cleanupOldJobs(daysToKeep?: number): Promise<number>;
+  /** Delete completed jobs older than N days. Deletes in batches for scale safety. Returns count deleted. */
+  cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
 
-  /** Delete job events older than N days. Returns count deleted. */
-  cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+  /** Delete job events older than N days. Deletes in batches for scale safety. Returns count deleted. */
+  cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
 
   /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
   reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
@@ -153,6 +177,117 @@ export interface QueueBackend {
   /** Get all events for a job, ordered by createdAt ASC. */
   getJobEvents(jobId: number): Promise<JobEvent[]>;
 
+  // ── Cron schedules ──────────────────────────────────────────────────
+
+  /** Create a cron schedule and return its ID. */
+  addCronSchedule(input: CronScheduleInput): Promise<number>;
+
+  /** Get a cron schedule by ID, or null if not found. */
+  getCronSchedule(id: number): Promise<CronScheduleRecord | null>;
+
+  /** Get a cron schedule by its unique name, or null if not found. */
+  getCronScheduleByName(name: string): Promise<CronScheduleRecord | null>;
+
+  /** List cron schedules, optionally filtered by status. */
+  listCronSchedules(status?: CronScheduleStatus): Promise<CronScheduleRecord[]>;
+
+  /** Delete a cron schedule by ID. */
+  removeCronSchedule(id: number): Promise<void>;
+
+  /** Pause a cron schedule. */
+  pauseCronSchedule(id: number): Promise<void>;
+
+  /** Resume a cron schedule. */
+  resumeCronSchedule(id: number): Promise<void>;
+
+  /** Edit a cron schedule. */
+  editCronSchedule(
+    id: number,
+    updates: EditCronScheduleOptions,
+    nextRunAt?: Date | null,
+  ): Promise<void>;
+
+  /**
+   * Atomically fetch all active cron schedules whose nextRunAt <= now.
+   * In PostgreSQL this uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing.
+   */
+  getDueCronSchedules(): Promise<CronScheduleRecord[]>;
+
+  /**
+   * Update a cron schedule after a job has been enqueued.
+   * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
+   */
+  updateCronScheduleAfterEnqueue(
+    id: number,
+    lastEnqueuedAt: Date,
+    lastJobId: number,
+    nextRunAt: Date | null,
+  ): Promise<void>;
+
+  // ── Wait / step-data support ────────────────────────────────────────
+
+  /**
+   * Transition a job from 'processing' to 'waiting' status.
+   * Persists step data so the handler can resume from where it left off.
+   *
+   * @param jobId - The job to pause.
+   * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+   */
+  waitJob(
+    jobId: number,
+    options: {
+      waitUntil?: Date;
+      waitTokenId?: string;
+      stepData: Record<string, any>;
+    },
+  ): Promise<void>;
+
+  /**
+   * Persist step data for a job. Called after each `ctx.run()` step completes
+   * to save intermediate progress. Best-effort: should not throw.
+   *
+   * @param jobId - The job to update.
+   * @param stepData - The step data to persist.
+   */
+  updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+
+  /**
+   * Create a waitpoint token that can pause a job until an external signal completes it.
+   *
+   * @param jobId - The job ID to associate with the token (null if created outside a handler).
+   * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+   * @returns The created waitpoint with its unique ID.
+   */
+  createWaitpoint(
+    jobId: number | null,
+    options?: CreateTokenOptions,
+  ): Promise<{ id: string }>;
+
+  /**
+   * Complete a waitpoint token, optionally providing output data.
+   * Moves the associated job from 'waiting' back to 'pending' so it gets picked up.
+   *
+   * @param tokenId - The waitpoint token ID to complete.
+   * @param data - Optional data to pass to the waiting handler.
+   */
+  completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+
+  /**
+   * Retrieve a waitpoint token by its ID.
+   *
+   * @param tokenId - The waitpoint token ID to look up.
+   * @returns The waitpoint record, or null if not found.
+   */
+  getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+
+  /**
+   * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+   * Should be called periodically (e.g., alongside reclaimStuckJobs).
+   *
+   * @returns The number of tokens that were expired.
+   */
+  expireTimedOutWaitpoints(): Promise<number>;
+
   // ── Internal helpers ──────────────────────────────────────────────────
 
   /** Set a pending reason for unpicked jobs of a given type. */