@nicnocquee/dataqueue 1.30.0 → 1.32.0

package/src/backends/postgres.ts

@@ -7,10 +7,58 @@ import {
   JobEventType,
   TagQueryMode,
   JobType,
+  CronScheduleRecord,
+  CronScheduleStatus,
+  EditCronScheduleOptions,
+  WaitpointRecord,
+  CreateTokenOptions,
 } from '../types.js';
-import { QueueBackend, JobFilters, JobUpdates } from '../backend.js';
+import { randomUUID } from 'crypto';
+import {
+  QueueBackend,
+  JobFilters,
+  JobUpdates,
+  CronScheduleInput,
+} from '../backend.js';
 import { log } from '../log-context.js';
 
+const MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1000;
+
+/** Parse a timeout string like '10m', '1h', '24h', '7d' into milliseconds. */
+function parseTimeoutString(timeout: string): number {
+  const match = timeout.match(/^(\d+)(s|m|h|d)$/);
+  if (!match) {
+    throw new Error(
+      `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`,
+    );
+  }
+  const value = parseInt(match[1], 10);
+  const unit = match[2];
+  let ms: number;
+  switch (unit) {
+    case 's':
+      ms = value * 1000;
+      break;
+    case 'm':
+      ms = value * 60 * 1000;
+      break;
+    case 'h':
+      ms = value * 60 * 60 * 1000;
+      break;
+    case 'd':
+      ms = value * 24 * 60 * 60 * 1000;
+      break;
+    default:
+      throw new Error(`Unknown timeout unit: "${unit}"`);
+  }
+  if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
+    throw new Error(
+      `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`,
+    );
+  }
+  return ms;
+}
+
 export class PostgresBackend implements QueueBackend {
   constructor(private pool: Pool) {}
 
@@ -979,46 +1027,93 @@ export class PostgresBackend implements QueueBackend {
     }
   }
 
-  async cleanupOldJobs(daysToKeep = 30): Promise<number> {
-    const client = await this.pool.connect();
+  /**
+   * Delete completed jobs older than the given number of days.
+   * Deletes in batches of 1000 to avoid long-running transactions
+   * and excessive WAL bloat at scale.
+   *
+   * @param daysToKeep - Number of days to retain completed jobs (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000).
+   * @returns Total number of deleted jobs.
+   */
+  async cleanupOldJobs(daysToKeep = 30, batchSize = 1000): Promise<number> {
+    let totalDeleted = 0;
+
     try {
-      const result = await client.query(
-        `
-        DELETE FROM job_queue
-        WHERE status = 'completed'
-        AND updated_at < NOW() - INTERVAL '1 day' * $1::int
-        RETURNING id
-      `,
-        [daysToKeep],
-      );
-      log(`Deleted ${result.rowCount} old jobs`);
-      return result.rowCount || 0;
+      let deletedInBatch: number;
+      do {
+        const client = await this.pool.connect();
+        try {
+          const result = await client.query(
+            `
+            DELETE FROM job_queue
+            WHERE id IN (
+              SELECT id FROM job_queue
+              WHERE status = 'completed'
+              AND updated_at < NOW() - INTERVAL '1 day' * $1::int
+              LIMIT $2
+            )
+          `,
+            [daysToKeep, batchSize],
+          );
+          deletedInBatch = result.rowCount || 0;
+          totalDeleted += deletedInBatch;
+        } finally {
+          client.release();
+        }
+      } while (deletedInBatch === batchSize);
+
+      log(`Deleted ${totalDeleted} old jobs`);
+      return totalDeleted;
     } catch (error) {
       log(`Error cleaning up old jobs: ${error}`);
       throw error;
-    } finally {
-      client.release();
     }
   }
 
-  async cleanupOldJobEvents(daysToKeep = 30): Promise<number> {
-    const client = await this.pool.connect();
+  /**
+   * Delete job events older than the given number of days.
+   * Deletes in batches of 1000 to avoid long-running transactions
+   * and excessive WAL bloat at scale.
+   *
+   * @param daysToKeep - Number of days to retain events (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000).
+   * @returns Total number of deleted events.
+   */
+  async cleanupOldJobEvents(
+    daysToKeep = 30,
+    batchSize = 1000,
+  ): Promise<number> {
+    let totalDeleted = 0;
+
     try {
-      const result = await client.query(
-        `
-        DELETE FROM job_events
-        WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
-        RETURNING id
-      `,
-        [daysToKeep],
-      );
-      log(`Deleted ${result.rowCount} old job events`);
-      return result.rowCount || 0;
+      let deletedInBatch: number;
+      do {
+        const client = await this.pool.connect();
+        try {
+          const result = await client.query(
+            `
+            DELETE FROM job_events
+            WHERE id IN (
+              SELECT id FROM job_events
+              WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
+              LIMIT $2
+            )
+          `,
+            [daysToKeep, batchSize],
+          );
+          deletedInBatch = result.rowCount || 0;
+          totalDeleted += deletedInBatch;
+        } finally {
+          client.release();
+        }
+      } while (deletedInBatch === batchSize);
+
+      log(`Deleted ${totalDeleted} old job events`);
+      return totalDeleted;
     } catch (error) {
       log(`Error cleaning up old job events: ${error}`);
       throw error;
-    } finally {
-      client.release();
     }
   }
 
@@ -1083,6 +1178,557 @@ export class PostgresBackend implements QueueBackend {
     }
   }
 
+  // ── Cron schedules ──────────────────────────────────────────────────
+
+  /** Create a cron schedule and return its ID. */
+  async addCronSchedule(input: CronScheduleInput): Promise<number> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `INSERT INTO cron_schedules
+          (schedule_name, cron_expression, job_type, payload, max_attempts,
+           priority, timeout_ms, force_kill_on_timeout, tags, timezone,
+           allow_overlap, next_run_at)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+         RETURNING id`,
+        [
+          input.scheduleName,
+          input.cronExpression,
+          input.jobType,
+          input.payload,
+          input.maxAttempts,
+          input.priority,
+          input.timeoutMs,
+          input.forceKillOnTimeout,
+          input.tags ?? null,
+          input.timezone,
+          input.allowOverlap,
+          input.nextRunAt,
+        ],
+      );
+      const id = result.rows[0].id;
+      log(`Added cron schedule ${id}: "${input.scheduleName}"`);
+      return id;
+    } catch (error: any) {
+      // Unique constraint violation on schedule_name
+      if (error?.code === '23505') {
+        throw new Error(
+          `Cron schedule with name "${input.scheduleName}" already exists`,
+        );
+      }
+      log(`Error adding cron schedule: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Get a cron schedule by ID. */
+  async getCronSchedule(id: number): Promise<CronScheduleRecord | null> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
+                job_type AS "jobType", payload, max_attempts AS "maxAttempts",
+                priority, timeout_ms AS "timeoutMs",
+                force_kill_on_timeout AS "forceKillOnTimeout", tags,
+                timezone, allow_overlap AS "allowOverlap", status,
+                last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
+                next_run_at AS "nextRunAt",
+                created_at AS "createdAt", updated_at AS "updatedAt"
+         FROM cron_schedules WHERE id = $1`,
+        [id],
+      );
+      if (result.rows.length === 0) return null;
+      return result.rows[0] as CronScheduleRecord;
+    } catch (error) {
+      log(`Error getting cron schedule ${id}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Get a cron schedule by its unique name. */
+  async getCronScheduleByName(
+    name: string,
+  ): Promise<CronScheduleRecord | null> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
+                job_type AS "jobType", payload, max_attempts AS "maxAttempts",
+                priority, timeout_ms AS "timeoutMs",
+                force_kill_on_timeout AS "forceKillOnTimeout", tags,
+                timezone, allow_overlap AS "allowOverlap", status,
+                last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
+                next_run_at AS "nextRunAt",
+                created_at AS "createdAt", updated_at AS "updatedAt"
+         FROM cron_schedules WHERE schedule_name = $1`,
+        [name],
+      );
+      if (result.rows.length === 0) return null;
+      return result.rows[0] as CronScheduleRecord;
+    } catch (error) {
+      log(`Error getting cron schedule by name "${name}": ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** List cron schedules, optionally filtered by status. */
+  async listCronSchedules(
+    status?: CronScheduleStatus,
+  ): Promise<CronScheduleRecord[]> {
+    const client = await this.pool.connect();
+    try {
+      let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
+                job_type AS "jobType", payload, max_attempts AS "maxAttempts",
+                priority, timeout_ms AS "timeoutMs",
+                force_kill_on_timeout AS "forceKillOnTimeout", tags,
+                timezone, allow_overlap AS "allowOverlap", status,
+                last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
+                next_run_at AS "nextRunAt",
+                created_at AS "createdAt", updated_at AS "updatedAt"
+         FROM cron_schedules`;
+      const params: any[] = [];
+      if (status) {
+        query += ` WHERE status = $1`;
+        params.push(status);
+      }
+      query += ` ORDER BY created_at ASC`;
+      const result = await client.query(query, params);
+      return result.rows as CronScheduleRecord[];
+    } catch (error) {
+      log(`Error listing cron schedules: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Delete a cron schedule by ID. */
+  async removeCronSchedule(id: number): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
+      log(`Removed cron schedule ${id}`);
+    } catch (error) {
+      log(`Error removing cron schedule ${id}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Pause a cron schedule. */
+  async pauseCronSchedule(id: number): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(
+        `UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
+        [id],
+      );
+      log(`Paused cron schedule ${id}`);
+    } catch (error) {
+      log(`Error pausing cron schedule ${id}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Resume a paused cron schedule. */
+  async resumeCronSchedule(id: number): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(
+        `UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
+        [id],
+      );
+      log(`Resumed cron schedule ${id}`);
+    } catch (error) {
+      log(`Error resuming cron schedule ${id}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Edit a cron schedule. */
+  async editCronSchedule(
+    id: number,
+    updates: EditCronScheduleOptions,
+    nextRunAt?: Date | null,
+  ): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      const updateFields: string[] = [];
+      const params: any[] = [];
+      let paramIdx = 1;
+
+      if (updates.cronExpression !== undefined) {
+        updateFields.push(`cron_expression = $${paramIdx++}`);
+        params.push(updates.cronExpression);
+      }
+      if (updates.payload !== undefined) {
+        updateFields.push(`payload = $${paramIdx++}`);
+        params.push(updates.payload);
+      }
+      if (updates.maxAttempts !== undefined) {
+        updateFields.push(`max_attempts = $${paramIdx++}`);
+        params.push(updates.maxAttempts);
+      }
+      if (updates.priority !== undefined) {
+        updateFields.push(`priority = $${paramIdx++}`);
+        params.push(updates.priority);
+      }
+      if (updates.timeoutMs !== undefined) {
+        updateFields.push(`timeout_ms = $${paramIdx++}`);
+        params.push(updates.timeoutMs);
+      }
+      if (updates.forceKillOnTimeout !== undefined) {
+        updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
+        params.push(updates.forceKillOnTimeout);
+      }
+      if (updates.tags !== undefined) {
+        updateFields.push(`tags = $${paramIdx++}`);
+        params.push(updates.tags);
+      }
+      if (updates.timezone !== undefined) {
+        updateFields.push(`timezone = $${paramIdx++}`);
+        params.push(updates.timezone);
+      }
+      if (updates.allowOverlap !== undefined) {
+        updateFields.push(`allow_overlap = $${paramIdx++}`);
+        params.push(updates.allowOverlap);
+      }
+      if (nextRunAt !== undefined) {
+        updateFields.push(`next_run_at = $${paramIdx++}`);
+        params.push(nextRunAt);
+      }
+
+      if (updateFields.length === 0) {
+        log(`No fields to update for cron schedule ${id}`);
+        return;
+      }
+
+      updateFields.push(`updated_at = NOW()`);
+      params.push(id);
+
+      const query = `UPDATE cron_schedules SET ${updateFields.join(', ')} WHERE id = $${paramIdx}`;
+      await client.query(query, params);
+      log(`Edited cron schedule ${id}`);
+    } catch (error) {
+      log(`Error editing cron schedule ${id}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
+   * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
+   */
+  async getDueCronSchedules(): Promise<CronScheduleRecord[]> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
+                job_type AS "jobType", payload, max_attempts AS "maxAttempts",
+                priority, timeout_ms AS "timeoutMs",
+                force_kill_on_timeout AS "forceKillOnTimeout", tags,
+                timezone, allow_overlap AS "allowOverlap", status,
+                last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
+                next_run_at AS "nextRunAt",
+                created_at AS "createdAt", updated_at AS "updatedAt"
+         FROM cron_schedules
+         WHERE status = 'active'
+           AND next_run_at IS NOT NULL
+           AND next_run_at <= NOW()
+         ORDER BY next_run_at ASC
+         FOR UPDATE SKIP LOCKED`,
+      );
+      log(`Found ${result.rows.length} due cron schedules`);
+      return result.rows as CronScheduleRecord[];
+    } catch (error: any) {
+      // 42P01 = undefined_table — cron migration hasn't been run yet
+      if (error?.code === '42P01') {
+        log('cron_schedules table does not exist, skipping cron enqueue');
+        return [];
+      }
+      log(`Error getting due cron schedules: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Update a cron schedule after a job has been enqueued.
+   * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
+   */
+  async updateCronScheduleAfterEnqueue(
+    id: number,
+    lastEnqueuedAt: Date,
+    lastJobId: number,
+    nextRunAt: Date | null,
+  ): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(
+        `UPDATE cron_schedules
+         SET last_enqueued_at = $2,
+             last_job_id = $3,
+             next_run_at = $4,
+             updated_at = NOW()
+         WHERE id = $1`,
+        [id, lastEnqueuedAt, lastJobId, nextRunAt],
+      );
+      log(
+        `Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? 'null'}`,
+      );
+    } catch (error) {
+      log(`Error updating cron schedule ${id} after enqueue: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  // ── Wait / step-data support ────────────────────────────────────────
+
+  /**
+   * Transition a job from 'processing' to 'waiting' status.
+   * Persists step data so the handler can resume from where it left off.
+   *
+   * @param jobId - The job to pause.
+   * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+   */
+  async waitJob(
+    jobId: number,
+    options: {
+      waitUntil?: Date;
+      waitTokenId?: string;
+      stepData: Record<string, any>;
+    },
+  ): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `
+        UPDATE job_queue
+        SET status = 'waiting',
+            wait_until = $2,
+            wait_token_id = $3,
+            step_data = $4,
+            locked_at = NULL,
+            locked_by = NULL,
+            updated_at = NOW()
+        WHERE id = $1 AND status = 'processing'
+      `,
+        [
+          jobId,
+          options.waitUntil ?? null,
+          options.waitTokenId ?? null,
+          JSON.stringify(options.stepData),
+        ],
+      );
+      if (result.rowCount === 0) {
+        log(
+          `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`,
+        );
+        return;
+      }
+      await this.recordJobEvent(jobId, JobEventType.Waiting, {
+        waitUntil: options.waitUntil?.toISOString() ?? null,
+        waitTokenId: options.waitTokenId ?? null,
+      });
+      log(`Job ${jobId} set to waiting`);
+    } catch (error) {
+      log(`Error setting job ${jobId} to waiting: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Persist step data for a job. Called after each ctx.run() step completes.
+   * Best-effort: does not throw to avoid killing the running handler.
+   *
+   * @param jobId - The job to update.
+   * @param stepData - The step data to persist.
+   */
+  async updateStepData(
+    jobId: number,
+    stepData: Record<string, any>,
+  ): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(
+        `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
+        [jobId, JSON.stringify(stepData)],
+      );
+    } catch (error) {
+      log(`Error updating step_data for job ${jobId}: ${error}`);
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Create a waitpoint token in the database.
+   *
+   * @param jobId - The job ID to associate with the token (null if created outside a handler).
+   * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+   * @returns The created waitpoint with its unique ID.
+   */
+  async createWaitpoint(
+    jobId: number | null,
+    options?: CreateTokenOptions,
+  ): Promise<{ id: string }> {
+    const client = await this.pool.connect();
+    try {
+      const id = `wp_${randomUUID()}`;
+      let timeoutAt: Date | null = null;
+
+      if (options?.timeout) {
+        const ms = parseTimeoutString(options.timeout);
+        timeoutAt = new Date(Date.now() + ms);
+      }
+
+      await client.query(
+        `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
+        [id, jobId, timeoutAt, options?.tags ?? null],
+      );
+
+      log(`Created waitpoint ${id} for job ${jobId}`);
+      return { id };
+    } catch (error) {
+      log(`Error creating waitpoint: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Complete a waitpoint token and move the associated job back to 'pending'.
+   *
+   * @param tokenId - The waitpoint token ID to complete.
+   * @param data - Optional data to pass to the waiting handler.
+   */
+  async completeWaitpoint(tokenId: string, data?: any): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+
+      const wpResult = await client.query(
+        `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
+         WHERE id = $1 AND status = 'waiting'
+         RETURNING job_id`,
+        [tokenId, data != null ? JSON.stringify(data) : null],
+      );
+
+      if (wpResult.rows.length === 0) {
+        await client.query('ROLLBACK');
+        log(`Waitpoint ${tokenId} not found or already completed`);
+        return;
+      }
+
+      const jobId = wpResult.rows[0].job_id;
+
+      if (jobId != null) {
+        await client.query(
+          `UPDATE job_queue
+           SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
+           WHERE id = $1 AND status = 'waiting'`,
+          [jobId],
+        );
+      }
+
+      await client.query('COMMIT');
+      log(`Completed waitpoint ${tokenId} for job ${jobId}`);
+    } catch (error) {
+      await client.query('ROLLBACK');
+      log(`Error completing waitpoint ${tokenId}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Retrieve a waitpoint token by its ID.
+   *
+   * @param tokenId - The waitpoint token ID to look up.
+   * @returns The waitpoint record, or null if not found.
+   */
+  async getWaitpoint(tokenId: string): Promise<WaitpointRecord | null> {
+    const client = await this.pool.connect();
+    try {
+      const result = await client.query(
+        `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
+        [tokenId],
+      );
+      if (result.rows.length === 0) return null;
+      return result.rows[0] as WaitpointRecord;
+    } catch (error) {
+      log(`Error getting waitpoint ${tokenId}: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /**
+   * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+   *
+   * @returns The number of tokens that were expired.
+   */
+  async expireTimedOutWaitpoints(): Promise<number> {
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+
+      const result = await client.query(
+        `UPDATE waitpoints
+         SET status = 'timed_out'
+         WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
+         RETURNING id, job_id`,
+      );
+
+      for (const row of result.rows) {
+        if (row.job_id != null) {
+          await client.query(
+            `UPDATE job_queue
+             SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
+             WHERE id = $1 AND status = 'waiting'`,
+            [row.job_id],
+          );
+        }
+      }
+
+      await client.query('COMMIT');
+      const count = result.rowCount || 0;
+      if (count > 0) {
+        log(`Expired ${count} timed-out waitpoints`);
+      }
+      return count;
+    } catch (error) {
+      await client.query('ROLLBACK');
+      log(`Error expiring timed-out waitpoints: ${error}`);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  // ── Internal helpers ──────────────────────────────────────────────────
+
   async setPendingReasonForUnpickedJobs(
     reason: string,
     jobType?: string | string[],