@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/backends/redis.ts

@@ -14,6 +14,7 @@ import {
   EditCronScheduleOptions,
   WaitpointRecord,
   CreateTokenOptions,
+  AddJobOptions,
 } from '../types.js';
 import {
   QueueBackend,
@@ -61,6 +62,7 @@ function parseTimeoutString(timeout: string): number {
 }
 import {
   ADD_JOB_SCRIPT,
+  ADD_JOBS_SCRIPT,
   GET_NEXT_BATCH_SCRIPT,
   COMPLETE_JOB_SCRIPT,
   FAIL_JOB_SCRIPT,
@@ -162,9 +164,30 @@ function deserializeJob<PayloadMap, T extends JobType<PayloadMap>>(
     waitUntil: dateOrNull(h.waitUntil),
     waitTokenId: nullish(h.waitTokenId) as string | null | undefined,
     stepData: parseStepData(h.stepData),
+    retryDelay: numOrNull(h.retryDelay),
+    retryBackoff:
+      h.retryBackoff === 'true'
+        ? true
+        : h.retryBackoff === 'false'
+          ? false
+          : null,
+    retryDelayMax: numOrNull(h.retryDelayMax),
+    groupId: nullish(h.groupId) as string | null | undefined,
+    groupTier: nullish(h.groupTier) as string | null | undefined,
+    output: parseJsonField(h.output),
   };
 }
 
+/** Parse a JSON field from a Redis hash, returning null for missing/null values. */
+function parseJsonField(raw: string | undefined): unknown {
+  if (!raw || raw === 'null') return null;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
 /** Parse step data from a Redis hash field. */
 function parseStepData(
   raw: string | undefined,
@@ -181,8 +204,30 @@ export class RedisBackend implements QueueBackend {
   private client: RedisType;
   private prefix: string;
 
-  constructor(redisConfig: RedisJobQueueConfig['redisConfig']) {
-    // Dynamically require ioredis to avoid hard dep
+  /**
+   * Create a RedisBackend.
+   *
+   * @param configOrClient - Either `redisConfig` from the config file (the
+   *   library creates a new ioredis client) or an existing ioredis client
+   *   instance (bring your own).
+   * @param keyPrefix - Key prefix, only used when `configOrClient` is an
+   *   external client. Ignored when `redisConfig` is passed (uses
+   *   `redisConfig.keyPrefix` instead). Default: `'dq:'`.
+   */
+  constructor(
+    configOrClient: RedisJobQueueConfig['redisConfig'] | RedisType,
+    keyPrefix?: string,
+  ) {
+    if (configOrClient && typeof (configOrClient as any).eval === 'function') {
+      this.client = configOrClient as RedisType;
+      this.prefix = keyPrefix ?? 'dq:';
+      return;
+    }
+
+    const redisConfig = configOrClient as NonNullable<
+      RedisJobQueueConfig['redisConfig']
+    >;
+
     let IORedis: any;
     try {
       const _require = createRequire(import.meta.url);
@@ -260,17 +305,30 @@ export class RedisBackend implements QueueBackend {
 
   // ── Job CRUD ──────────────────────────────────────────────────────────
 
-  async addJob<PayloadMap, T extends JobType<PayloadMap>>({
-    jobType,
-    payload,
-    maxAttempts = 3,
-    priority = 0,
-    runAt = null,
-    timeoutMs = undefined,
-    forceKillOnTimeout = false,
-    tags = undefined,
-    idempotencyKey = undefined,
-  }: JobOptions<PayloadMap, T>): Promise<number> {
+  async addJob<PayloadMap, T extends JobType<PayloadMap>>(
+    {
+      jobType,
+      payload,
+      maxAttempts = 3,
+      priority = 0,
+      runAt = null,
+      timeoutMs = undefined,
+      forceKillOnTimeout = false,
+      tags = undefined,
+      idempotencyKey = undefined,
+      retryDelay = undefined,
+      retryBackoff = undefined,
+      retryDelayMax = undefined,
+      group = undefined,
+    }: JobOptions<PayloadMap, T>,
+    options?: AddJobOptions,
+  ): Promise<number> {
+    if (options?.db) {
+      throw new Error(
+        'The db option is not supported with the Redis backend. ' +
+          'Transactional job creation is only available with PostgreSQL.',
+      );
+    }
     const now = this.nowMs();
     const runAtMs = runAt ? runAt.getTime() : 0;
 
@@ -288,6 +346,11 @@ export class RedisBackend implements QueueBackend {
       tags ? JSON.stringify(tags) : 'null',
       idempotencyKey ?? 'null',
       now,
+      retryDelay !== undefined ? retryDelay.toString() : 'null',
+      retryBackoff !== undefined ? retryBackoff.toString() : 'null',
+      retryDelayMax !== undefined ? retryDelayMax.toString() : 'null',
+      group?.id ?? 'null',
+      group?.tier ?? 'null',
     )) as number;
 
     const jobId = Number(result);
@@ -303,6 +366,85 @@ export class RedisBackend implements QueueBackend {
     return jobId;
   }
 
+  /**
+   * Insert multiple jobs atomically via a single Lua script.
+   * Returns IDs in the same order as the input array.
+   */
+  async addJobs<PayloadMap, T extends JobType<PayloadMap>>(
+    jobs: JobOptions<PayloadMap, T>[],
+    options?: AddJobOptions,
+  ): Promise<number[]> {
+    if (jobs.length === 0) return [];
+
+    if (options?.db) {
+      throw new Error(
+        'The db option is not supported with the Redis backend. ' +
+          'Transactional job creation is only available with PostgreSQL.',
+      );
+    }
+
+    const now = this.nowMs();
+
+    const jobsPayload = jobs.map((job) => ({
+      jobType: job.jobType,
+      payload: JSON.stringify(job.payload),
+      maxAttempts: job.maxAttempts ?? 3,
+      priority: job.priority ?? 0,
+      runAtMs: job.runAt ? job.runAt.getTime() : 0,
+      timeoutMs:
+        job.timeoutMs !== undefined ? job.timeoutMs.toString() : 'null',
+      forceKillOnTimeout: job.forceKillOnTimeout ? 'true' : 'false',
+      tags: job.tags ? JSON.stringify(job.tags) : 'null',
+      idempotencyKey: job.idempotencyKey ?? 'null',
+      retryDelay:
+        job.retryDelay !== undefined ? job.retryDelay.toString() : 'null',
+      retryBackoff:
+        job.retryBackoff !== undefined ? job.retryBackoff.toString() : 'null',
+      retryDelayMax:
+        job.retryDelayMax !== undefined ? job.retryDelayMax.toString() : 'null',
+      groupId: job.group?.id ?? 'null',
+      groupTier: job.group?.tier ?? 'null',
+    }));
+
+    const result = (await this.client.eval(
+      ADD_JOBS_SCRIPT,
+      1,
+      this.prefix,
+      JSON.stringify(jobsPayload),
+      now,
+    )) as number[];
+
+    const ids = result.map(Number);
+    log(`Batch-inserted ${jobs.length} jobs, IDs: [${ids.join(', ')}]`);
+
+    // Record events for newly inserted jobs (skip idempotency duplicates)
+    const existingIdempotencyIds = new Set<number>();
+    for (let i = 0; i < jobs.length; i++) {
+      if (jobs[i].idempotencyKey) {
+        // If the returned ID existed before this batch, it was a duplicate.
+        // We detect this by checking if the same ID appears for a different
+        // idempotency-keyed job (unlikely) or by checking if the ID was less
+        // than what we'd expect. The simplest approach: record events for all,
+        // since the Lua script returns the existing ID for duplicates but
+        // doesn't tell us if it was newly created. We can compare: if
+        // multiple jobs have the same idempotency key in the batch and got
+        // the same ID, only record once.
+        if (existingIdempotencyIds.has(ids[i])) {
+          continue;
+        }
+        existingIdempotencyIds.add(ids[i]);
+      }
+      await this.recordJobEvent(ids[i], JobEventType.Added, {
+        jobType: jobs[i].jobType,
+        payload: jobs[i].payload,
+        tags: jobs[i].tags,
+        idempotencyKey: jobs[i].idempotencyKey,
+      });
+    }
+
+    return ids;
+  }
+
   async getJob<PayloadMap, T extends JobType<PayloadMap>>(
     id: number,
   ): Promise<JobRecord<PayloadMap, T> | null> {
@@ -420,6 +562,7 @@ export class RedisBackend implements QueueBackend {
     workerId: string,
     batchSize = 10,
     jobType?: string | string[],
+    groupConcurrency?: number,
   ): Promise<JobRecord<PayloadMap, T>[]> {
     const now = this.nowMs();
     const jobTypeFilter =
@@ -437,6 +580,7 @@ export class RedisBackend implements QueueBackend {
       batchSize,
       now,
       jobTypeFilter,
+      groupConcurrency !== undefined ? groupConcurrency : 'null',
     )) as string[];
 
     if (!result || result.length === 0) {
@@ -469,9 +613,18 @@ export class RedisBackend implements QueueBackend {
     return jobs;
   }
 
-  async completeJob(jobId: number): Promise<void> {
+  async completeJob(jobId: number, output?: unknown): Promise<void> {
     const now = this.nowMs();
-    await this.client.eval(COMPLETE_JOB_SCRIPT, 1, this.prefix, jobId, now);
+    const outputArg =
+      output !== undefined ? JSON.stringify(output) : '__NONE__';
+    await this.client.eval(
+      COMPLETE_JOB_SCRIPT,
+      1,
+      this.prefix,
+      jobId,
+      now,
+      outputArg,
+    );
     await this.recordJobEvent(jobId, JobEventType.Completed);
     log(`Completed job ${jobId}`);
   }
@@ -535,6 +688,24 @@ export class RedisBackend implements QueueBackend {
     }
   }
 
+  // ── Output ────────────────────────────────────────────────────────────
+
+  async updateOutput(jobId: number, output: unknown): Promise<void> {
+    try {
+      const now = this.nowMs();
+      await this.client.hset(
+        `${this.prefix}job:${jobId}`,
+        'output',
+        JSON.stringify(output),
+        'updatedAt',
+        now.toString(),
+      );
+      log(`Updated output for job ${jobId}`);
+    } catch (error) {
+      log(`Error updating output for job ${jobId}: ${error}`);
+    }
+  }
+
   // ── Job management ────────────────────────────────────────────────────
 
   async retryJob(jobId: number): Promise<void> {
@@ -657,6 +828,31 @@ export class RedisBackend implements QueueBackend {
       }
       metadata.tags = updates.tags;
     }
+    if (updates.retryDelay !== undefined) {
+      fields.push(
+        'retryDelay',
+        updates.retryDelay !== null ? updates.retryDelay.toString() : 'null',
+      );
+      metadata.retryDelay = updates.retryDelay;
+    }
+    if (updates.retryBackoff !== undefined) {
+      fields.push(
+        'retryBackoff',
+        updates.retryBackoff !== null
+          ? updates.retryBackoff.toString()
+          : 'null',
+      );
+      metadata.retryBackoff = updates.retryBackoff;
+    }
+    if (updates.retryDelayMax !== undefined) {
+      fields.push(
+        'retryDelayMax',
+        updates.retryDelayMax !== null
+          ? updates.retryDelayMax.toString()
+          : 'null',
+      );
+      metadata.retryDelayMax = updates.retryDelayMax;
+    }
 
     if (fields.length === 0) {
       log(`No fields to update for job ${jobId}`);
@@ -1236,6 +1432,18 @@ export class RedisBackend implements QueueBackend {
       now.toString(),
       'updatedAt',
       now.toString(),
+      'retryDelay',
+      input.retryDelay !== null && input.retryDelay !== undefined
+        ? input.retryDelay.toString()
+        : 'null',
+      'retryBackoff',
+      input.retryBackoff !== null && input.retryBackoff !== undefined
+        ? input.retryBackoff.toString()
+        : 'null',
+      'retryDelayMax',
+      input.retryDelayMax !== null && input.retryDelayMax !== undefined
+        ? input.retryDelayMax.toString()
+        : 'null',
     ];
 
     await (this.client as any).hmset(key, ...fields);
@@ -1417,6 +1625,28 @@ export class RedisBackend implements QueueBackend {
     if (updates.allowOverlap !== undefined) {
       fields.push('allowOverlap', updates.allowOverlap ? 'true' : 'false');
     }
+    if (updates.retryDelay !== undefined) {
+      fields.push(
+        'retryDelay',
+        updates.retryDelay !== null ? updates.retryDelay.toString() : 'null',
+      );
+    }
+    if (updates.retryBackoff !== undefined) {
+      fields.push(
+        'retryBackoff',
+        updates.retryBackoff !== null
+          ? updates.retryBackoff.toString()
+          : 'null',
+      );
+    }
+    if (updates.retryDelayMax !== undefined) {
+      fields.push(
+        'retryDelayMax',
+        updates.retryDelayMax !== null
+          ? updates.retryDelayMax.toString()
+          : 'null',
+      );
+    }
     if (nextRunAt !== undefined) {
       const val = nextRunAt !== null ? nextRunAt.getTime().toString() : 'null';
       fields.push('nextRunAt', val);
@@ -1557,6 +1787,14 @@ export class RedisBackend implements QueueBackend {
       nextRunAt: dateOrNull(h.nextRunAt),
       createdAt: new Date(Number(h.createdAt)),
       updatedAt: new Date(Number(h.updatedAt)),
+      retryDelay: numOrNull(h.retryDelay),
+      retryBackoff:
+        h.retryBackoff === 'true'
+          ? true
+          : h.retryBackoff === 'false'
+            ? false
+            : null,
+      retryDelayMax: numOrNull(h.retryDelayMax),
     };
   }
 

package/src/db-util.ts

@@ -27,7 +27,7 @@ function loadPemOrFile(value?: string): string | undefined {
  *   }
  */
 export const createPool = (
-  config: PostgresJobQueueConfig['databaseConfig'],
+  config: NonNullable<PostgresJobQueueConfig['databaseConfig']>,
 ): Pool => {
   let searchPath: string | undefined;
   let ssl: any = undefined;