@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/backends/postgres.ts

@@ -12,6 +12,8 @@ import {
   EditCronScheduleOptions,
   WaitpointRecord,
   CreateTokenOptions,
+  AddJobOptions,
+  DatabaseClient,
 } from '../types.js';
 import { randomUUID } from 'crypto';
 import {
@@ -103,18 +105,35 @@ export class PostgresBackend implements QueueBackend {
 
   // ── Job CRUD ──────────────────────────────────────────────────────────
 
-  async addJob<PayloadMap, T extends JobType<PayloadMap>>({
-    jobType,
-    payload,
-    maxAttempts = 3,
-    priority = 0,
-    runAt = null,
-    timeoutMs = undefined,
-    forceKillOnTimeout = false,
-    tags = undefined,
-    idempotencyKey = undefined,
-  }: JobOptions<PayloadMap, T>): Promise<number> {
-    const client = await this.pool.connect();
+  /**
+   * Add a job and return its numeric ID.
+   *
+   * @param job - Job configuration.
+   * @param options - Optional. Pass `{ db }` to run the INSERT on an external
+   *   client (e.g., inside a transaction) so the job is part of the caller's
+   *   transaction. The event INSERT also uses the same client.
+   */
+  async addJob<PayloadMap, T extends JobType<PayloadMap>>(
+    {
+      jobType,
+      payload,
+      maxAttempts = 3,
+      priority = 0,
+      runAt = null,
+      timeoutMs = undefined,
+      forceKillOnTimeout = false,
+      tags = undefined,
+      idempotencyKey = undefined,
+      retryDelay = undefined,
+      retryBackoff = undefined,
+      retryDelayMax = undefined,
+      group = undefined,
+    }: JobOptions<PayloadMap, T>,
+    options?: AddJobOptions,
+  ): Promise<number> {
+    const externalClient = options?.db;
+    const client: DatabaseClient =
+      externalClient ?? (await this.pool.connect());
     try {
       let result;
       const onConflict = idempotencyKey
@@ -124,8 +143,8 @@ export class PostgresBackend implements QueueBackend {
       if (runAt) {
         result = await client.query(
           `INSERT INTO job_queue 
-            (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key) 
-           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) 
+            (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key, retry_delay, retry_backoff, retry_delay_max, group_id, group_tier) 
+           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) 
            ${onConflict}
            RETURNING id`,
           [
@@ -138,13 +157,18 @@ export class PostgresBackend implements QueueBackend {
             forceKillOnTimeout ?? false,
             tags ?? null,
             idempotencyKey ?? null,
+            retryDelay ?? null,
+            retryBackoff ?? null,
+            retryDelayMax ?? null,
+            group?.id ?? null,
+            group?.tier ?? null,
           ],
         );
       } else {
         result = await client.query(
           `INSERT INTO job_queue 
-            (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key) 
-           VALUES ($1, $2, $3, $4, $5, $6, $7, $8) 
+            (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key, retry_delay, retry_backoff, retry_delay_max, group_id, group_tier) 
+           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) 
            ${onConflict}
            RETURNING id`,
           [
@@ -156,11 +180,15 @@ export class PostgresBackend implements QueueBackend {
             forceKillOnTimeout ?? false,
             tags ?? null,
             idempotencyKey ?? null,
+            retryDelay ?? null,
+            retryBackoff ?? null,
+            retryDelayMax ?? null,
+            group?.id ?? null,
+            group?.tier ?? null,
           ],
         );
       }
 
-      // If ON CONFLICT DO NOTHING was triggered, no rows are returned.
       if (result.rows.length === 0 && idempotencyKey) {
         const existing = await client.query(
           `SELECT id FROM job_queue WHERE idempotency_key = $1`,
@@ -181,18 +209,220 @@ export class PostgresBackend implements QueueBackend {
       log(
         `Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ''}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ''}`,
       );
-      await this.recordJobEvent(jobId, JobEventType.Added, {
-        jobType,
-        payload,
-        tags,
-        idempotencyKey,
-      });
+
+      if (externalClient) {
+        try {
+          await client.query(
+            `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
+            [
+              jobId,
+              JobEventType.Added,
+              JSON.stringify({ jobType, payload, tags, idempotencyKey }),
+            ],
+          );
+        } catch (error) {
+          log(`Error recording job event for job ${jobId}: ${error}`);
+        }
+      } else {
+        await this.recordJobEvent(jobId, JobEventType.Added, {
+          jobType,
+          payload,
+          tags,
+          idempotencyKey,
+        });
+      }
       return jobId;
     } catch (error) {
       log(`Error adding job: ${error}`);
       throw error;
     } finally {
-      client.release();
+      if (!externalClient) (client as any).release();
+    }
+  }
+
+  /**
+   * Insert multiple jobs in a single database round-trip.
+   *
+   * Uses a multi-row INSERT with ON CONFLICT handling for idempotency keys.
+   * Returns IDs in the same order as the input array.
+   */
+  async addJobs<PayloadMap, T extends JobType<PayloadMap>>(
+    jobs: JobOptions<PayloadMap, T>[],
+    options?: AddJobOptions,
+  ): Promise<number[]> {
+    if (jobs.length === 0) return [];
+
+    const externalClient = options?.db;
+    const client: DatabaseClient =
+      externalClient ?? (await this.pool.connect());
+    try {
+      const COLS_PER_JOB = 14;
+      const valueClauses: string[] = [];
+      const params: any[] = [];
+
+      const hasAnyIdempotencyKey = jobs.some((j) => j.idempotencyKey);
+
+      for (let i = 0; i < jobs.length; i++) {
+        const {
+          jobType,
+          payload,
+          maxAttempts = 3,
+          priority = 0,
+          runAt = null,
+          timeoutMs = undefined,
+          forceKillOnTimeout = false,
+          tags = undefined,
+          idempotencyKey = undefined,
+          retryDelay = undefined,
+          retryBackoff = undefined,
+          retryDelayMax = undefined,
+          group = undefined,
+        } = jobs[i];
+
+        const base = i * COLS_PER_JOB;
+        valueClauses.push(
+          `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, ` +
+            `COALESCE($${base + 5}::timestamptz, CURRENT_TIMESTAMP), ` +
+            `$${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, ` +
+            `$${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14})`,
+        );
+        params.push(
+          jobType,
+          payload,
+          maxAttempts,
+          priority,
+          runAt,
+          timeoutMs ?? null,
+          forceKillOnTimeout ?? false,
+          tags ?? null,
+          idempotencyKey ?? null,
+          retryDelay ?? null,
+          retryBackoff ?? null,
+          retryDelayMax ?? null,
+          group?.id ?? null,
+          group?.tier ?? null,
+        );
+      }
+
+      const onConflict = hasAnyIdempotencyKey
+        ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`
+        : '';
+
+      const result = await client.query(
+        `INSERT INTO job_queue
+          (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key, retry_delay, retry_backoff, retry_delay_max, group_id, group_tier)
+         VALUES ${valueClauses.join(', ')}
+         ${onConflict}
+         RETURNING id, idempotency_key`,
+        params,
+      );
+
+      // Build a map of idempotency_key -> id from returned rows
+      const returnedKeyToId = new Map<string, number>();
+      const returnedNullKeyIds: number[] = [];
+      for (const row of result.rows) {
+        if (row.idempotency_key != null) {
+          returnedKeyToId.set(row.idempotency_key, row.id);
+        } else {
+          returnedNullKeyIds.push(row.id);
+        }
+      }
+
+      // Identify idempotency keys that conflicted (not in RETURNING)
+      const missingKeys: string[] = [];
+      for (const job of jobs) {
+        if (job.idempotencyKey && !returnedKeyToId.has(job.idempotencyKey)) {
+          missingKeys.push(job.idempotencyKey);
+        }
+      }
+
+      // Batch-fetch existing IDs for conflicted keys
+      if (missingKeys.length > 0) {
+        const existing = await client.query(
+          `SELECT id, idempotency_key FROM job_queue WHERE idempotency_key = ANY($1)`,
+          [missingKeys],
+        );
+        for (const row of existing.rows) {
+          returnedKeyToId.set(row.idempotency_key, row.id);
+        }
+      }
+
+      // Assemble result array in input order
+      let nullKeyIdx = 0;
+      const ids: number[] = [];
+      for (const job of jobs) {
+        if (job.idempotencyKey) {
+          const id = returnedKeyToId.get(job.idempotencyKey);
+          if (id === undefined) {
+            throw new Error(
+              `Failed to resolve job ID for idempotency key "${job.idempotencyKey}"`,
+            );
+          }
+          ids.push(id);
+        } else {
+          ids.push(returnedNullKeyIds[nullKeyIdx++]);
+        }
+      }
+
+      log(`Batch-inserted ${jobs.length} jobs, IDs: [${ids.join(', ')}]`);
+
+      // Record 'added' events — only for newly inserted jobs
+      const newJobEvents: {
+        jobId: number;
+        eventType: JobEventType;
+        metadata?: any;
+      }[] = [];
+      for (let i = 0; i < jobs.length; i++) {
+        const job = jobs[i];
+        const wasInserted =
+          !job.idempotencyKey || !missingKeys.includes(job.idempotencyKey);
+        if (wasInserted) {
+          newJobEvents.push({
+            jobId: ids[i],
+            eventType: JobEventType.Added,
+            metadata: {
+              jobType: job.jobType,
+              payload: job.payload,
+              tags: job.tags,
+              idempotencyKey: job.idempotencyKey,
+            },
+          });
+        }
+      }
+
+      if (newJobEvents.length > 0) {
+        if (externalClient) {
+          // Record events on the same transaction client
+          const evtValues: string[] = [];
+          const evtParams: any[] = [];
+          let evtIdx = 1;
+          for (const evt of newJobEvents) {
+            evtValues.push(`($${evtIdx++}, $${evtIdx++}, $${evtIdx++})`);
+            evtParams.push(
+              evt.jobId,
+              evt.eventType,
+              evt.metadata ? JSON.stringify(evt.metadata) : null,
+            );
+          }
+          try {
+            await client.query(
+              `INSERT INTO job_events (job_id, event_type, metadata) VALUES ${evtValues.join(', ')}`,
+              evtParams,
+            );
+          } catch (error) {
+            log(`Error recording batch job events: ${error}`);
+          }
+        } else {
+          await this.recordJobEventsBatch(newJobEvents);
+        }
+      }
+
+      return ids;
+    } catch (error) {
+      log(`Error batch-inserting jobs: ${error}`);
+      throw error;
+    } finally {
+      if (!externalClient) (client as any).release();
     }
   }
 
@@ -202,7 +432,7 @@ export class PostgresBackend implements QueueBackend {
     const client = await this.pool.connect();
     try {
       const result = await client.query(
-        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
+        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output FROM job_queue WHERE id = $1`,
         [id],
       );
 
@@ -236,7 +466,7 @@ export class PostgresBackend implements QueueBackend {
     const client = await this.pool.connect();
     try {
       const result = await client.query(
-        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
+        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
         [status, limit, offset],
       );
       log(`Found ${result.rows.length} jobs by status ${status}`);
@@ -262,7 +492,7 @@ export class PostgresBackend implements QueueBackend {
     const client = await this.pool.connect();
     try {
       const result = await client.query(
-        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
+        `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
         [limit, offset],
       );
       log(`Found ${result.rows.length} jobs (all)`);
@@ -287,7 +517,7 @@ export class PostgresBackend implements QueueBackend {
   ): Promise<JobRecord<PayloadMap, T>[]> {
     const client = await this.pool.connect();
     try {
-      let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
+      let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output FROM job_queue`;
       const params: any[] = [];
       const where: string[] = [];
       let paramIdx = 1;
@@ -414,7 +644,7 @@ export class PostgresBackend implements QueueBackend {
   ): Promise<JobRecord<PayloadMap, T>[]> {
     const client = await this.pool.connect();
     try {
-      let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
+      let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output
          FROM job_queue`;
       let params: any[] = [];
       switch (mode) {
@@ -467,6 +697,7 @@ export class PostgresBackend implements QueueBackend {
     workerId: string,
     batchSize = 10,
     jobType?: string | string[],
+    groupConcurrency?: number,
   ): Promise<JobRecord<PayloadMap, T>[]> {
     const client = await this.pool.connect();
     try {
@@ -476,50 +707,121 @@ export class PostgresBackend implements QueueBackend {
       const params: any[] = [workerId, batchSize];
       if (jobType) {
         if (Array.isArray(jobType)) {
-          jobTypeFilter = ` AND job_type = ANY($3)`;
+          jobTypeFilter = ` AND candidate.job_type = ANY($3)`;
           params.push(jobType);
         } else {
-          jobTypeFilter = ` AND job_type = $3`;
+          jobTypeFilter = ` AND candidate.job_type = $3`;
           params.push(jobType);
         }
       }
 
-      const result = await client.query(
-        `
-        UPDATE job_queue
-        SET status = 'processing', 
-            locked_at = NOW(), 
-            locked_by = $1,
-            attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
-            updated_at = NOW(),
-            pending_reason = NULL,
-            started_at = COALESCE(started_at, NOW()),
-            last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
-            wait_until = NULL
-        WHERE id IN (
-          SELECT id FROM job_queue
-          WHERE (
-            (
-              (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
-              AND (attempts < max_attempts)
-              AND run_at <= NOW()
+      let result;
+      if (groupConcurrency === undefined) {
+        result = await client.query(
+          `
+          UPDATE job_queue
+          SET status = 'processing', 
+              locked_at = NOW(), 
+              locked_by = $1,
+              attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
+              updated_at = NOW(),
+              pending_reason = NULL,
+              started_at = COALESCE(started_at, NOW()),
+              last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
+              wait_until = NULL
+          WHERE id IN (
+            SELECT id FROM job_queue candidate
+            WHERE (
+              (
+                (candidate.status = 'pending' OR (candidate.status = 'failed' AND candidate.next_attempt_at <= NOW()))
+                AND (candidate.attempts < candidate.max_attempts)
+                AND candidate.run_at <= NOW()
+              )
+              OR (
+                candidate.status = 'waiting'
+                AND candidate.wait_until IS NOT NULL
+                AND candidate.wait_until <= NOW()
+                AND candidate.wait_token_id IS NULL
+              )
             )
-            OR (
-              status = 'waiting'
-              AND wait_until IS NOT NULL
-              AND wait_until <= NOW()
-              AND wait_token_id IS NULL
+            ${jobTypeFilter}
+            ORDER BY candidate.priority DESC, candidate.created_at ASC
+            LIMIT $2
+            FOR UPDATE SKIP LOCKED
+          )
+          RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output
+        `,
+          params,
+        );
+      } else {
+        const constrainedParams = [...params, groupConcurrency];
+        const groupConcurrencyParamIndex = constrainedParams.length;
+        result = await client.query(
+          `
+          WITH eligible AS (
+            SELECT candidate.id, candidate.group_id, candidate.priority, candidate.created_at
+            FROM job_queue candidate
+            WHERE (
+              (
+                (candidate.status = 'pending' OR (candidate.status = 'failed' AND candidate.next_attempt_at <= NOW()))
+                AND (candidate.attempts < candidate.max_attempts)
+                AND candidate.run_at <= NOW()
+              )
+              OR (
+                candidate.status = 'waiting'
+                AND candidate.wait_until IS NOT NULL
+                AND candidate.wait_until <= NOW()
+                AND candidate.wait_token_id IS NULL
+              )
             )
+            ${jobTypeFilter}
+            FOR UPDATE SKIP LOCKED
+          ),
+          ranked AS (
+            SELECT
+              eligible.id,
+              eligible.group_id,
+              eligible.priority,
+              eligible.created_at,
+              ROW_NUMBER() OVER (
+                PARTITION BY eligible.group_id
+                ORDER BY eligible.priority DESC, eligible.created_at ASC
+              ) AS group_rank,
+              COALESCE((
+                SELECT COUNT(*)
+                FROM job_queue processing_jobs
+                WHERE processing_jobs.status = 'processing'
+                  AND processing_jobs.group_id = eligible.group_id
+              ), 0) AS active_group_count
+            FROM eligible
+          ),
+          selected AS (
+            SELECT ranked.id
+            FROM ranked
+            WHERE ranked.group_id IS NULL
+              OR (
+                ranked.active_group_count < $${groupConcurrencyParamIndex}
+                AND ranked.group_rank <= ($${groupConcurrencyParamIndex} - ranked.active_group_count)
+              )
+            ORDER BY ranked.priority DESC, ranked.created_at ASC
+            LIMIT $2
           )
-          ${jobTypeFilter}
-          ORDER BY priority DESC, created_at ASC
-          LIMIT $2
-          FOR UPDATE SKIP LOCKED
-        )
-        RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
-      `,
-        params,
-      );
+          UPDATE job_queue
+          SET status = 'processing', 
+              locked_at = NOW(), 
+              locked_by = $1,
+              attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
+              updated_at = NOW(),
+              pending_reason = NULL,
+              started_at = COALESCE(started_at, NOW()),
+              last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
+              wait_until = NULL
+          WHERE id IN (SELECT id FROM selected)
+          RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress, retry_delay AS "retryDelay", retry_backoff AS "retryBackoff", retry_delay_max AS "retryDelayMax", group_id AS "groupId", group_tier AS "groupTier", output
+        `,
+          constrainedParams,
+        );
+      }
 
       log(`Found ${result.rows.length} jobs to process`);
       await client.query('COMMIT');
@@ -549,17 +851,19 @@ export class PostgresBackend implements QueueBackend {
     }
   }
 
-  async completeJob(jobId: number): Promise<void> {
+  async completeJob(jobId: number, output?: unknown): Promise<void> {
     const client = await this.pool.connect();
     try {
+      const outputJson = output !== undefined ? JSON.stringify(output) : null;
       const result = await client.query(
         `
         UPDATE job_queue
         SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
-            step_data = NULL, wait_until = NULL, wait_token_id = NULL
+            step_data = NULL, wait_until = NULL, wait_token_id = NULL,
+            output = COALESCE($2::jsonb, output)
         WHERE id = $1 AND status = 'processing'
       `,
-        [jobId],
+        [jobId, outputJson],
       );
       if (result.rowCount === 0) {
         log(
@@ -588,9 +892,17 @@ export class PostgresBackend implements QueueBackend {
         UPDATE job_queue
         SET status = 'failed', 
             updated_at = NOW(),
-            next_attempt_at = CASE 
-              WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
-              ELSE NULL
+            next_attempt_at = CASE
+              WHEN attempts >= max_attempts THEN NULL
+              WHEN retry_delay IS NULL AND retry_backoff IS NULL AND retry_delay_max IS NULL
+                THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
+              WHEN COALESCE(retry_backoff, true) = true
+                THEN NOW() + (LEAST(
+                  COALESCE(retry_delay_max, 2147483647),
+                  COALESCE(retry_delay, 60) * POWER(2, attempts)
+                ) * (0.5 + 0.5 * random()) * INTERVAL '1 second')
+              ELSE
+                NOW() + (COALESCE(retry_delay, 60) * INTERVAL '1 second')
             END,
             error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
             failure_reason = $3,
@@ -665,6 +977,23 @@ export class PostgresBackend implements QueueBackend {
     }
   }
 
+  // ── Output ────────────────────────────────────────────────────────────
+
+  async updateOutput(jobId: number, output: unknown): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      await client.query(
+        `UPDATE job_queue SET output = $2::jsonb, updated_at = NOW() WHERE id = $1`,
+        [jobId, JSON.stringify(output)],
+      );
+      log(`Updated output for job ${jobId}`);
+    } catch (error) {
+      log(`Error updating output for job ${jobId}: ${error}`);
+    } finally {
+      client.release();
+    }
+  }
+
   // ── Job management ────────────────────────────────────────────────────
 
   async retryJob(jobId: number): Promise<void> {
@@ -843,6 +1172,18 @@ export class PostgresBackend implements QueueBackend {
         updateFields.push(`tags = $${paramIdx++}`);
         params.push(updates.tags ?? null);
       }
+      if (updates.retryDelay !== undefined) {
+        updateFields.push(`retry_delay = $${paramIdx++}`);
+        params.push(updates.retryDelay ?? null);
+      }
+      if (updates.retryBackoff !== undefined) {
+        updateFields.push(`retry_backoff = $${paramIdx++}`);
+        params.push(updates.retryBackoff ?? null);
+      }
+      if (updates.retryDelayMax !== undefined) {
+        updateFields.push(`retry_delay_max = $${paramIdx++}`);
+        params.push(updates.retryDelayMax ?? null);
+      }
 
       if (updateFields.length === 0) {
         log(`No fields to update for job ${jobId}`);
@@ -869,6 +1210,12 @@ export class PostgresBackend implements QueueBackend {
       if (updates.timeoutMs !== undefined)
         metadata.timeoutMs = updates.timeoutMs;
       if (updates.tags !== undefined) metadata.tags = updates.tags;
+      if (updates.retryDelay !== undefined)
+        metadata.retryDelay = updates.retryDelay;
+      if (updates.retryBackoff !== undefined)
+        metadata.retryBackoff = updates.retryBackoff;
+      if (updates.retryDelayMax !== undefined)
+        metadata.retryDelayMax = updates.retryDelayMax;
 
       await this.recordJobEvent(jobId, JobEventType.Edited, metadata);
       log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
@@ -918,6 +1265,18 @@ export class PostgresBackend implements QueueBackend {
         updateFields.push(`tags = $${paramIdx++}`);
         params.push(updates.tags ?? null);
       }
+      if (updates.retryDelay !== undefined) {
+        updateFields.push(`retry_delay = $${paramIdx++}`);
+        params.push(updates.retryDelay ?? null);
+      }
+      if (updates.retryBackoff !== undefined) {
+        updateFields.push(`retry_backoff = $${paramIdx++}`);
+        params.push(updates.retryBackoff ?? null);
+      }
+      if (updates.retryDelayMax !== undefined) {
+        updateFields.push(`retry_delay_max = $${paramIdx++}`);
+        params.push(updates.retryDelayMax ?? null);
+      }
 
       if (updateFields.length === 0) {
         log(`No fields to update for batch edit`);
@@ -1188,8 +1547,8 @@ export class PostgresBackend implements QueueBackend {
         `INSERT INTO cron_schedules
           (schedule_name, cron_expression, job_type, payload, max_attempts,
            priority, timeout_ms, force_kill_on_timeout, tags, timezone,
-           allow_overlap, next_run_at)
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+           allow_overlap, next_run_at, retry_delay, retry_backoff, retry_delay_max)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
          RETURNING id`,
         [
           input.scheduleName,
@@ -1204,6 +1563,9 @@ export class PostgresBackend implements QueueBackend {
           input.timezone,
           input.allowOverlap,
           input.nextRunAt,
+          input.retryDelay,
+          input.retryBackoff,
+          input.retryDelayMax,
         ],
       );
       const id = result.rows[0].id;
@@ -1235,7 +1597,9 @@ export class PostgresBackend implements QueueBackend {
                 timezone, allow_overlap AS "allowOverlap", status,
                 last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
                 next_run_at AS "nextRunAt",
-                created_at AS "createdAt", updated_at AS "updatedAt"
+                created_at AS "createdAt", updated_at AS "updatedAt",
+                retry_delay AS "retryDelay", retry_backoff AS "retryBackoff",
+                retry_delay_max AS "retryDelayMax"
          FROM cron_schedules WHERE id = $1`,
         [id],
       );
@@ -1263,7 +1627,9 @@ export class PostgresBackend implements QueueBackend {
                 timezone, allow_overlap AS "allowOverlap", status,
                 last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
                 next_run_at AS "nextRunAt",
-                created_at AS "createdAt", updated_at AS "updatedAt"
+                created_at AS "createdAt", updated_at AS "updatedAt",
+                retry_delay AS "retryDelay", retry_backoff AS "retryBackoff",
+                retry_delay_max AS "retryDelayMax"
          FROM cron_schedules WHERE schedule_name = $1`,
         [name],
       );
@@ -1290,7 +1656,9 @@ export class PostgresBackend implements QueueBackend {
                 timezone, allow_overlap AS "allowOverlap", status,
                 last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
                 next_run_at AS "nextRunAt",
-                created_at AS "createdAt", updated_at AS "updatedAt"
+                created_at AS "createdAt", updated_at AS "updatedAt",
+                retry_delay AS "retryDelay", retry_backoff AS "retryBackoff",
+                retry_delay_max AS "retryDelayMax"
          FROM cron_schedules`;
       const params: any[] = [];
       if (status) {
@@ -1404,6 +1772,18 @@ export class PostgresBackend implements QueueBackend {
         updateFields.push(`allow_overlap = $${paramIdx++}`);
         params.push(updates.allowOverlap);
       }
+      if (updates.retryDelay !== undefined) {
+        updateFields.push(`retry_delay = $${paramIdx++}`);
+        params.push(updates.retryDelay);
+      }
+      if (updates.retryBackoff !== undefined) {
+        updateFields.push(`retry_backoff = $${paramIdx++}`);
+        params.push(updates.retryBackoff);
+      }
+      if (updates.retryDelayMax !== undefined) {
+        updateFields.push(`retry_delay_max = $${paramIdx++}`);
+        params.push(updates.retryDelayMax);
+      }
       if (nextRunAt !== undefined) {
         updateFields.push(`next_run_at = $${paramIdx++}`);
         params.push(nextRunAt);
@@ -1443,7 +1823,9 @@ export class PostgresBackend implements QueueBackend {
                 timezone, allow_overlap AS "allowOverlap", status,
                 last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
                 next_run_at AS "nextRunAt",
-                created_at AS "createdAt", updated_at AS "updatedAt"
+                created_at AS "createdAt", updated_at AS "updatedAt",
+                retry_delay AS "retryDelay", retry_backoff AS "retryBackoff",
+                retry_delay_max AS "retryDelayMax"
          FROM cron_schedules
          WHERE status = 'active'
            AND next_run_at IS NOT NULL