@nicnocquee/dataqueue 1.22.0 → 1.24.0

package/src/index.test.ts

@@ -228,6 +228,87 @@ describe('index integration', () => {
     expect(job2?.status).toBe('cancelled');
   });
 
+  it('should edit all pending jobs via JobQueue API', async () => {
+    // Add three pending jobs
+    const jobId1 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'batch1@example.com' },
+      priority: 0,
+    });
+    const jobId2 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'batch2@example.com' },
+      priority: 0,
+    });
+    const jobId3 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'batch3@example.com' },
+      priority: 0,
+    });
+    // Add a completed job
+    const jobId4 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'done@example.com' },
+      priority: 0,
+    });
+    await pool.query(
+      `UPDATE job_queue SET status = 'completed' WHERE id = $1`,
+      [jobId4],
+    );
+
+    // Edit all pending jobs
+    const editedCount = await jobQueue.editAllPendingJobs(undefined, {
+      priority: 10,
+    });
+    expect(editedCount).toBeGreaterThanOrEqual(3);
+
+    // Check that all pending jobs are updated
+    const job1 = await jobQueue.getJob(jobId1);
+    const job2 = await jobQueue.getJob(jobId2);
+    const job3 = await jobQueue.getJob(jobId3);
+    expect(job1?.priority).toBe(10);
+    expect(job2?.priority).toBe(10);
+    expect(job3?.priority).toBe(10);
+
+    // Completed job should remain unchanged
+    const completedJob = await jobQueue.getJob(jobId4);
+    expect(completedJob?.priority).toBe(0);
+  });
+
+  it('should edit pending jobs with filters via JobQueue API', async () => {
+    const emailJobId1 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'email1@example.com' },
+      priority: 0,
+    });
+    const emailJobId2 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'email2@example.com' },
+      priority: 0,
+    });
+    const smsJobId = await jobQueue.addJob({
+      jobType: 'sms',
+      payload: { to: 'sms@example.com' },
+      priority: 0,
+    });
+
+    // Edit only email jobs
+    const editedCount = await jobQueue.editAllPendingJobs(
+      { jobType: 'email' },
+      {
+        priority: 5,
+      },
+    );
+    expect(editedCount).toBeGreaterThanOrEqual(2);
+
+    const emailJob1 = await jobQueue.getJob(emailJobId1);
+    const emailJob2 = await jobQueue.getJob(emailJobId2);
+    const smsJob = await jobQueue.getJob(smsJobId);
+    expect(emailJob1?.priority).toBe(5);
+    expect(emailJob2?.priority).toBe(5);
+    expect(smsJob?.priority).toBe(0);
+  });
+
   it('should cancel all upcoming jobs by runAt', async () => {
     const runAt = new Date(Date.now() + 60 * 60 * 1000); // 1 hour in future
     const jobId1 = await jobQueue.addJob({
@@ -302,4 +383,147 @@ describe('index integration', () => {
     expect(job1?.status).toBe('cancelled');
     expect(job2?.status).toBe('pending');
   });
+
+  it('should edit a pending job via JobQueue API', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'original@example.com' },
+      priority: 0,
+      maxAttempts: 3,
+    });
+
+    await jobQueue.editJob(jobId, {
+      payload: { to: 'updated@example.com' },
+      priority: 10,
+      maxAttempts: 5,
+    });
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.payload).toEqual({ to: 'updated@example.com' });
+    expect(job?.priority).toBe(10);
+    expect(job?.maxAttempts).toBe(5);
+    expect(job?.status).toBe('pending');
+  });
+
+  it('should edit a job and then process it correctly', async () => {
+    const handler = vi.fn(async (payload: { foo: string }, _signal) => {
+      expect(payload.foo).toBe('updated@example.com');
+    });
+    const jobId = await jobQueue.addJob({
+      jobType: 'test',
+      payload: { foo: 'original@example.com' },
+    });
+
+    // Edit the job before processing
+    await jobQueue.editJob(jobId, {
+      payload: { foo: 'updated@example.com' },
+    });
+
+    const processor = jobQueue.createProcessor(
+      {
+        email: vi.fn(async () => {}),
+        sms: vi.fn(async () => {}),
+        test: handler,
+      },
+      { pollInterval: 100 },
+    );
+    processor.start();
+    await new Promise((r) => setTimeout(r, 300));
+    processor.stop();
+
+    expect(handler).toHaveBeenCalledWith(
+      { foo: 'updated@example.com' },
+      expect.any(Object),
+    );
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('completed');
+  });
+
+  it('should silently fail when editing non-pending jobs', async () => {
+    // Try to edit a completed job
+    const jobId1 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'original@example.com' },
+    });
+    const processor = jobQueue.createProcessor(
+      {
+        email: vi.fn(async () => {}),
+        sms: vi.fn(async () => {}),
+        test: vi.fn(async () => {}),
+      },
+      { pollInterval: 100 },
+    );
+    processor.start();
+    await new Promise((r) => setTimeout(r, 300));
+    processor.stop();
+
+    const originalJob = await jobQueue.getJob(jobId1);
+    expect(originalJob?.status).toBe('completed');
+
+    await jobQueue.editJob(jobId1, {
+      payload: { to: 'updated@example.com' },
+    });
+
+    const job = await jobQueue.getJob(jobId1);
+    expect(job?.status).toBe('completed');
+    expect(job?.payload).toEqual({ to: 'original@example.com' });
+
+    // Try to edit a processing job
+    // Use a handler that takes longer to ensure job stays in processing state
+    const slowHandler = vi.fn(async (payload: { to: string }, _signal) => {
+      await new Promise((r) => setTimeout(r, 200));
+    });
+    const slowHandlerTest = vi.fn(async (payload: { foo: string }, _signal) => {
+      await new Promise((r) => setTimeout(r, 200));
+    });
+    const processor2 = jobQueue.createProcessor(
+      {
+        email: slowHandler,
+        sms: slowHandler,
+        test: slowHandlerTest,
+      },
+      { pollInterval: 100 },
+    );
+    const jobId2 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'processing@example.com' },
+    });
+    processor2.start();
+    // Wait a bit for job to be picked up
+    await new Promise((r) => setTimeout(r, 150));
+    // Job should be processing now
+    const processingJob = await jobQueue.getJob(jobId2);
+    if (processingJob?.status === 'processing') {
+      await jobQueue.editJob(jobId2, {
+        payload: { to: 'updated@example.com' },
+      });
+
+      const job2 = await jobQueue.getJob(jobId2);
+      // If still processing, payload should be unchanged
+      if (job2?.status === 'processing') {
+        expect(job2?.payload).toEqual({ to: 'processing@example.com' });
+      }
+    }
+    processor2.stop();
+  });
+
+  it('should record edited event when editing via JobQueue API', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'original@example.com' },
+    });
+
+    await jobQueue.editJob(jobId, {
+      payload: { to: 'updated@example.com' },
+      priority: 10,
+    });
+
+    const events = await jobQueue.getJobEvents(jobId);
+    const editEvent = events.find((e) => e.eventType === 'edited');
+    expect(editEvent).not.toBeUndefined();
+    expect(editEvent?.metadata).toMatchObject({
+      payload: { to: 'updated@example.com' },
+      priority: 10,
+    });
+  });
 });

package/src/index.ts

@@ -11,6 +11,8 @@ import {
   getJobEvents,
   getJobsByTags,
   getJobs,
+  editJob,
+  editAllPendingJobs,
 } from './queue.js';
 import { createProcessor } from './processor.js';
 import {
@@ -19,6 +21,7 @@ import {
   JobOptions,
   ProcessorOptions,
   JobHandlers,
+  JobType,
 } from './types.js';
 import { setLogContext } from './log-context.js';
 import { createPool } from './db-util.js';
@@ -78,6 +81,32 @@ export const initJobQueue = <PayloadMap = any>(
       (jobId: number) => cancelJob(pool, jobId),
       config.verbose ?? false,
     ),
+    editJob: withLogContext(
+      <T extends JobType<PayloadMap>>(
+        jobId: number,
+        updates: import('./types.js').EditJobOptions<PayloadMap, T>,
+      ) => editJob<PayloadMap, T>(pool, jobId, updates as any),
+      config.verbose ?? false,
+    ),
+    editAllPendingJobs: withLogContext(
+      <T extends JobType<PayloadMap>>(
+        filters:
+          | {
+              jobType?: string;
+              priority?: number;
+              runAt?:
+                | Date
+                | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
+              tags?: {
+                values: string[];
+                mode?: import('./types.js').TagQueryMode;
+              };
+            }
+          | undefined,
+        updates: import('./types.js').EditJobOptions<PayloadMap, T>,
+      ) => editAllPendingJobs<PayloadMap, T>(pool, filters, updates as any),
+      config.verbose ?? false,
+    ),
     cancelAllUpcomingJobs: withLogContext(
       (filters?: {
         jobType?: string;
@@ -123,3 +152,7 @@ const withLogContext =
   };
 
 export * from './types.js';
+export {
+  validateHandlerSerializable,
+  testHandlerSerialization,
+} from './handler-validation.js';

package/src/processor.test.ts

@@ -475,4 +475,59 @@ describe('per-job timeout', () => {
     const completed = await queue.getJob(pool, jobId);
     expect(completed?.status).toBe('completed');
   });
+
+  it('should forcefully terminate job when forceKillOnTimeout is true', async () => {
+    // Create a handler that ignores the abort signal (simulating a handler that doesn't check signal.aborted)
+    // Note: We use a real function (not vi.fn) because vi.fn doesn't serialize properly for worker threads
+    const handler: JobHandler<{ test: {} }, 'test'> = async (
+      _payload,
+      _signal,
+    ) => {
+      // This handler will run indefinitely, ignoring the abort signal
+      await new Promise((resolve) => {
+        setTimeout(resolve, 1000); // Will never complete in time
+      });
+    };
+    const handlers: { test: JobHandler<{ test: {} }, 'test'> } = {
+      test: handler,
+    };
+    const jobId = await queue.addJob<{ test: {} }, 'test'>(pool, {
+      jobType: 'test',
+      payload: {},
+      timeoutMs: 50, // 50ms timeout
+      forceKillOnTimeout: true, // Force kill on timeout
+    });
+    const job = await queue.getJob<{ test: {} }, 'test'>(pool, jobId);
+    expect(job).not.toBeNull();
+    expect(job?.forceKillOnTimeout).toBe(true);
+    await processJobWithHandlers(pool, job!, handlers);
+    const failed = await queue.getJob(pool, jobId);
+    expect(failed?.status).toBe('failed');
+    expect(failed?.errorHistory?.[0]?.message).toContain('timed out');
+    expect(failed?.failureReason).toBe(FailureReason.Timeout);
+  });
+
+  it('should complete job with forceKillOnTimeout if handler finishes before timeout', async () => {
+    // Note: We use a real function (not vi.fn) because vi.fn doesn't serialize properly for worker threads
+    const handler: JobHandler<{ test: {} }, 'test'> = async (
+      _payload,
+      _signal,
+    ) => {
+      await new Promise((r) => setTimeout(r, 20));
+    };
+    const handlers: { test: JobHandler<{ test: {} }, 'test'> } = {
+      test: handler,
+    };
+    const jobId = await queue.addJob<{ test: {} }, 'test'>(pool, {
+      jobType: 'test',
+      payload: {},
+      timeoutMs: 200, // 200ms
+      forceKillOnTimeout: true,
+    });
+    const job = await queue.getJob<{ test: {} }, 'test'>(pool, jobId);
+    expect(job).not.toBeNull();
+    await processJobWithHandlers(pool, job!, handlers);
+    const completed = await queue.getJob(pool, jobId);
+    expect(completed?.status).toBe('completed');
+  });
 });

package/src/processor.ts

@@ -1,4 +1,5 @@
 import { Pool } from 'pg';
+import { Worker } from 'worker_threads';
 import {
   JobRecord,
   ProcessorOptions,
@@ -16,6 +17,242 @@ import {
 } from './queue.js';
 import { log, setLogContext } from './log-context.js';
 
+/**
+ * Validates that a handler can be serialized for worker thread execution.
+ * Throws an error with helpful message if serialization fails.
+ */
+function validateHandlerSerializable<
+  PayloadMap,
+  T extends keyof PayloadMap & string,
+>(handler: JobHandler<PayloadMap, T>, jobType: string): void {
+  try {
+    const handlerString = handler.toString();
+
+    // Check for common patterns that indicate non-serializable handlers
+    // 1. Arrow functions that capture 'this' (indicated by 'this' in the function body but not in parameters)
+    if (
+      handlerString.includes('this.') &&
+      !handlerString.match(/\([^)]*this[^)]*\)/)
+    ) {
+      throw new Error(
+        `Handler for job type "${jobType}" uses 'this' context which cannot be serialized. ` +
+          `Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`,
+      );
+    }
+
+    // 2. Check if handler string looks like it might have closures
+    // This is a heuristic - we can't perfectly detect closures, but we can warn about common patterns
+    if (handlerString.includes('[native code]')) {
+      throw new Error(
+        `Handler for job type "${jobType}" contains native code which cannot be serialized. ` +
+          `Ensure your handler is a plain function when forceKillOnTimeout is enabled.`,
+      );
+    }
+
+    // 3. Try to create a function from the string to validate it's parseable
+    // This will catch syntax errors early
+    try {
+      new Function('return ' + handlerString);
+    } catch (parseError) {
+      throw new Error(
+        `Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. ` +
+          `When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`,
+      );
+    }
+  } catch (error) {
+    if (error instanceof Error) {
+      throw error;
+    }
+    throw new Error(
+      `Failed to validate handler serialization for job type "${jobType}": ${String(error)}`,
+    );
+  }
+}
+
+/**
+ * Run a handler in a worker thread for force-kill capability.
+ *
+ * **IMPORTANT**: The handler must be serializable for this to work. This means:
+ * - The handler should be a standalone function or arrow function
+ * - It should not capture variables from outer scopes (closures) that reference external dependencies
+ * - It should not use 'this' context unless it's a bound method
+ * - All dependencies must be importable in the worker thread context
+ *
+ * If your handler doesn't meet these requirements, use the default graceful shutdown
+ * (forceKillOnTimeout: false) and ensure your handler checks signal.aborted.
+ *
+ * @throws {Error} If the handler cannot be serialized
+ */
+async function runHandlerInWorker<
+  PayloadMap,
+  T extends keyof PayloadMap & string,
+>(
+  handler: JobHandler<PayloadMap, T>,
+  payload: PayloadMap[T],
+  timeoutMs: number,
+  jobType: string,
+): Promise<void> {
+  // Validate handler can be serialized before attempting to run in worker
+  validateHandlerSerializable(handler, jobType);
+
+  return new Promise((resolve, reject) => {
+    // Use inline worker code for better compatibility
+    // Note: This requires the handler to be serializable (no closures with external dependencies)
+    // Wrap in IIFE to allow return statements
+    const workerCode = `
+      (function() {
+        const { parentPort, workerData } = require('worker_threads');
+        const { handlerCode, payload, timeoutMs } = workerData;
+        
+        // Create an AbortController for the handler
+        const controller = new AbortController();
+        const signal = controller.signal;
+        
+        // Set up timeout
+        const timeoutId = setTimeout(() => {
+          controller.abort();
+          parentPort.postMessage({ type: 'timeout' });
+        }, timeoutMs);
+        
+        try {
+          // Execute the handler
+          // Note: This uses Function constructor which requires the handler to be serializable.
+          // The handler should be validated before reaching this point.
+          let handlerFn;
+          try {
+            // Wrap handlerCode in parentheses to ensure it's treated as an expression
+            // This handles both arrow functions and regular functions
+            const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
+              ? handlerCode
+              : '(' + handlerCode + ')';
+            handlerFn = new Function('return ' + wrappedCode)();
+          } catch (parseError) {
+            clearTimeout(timeoutId);
+            parentPort.postMessage({
+              type: 'error',
+              error: {
+                message: 'Handler cannot be deserialized in worker thread. ' +
+                  'Ensure your handler is a standalone function without closures over external variables. ' +
+                  'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
+                stack: parseError instanceof Error ? parseError.stack : undefined,
+                name: 'SerializationError',
+              },
+            });
+            return;
+          }
+          
+          // Ensure handlerFn is actually a function
+          if (typeof handlerFn !== 'function') {
+            clearTimeout(timeoutId);
+            parentPort.postMessage({
+              type: 'error',
+              error: {
+                message: 'Handler deserialization did not produce a function. ' +
+                  'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
+                name: 'SerializationError',
+              },
+            });
+            return;
+          }
+          
+          handlerFn(payload, signal)
+            .then(() => {
+              clearTimeout(timeoutId);
+              parentPort.postMessage({ type: 'success' });
+            })
+            .catch((error) => {
+              clearTimeout(timeoutId);
+              parentPort.postMessage({
+                type: 'error',
+                error: {
+                  message: error.message,
+                  stack: error.stack,
+                  name: error.name,
+                },
+              });
+            });
+        } catch (error) {
+          clearTimeout(timeoutId);
+          parentPort.postMessage({
+            type: 'error',
+            error: {
+              message: error.message,
+              stack: error.stack,
+              name: error.name,
+            },
+          });
+        }
+      })();
+    `;
+
+    const worker = new Worker(workerCode, {
+      eval: true,
+      workerData: {
+        handlerCode: handler.toString(),
+        payload,
+        timeoutMs,
+      },
+    });
+
+    let resolved = false;
+
+    worker.on('message', (message: { type: string; error?: any }) => {
+      if (resolved) return;
+      resolved = true;
+
+      if (message.type === 'success') {
+        resolve();
+      } else if (message.type === 'timeout') {
+        const timeoutError = new Error(
+          `Job timed out after ${timeoutMs} ms and was forcefully terminated`,
+        );
+        // @ts-ignore
+        timeoutError.failureReason = FailureReason.Timeout;
+        reject(timeoutError);
+      } else if (message.type === 'error') {
+        const error = new Error(message.error.message);
+        error.stack = message.error.stack;
+        error.name = message.error.name;
+        reject(error);
+      }
+    });
+
+    worker.on('error', (error) => {
+      if (resolved) return;
+      resolved = true;
+      reject(error);
+    });
+
+    worker.on('exit', (code) => {
+      if (resolved) return;
+      if (code !== 0) {
+        resolved = true;
+        reject(new Error(`Worker stopped with exit code ${code}`));
+      }
+    });
+
+    // Force terminate worker on timeout
+    setTimeout(() => {
+      if (!resolved) {
+        resolved = true;
+        worker
+          .terminate()
+          .then(() => {
+            const timeoutError = new Error(
+              `Job timed out after ${timeoutMs} ms and was forcefully terminated`,
+            );
+            // @ts-ignore
+            timeoutError.failureReason = FailureReason.Timeout;
+            reject(timeoutError);
+          })
+          .catch((err) => {
+            reject(err);
+          });
+      }
+    }, timeoutMs + 100); // Small buffer to ensure timeout is handled
+  });
+}
+
 /**
  * Process a single job using the provided handler map
  */
@@ -46,27 +283,34 @@ export async function processJobWithHandlers<
 
   // Per-job timeout logic
   const timeoutMs = job.timeoutMs ?? undefined;
+  const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
   let timeoutId: NodeJS.Timeout | undefined;
   const controller = new AbortController();
   try {
-    const jobPromise = handler(job.payload, controller.signal);
-    if (timeoutMs && timeoutMs > 0) {
-      await Promise.race([
-        jobPromise,
-        new Promise((_, reject) => {
-          timeoutId = setTimeout(() => {
-            controller.abort();
-            const timeoutError = new Error(
-              `Job timed out after ${timeoutMs} ms`,
-            );
-            // @ts-ignore
-            timeoutError.failureReason = FailureReason.Timeout;
-            reject(timeoutError);
-          }, timeoutMs);
-        }),
-      ]);
+    // If forceKillOnTimeout is true, run handler in a worker thread
+    if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
+      await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
     } else {
-      await jobPromise;
+      // Default: graceful shutdown with AbortController
+      const jobPromise = handler(job.payload, controller.signal);
+      if (timeoutMs && timeoutMs > 0) {
+        await Promise.race([
+          jobPromise,
+          new Promise((_, reject) => {
+            timeoutId = setTimeout(() => {
+              controller.abort();
+              const timeoutError = new Error(
+                `Job timed out after ${timeoutMs} ms`,
+              );
+              // @ts-ignore
+              timeoutError.failureReason = FailureReason.Timeout;
+              reject(timeoutError);
+            }, timeoutMs);
+          }),
+        ]);
+      } else {
+        await jobPromise;
+      }
     }
     if (timeoutId) clearTimeout(timeoutId);
     await completeJob(pool, job.id);