@nicnocquee/dataqueue 1.22.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/dist/index.cjs +2822 -583
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +589 -12
- package/dist/index.d.ts +589 -12
- package/dist/index.js +2818 -584
- package/dist/index.js.map +1 -1
- package/migrations/1751131910825_add_timeout_seconds_to_job_queue.sql +2 -2
- package/migrations/1751186053000_add_job_events_table.sql +12 -8
- package/migrations/1751984773000_add_tags_to_job_queue.sql +1 -1
- package/migrations/1765809419000_add_force_kill_on_timeout_to_job_queue.sql +6 -0
- package/migrations/1771100000000_add_idempotency_key_to_job_queue.sql +7 -0
- package/migrations/1781200000000_add_wait_support.sql +12 -0
- package/migrations/1781200000001_create_waitpoints_table.sql +18 -0
- package/migrations/1781200000002_add_performance_indexes.sql +34 -0
- package/migrations/1781200000003_add_progress_to_job_queue.sql +7 -0
- package/package.json +20 -6
- package/src/backend.ts +163 -0
- package/src/backends/postgres.ts +1111 -0
- package/src/backends/redis-scripts.ts +533 -0
- package/src/backends/redis.test.ts +543 -0
- package/src/backends/redis.ts +834 -0
- package/src/db-util.ts +4 -2
- package/src/handler-validation.test.ts +414 -0
- package/src/handler-validation.ts +168 -0
- package/src/index.test.ts +230 -1
- package/src/index.ts +128 -32
- package/src/processor.test.ts +612 -16
- package/src/processor.ts +759 -47
- package/src/queue.test.ts +736 -3
- package/src/queue.ts +346 -660
- package/src/test-util.ts +32 -0
- package/src/types.ts +451 -16
- package/src/wait.test.ts +698 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as pg from 'pg';
|
|
1
2
|
import { Pool } from 'pg';
|
|
2
3
|
|
|
3
4
|
type JobType<PayloadMap> = keyof PayloadMap & string;
|
|
@@ -11,18 +12,89 @@ interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
11
12
|
* Timeout for this job in milliseconds. If not set, uses the processor default or unlimited.
|
|
12
13
|
*/
|
|
13
14
|
timeoutMs?: number;
|
|
15
|
+
/**
|
|
16
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
17
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
18
|
+
*
|
|
19
|
+
* **⚠️ RUNTIME REQUIREMENTS**: This option requires **Node.js** and uses the `worker_threads` module.
|
|
20
|
+
* It will **not work** in Bun or other runtimes that don't support Node.js worker threads.
|
|
21
|
+
*
|
|
22
|
+
* **IMPORTANT**: When `forceKillOnTimeout` is true, the handler must be serializable. This means:
|
|
23
|
+
* - The handler should be a standalone function (not a closure over external variables)
|
|
24
|
+
* - It should not capture variables from outer scopes that reference external dependencies
|
|
25
|
+
* - It should not use 'this' context unless it's a bound method
|
|
26
|
+
* - All dependencies must be importable in the worker thread context
|
|
27
|
+
*
|
|
28
|
+
* **Examples of serializable handlers:**
|
|
29
|
+
* ```ts
|
|
30
|
+
* // ✅ Good - standalone function
|
|
31
|
+
* const handler = async (payload, signal) => {
|
|
32
|
+
* await doSomething(payload);
|
|
33
|
+
* };
|
|
34
|
+
*
|
|
35
|
+
* // ✅ Good - function that imports dependencies
|
|
36
|
+
* const handler = async (payload, signal) => {
|
|
37
|
+
* const { api } = await import('./api');
|
|
38
|
+
* await api.call(payload);
|
|
39
|
+
* };
|
|
40
|
+
*
|
|
41
|
+
* // ❌ Bad - closure over external variable
|
|
42
|
+
* const db = getDatabase();
|
|
43
|
+
* const handler = async (payload, signal) => {
|
|
44
|
+
* await db.query(payload); // 'db' is captured from closure
|
|
45
|
+
* };
|
|
46
|
+
*
|
|
47
|
+
* // ❌ Bad - uses 'this' context
|
|
48
|
+
* class MyHandler {
|
|
49
|
+
* async handle(payload, signal) {
|
|
50
|
+
* await this.doSomething(payload); // 'this' won't work
|
|
51
|
+
* }
|
|
52
|
+
* }
|
|
53
|
+
* ```
|
|
54
|
+
*
|
|
55
|
+
* If your handler doesn't meet these requirements, use `forceKillOnTimeout: false` (default)
|
|
56
|
+
* and ensure your handler checks `signal.aborted` to exit gracefully.
|
|
57
|
+
*
|
|
58
|
+
* Note: forceKillOnTimeout requires timeoutMs to be set.
|
|
59
|
+
*/
|
|
60
|
+
forceKillOnTimeout?: boolean;
|
|
14
61
|
/**
|
|
15
62
|
* Tags for this job. Used for grouping, searching, or batch operations.
|
|
16
63
|
*/
|
|
17
64
|
tags?: string[];
|
|
65
|
+
/**
|
|
66
|
+
* Optional idempotency key. When provided, ensures that only one job exists for a given key.
|
|
67
|
+
* If a job with the same idempotency key already exists, `addJob` returns the existing job's ID
|
|
68
|
+
* instead of creating a duplicate.
|
|
69
|
+
*
|
|
70
|
+
* Useful for preventing duplicate jobs caused by retries, double-clicks, webhook replays,
|
|
71
|
+
* or serverless function re-invocations.
|
|
72
|
+
*
|
|
73
|
+
* The key is unique across the entire `job_queue` table regardless of job status.
|
|
74
|
+
* Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
|
|
75
|
+
*/
|
|
76
|
+
idempotencyKey?: string;
|
|
18
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Options for editing a pending job.
|
|
80
|
+
* All fields are optional and only provided fields will be updated.
|
|
81
|
+
* Note: jobType cannot be changed.
|
|
82
|
+
* timeoutMs and tags can be set to null to clear them.
|
|
83
|
+
*/
|
|
84
|
+
type EditJobOptions<PayloadMap, T extends JobType<PayloadMap>> = Partial<Omit<JobOptions<PayloadMap, T>, 'jobType'>> & {
|
|
85
|
+
timeoutMs?: number | null;
|
|
86
|
+
tags?: string[] | null;
|
|
87
|
+
};
|
|
19
88
|
declare enum JobEventType {
|
|
20
89
|
Added = "added",
|
|
21
90
|
Processing = "processing",
|
|
22
91
|
Completed = "completed",
|
|
23
92
|
Failed = "failed",
|
|
24
93
|
Cancelled = "cancelled",
|
|
25
|
-
Retried = "retried"
|
|
94
|
+
Retried = "retried",
|
|
95
|
+
Edited = "edited",
|
|
96
|
+
Prolonged = "prolonged",
|
|
97
|
+
Waiting = "waiting"
|
|
26
98
|
}
|
|
27
99
|
interface JobEvent {
|
|
28
100
|
id: number;
|
|
@@ -36,7 +108,7 @@ declare enum FailureReason {
|
|
|
36
108
|
HandlerError = "handler_error",
|
|
37
109
|
NoHandler = "no_handler"
|
|
38
110
|
}
|
|
39
|
-
type JobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled';
|
|
111
|
+
type JobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled' | 'waiting';
|
|
40
112
|
interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
41
113
|
id: number;
|
|
42
114
|
jobType: T;
|
|
@@ -60,6 +132,11 @@ interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
60
132
|
* Timeout for this job in milliseconds (null means no timeout).
|
|
61
133
|
*/
|
|
62
134
|
timeoutMs?: number | null;
|
|
135
|
+
/**
|
|
136
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
137
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
138
|
+
*/
|
|
139
|
+
forceKillOnTimeout?: boolean | null;
|
|
63
140
|
/**
|
|
64
141
|
* The reason for the last failure, if any.
|
|
65
142
|
*/
|
|
@@ -88,8 +165,190 @@ interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
88
165
|
* Tags for this job. Used for grouping, searching, or batch operations.
|
|
89
166
|
*/
|
|
90
167
|
tags?: string[];
|
|
168
|
+
/**
|
|
169
|
+
* The idempotency key for this job, if one was provided when the job was created.
|
|
170
|
+
*/
|
|
171
|
+
idempotencyKey?: string | null;
|
|
172
|
+
/**
|
|
173
|
+
* The time the job is waiting until (for time-based waits).
|
|
174
|
+
*/
|
|
175
|
+
waitUntil?: Date | null;
|
|
176
|
+
/**
|
|
177
|
+
* The waitpoint token ID the job is waiting for (for token-based waits).
|
|
178
|
+
*/
|
|
179
|
+
waitTokenId?: string | null;
|
|
180
|
+
/**
|
|
181
|
+
* Step data for the job. Stores completed step results for replay on re-invocation.
|
|
182
|
+
*/
|
|
183
|
+
stepData?: Record<string, any>;
|
|
184
|
+
/**
|
|
185
|
+
* Progress percentage for the job (0-100), or null if no progress has been reported.
|
|
186
|
+
* Updated by the handler via `ctx.setProgress(percent)`.
|
|
187
|
+
*/
|
|
188
|
+
progress?: number | null;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Callback registered via `onTimeout`. Invoked when the timeout fires, before the AbortSignal is triggered.
|
|
192
|
+
* Return a number (ms) to extend the timeout, or return nothing to let the timeout proceed.
|
|
193
|
+
*/
|
|
194
|
+
type OnTimeoutCallback = () => number | void | undefined;
|
|
195
|
+
/**
|
|
196
|
+
* Context object passed to job handlers as the third argument.
|
|
197
|
+
* Provides mechanisms to extend the job's timeout while it's running,
|
|
198
|
+
* as well as step tracking and wait capabilities.
|
|
199
|
+
*/
|
|
200
|
+
interface JobContext {
|
|
201
|
+
/**
|
|
202
|
+
* Proactively reset the timeout deadline.
|
|
203
|
+
* - If `ms` is provided, sets the deadline to `ms` milliseconds from now.
|
|
204
|
+
* - If omitted, resets the deadline to the original `timeoutMs` from now (heartbeat-style).
|
|
205
|
+
* - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
|
|
206
|
+
*/
|
|
207
|
+
prolong: (ms?: number) => void;
|
|
208
|
+
/**
|
|
209
|
+
* Register a callback that is invoked when the timeout fires, **before** the AbortSignal is triggered.
|
|
210
|
+
* - If the callback returns a number > 0, the timeout is reset to that many ms from now.
|
|
211
|
+
* - If the callback returns `undefined`, `null`, `0`, or a negative number, the timeout proceeds normally.
|
|
212
|
+
* - The callback may be invoked multiple times if the job keeps extending.
|
|
213
|
+
* - Only one callback can be registered; subsequent calls replace the previous one.
|
|
214
|
+
* - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
|
|
215
|
+
*/
|
|
216
|
+
onTimeout: (callback: OnTimeoutCallback) => void;
|
|
217
|
+
/**
|
|
218
|
+
* Execute a named step with memoization. If the step was already completed
|
|
219
|
+
* in a previous invocation (e.g., before a wait), the cached result is returned
|
|
220
|
+
* without re-executing the function.
|
|
221
|
+
*
|
|
222
|
+
* Step names must be unique within a handler and stable across re-invocations.
|
|
223
|
+
*
|
|
224
|
+
* @param stepName - A unique identifier for this step.
|
|
225
|
+
* @param fn - The function to execute. Its return value is cached.
|
|
226
|
+
* @returns The result of the step (from cache or fresh execution).
|
|
227
|
+
*/
|
|
228
|
+
run: <T>(stepName: string, fn: () => Promise<T>) => Promise<T>;
|
|
229
|
+
/**
|
|
230
|
+
* Wait for a specified duration before continuing execution.
|
|
231
|
+
* The job will be paused and resumed after the duration elapses.
|
|
232
|
+
*
|
|
233
|
+
* When this is called, the handler throws a WaitSignal internally.
|
|
234
|
+
* The job is set to 'waiting' status and will be re-invoked after the
|
|
235
|
+
* specified duration. All steps completed via `ctx.run()` before this
|
|
236
|
+
* call will be replayed from cache on re-invocation.
|
|
237
|
+
*
|
|
238
|
+
* @param duration - The duration to wait (e.g., `{ hours: 1 }`, `{ days: 7 }`).
|
|
239
|
+
*/
|
|
240
|
+
waitFor: (duration: WaitDuration) => Promise<void>;
|
|
241
|
+
/**
|
|
242
|
+
* Wait until a specific date/time before continuing execution.
|
|
243
|
+
* The job will be paused and resumed at (or after) the specified date.
|
|
244
|
+
*
|
|
245
|
+
* @param date - The date to wait until.
|
|
246
|
+
*/
|
|
247
|
+
waitUntil: (date: Date) => Promise<void>;
|
|
248
|
+
/**
|
|
249
|
+
* Create a waitpoint token. The token can be completed externally
|
|
250
|
+
* (by calling `jobQueue.completeToken()`) to resume a waiting job.
|
|
251
|
+
*
|
|
252
|
+
* Tokens can be created inside handlers or outside (via `jobQueue.createToken()`).
|
|
253
|
+
*
|
|
254
|
+
* @param options - Optional token configuration (timeout, tags).
|
|
255
|
+
* @returns A token object with `id` that can be passed to `waitForToken()`.
|
|
256
|
+
*/
|
|
257
|
+
createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
|
|
258
|
+
/**
|
|
259
|
+
* Wait for a waitpoint token to be completed by an external signal.
|
|
260
|
+
* The job will be paused until `jobQueue.completeToken(tokenId, data)` is called
|
|
261
|
+
* or the token times out.
|
|
262
|
+
*
|
|
263
|
+
* @param tokenId - The ID of the token to wait for.
|
|
264
|
+
* @returns A result object indicating success or timeout.
|
|
265
|
+
*/
|
|
266
|
+
waitForToken: <T = any>(tokenId: string) => Promise<WaitTokenResult<T>>;
|
|
267
|
+
/**
|
|
268
|
+
* Report progress for this job (0-100).
|
|
269
|
+
* The value is persisted to the database and can be read by clients
|
|
270
|
+
* via `getJob()` or the React SDK's `useJob()` hook.
|
|
271
|
+
*
|
|
272
|
+
* @param percent - Progress percentage (0-100). Values are rounded to the nearest integer.
|
|
273
|
+
* @throws If percent is outside the 0-100 range.
|
|
274
|
+
*/
|
|
275
|
+
setProgress: (percent: number) => Promise<void>;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Duration specification for `ctx.waitFor()`.
|
|
279
|
+
* At least one field must be provided. Fields are additive.
|
|
280
|
+
*/
|
|
281
|
+
interface WaitDuration {
|
|
282
|
+
seconds?: number;
|
|
283
|
+
minutes?: number;
|
|
284
|
+
hours?: number;
|
|
285
|
+
days?: number;
|
|
286
|
+
weeks?: number;
|
|
287
|
+
months?: number;
|
|
288
|
+
years?: number;
|
|
91
289
|
}
|
|
92
|
-
|
|
290
|
+
/**
|
|
291
|
+
* Options for creating a waitpoint token.
|
|
292
|
+
*/
|
|
293
|
+
interface CreateTokenOptions {
|
|
294
|
+
/**
|
|
295
|
+
* Maximum time to wait for the token to be completed.
|
|
296
|
+
* Accepts a duration string like '10m', '1h', '24h', '7d'.
|
|
297
|
+
* If not provided, the token has no timeout.
|
|
298
|
+
*/
|
|
299
|
+
timeout?: string;
|
|
300
|
+
/**
|
|
301
|
+
* Tags to attach to the token for filtering.
|
|
302
|
+
*/
|
|
303
|
+
tags?: string[];
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* A waitpoint token returned by `ctx.createToken()`.
|
|
307
|
+
*/
|
|
308
|
+
interface WaitToken {
|
|
309
|
+
/** The unique token ID. */
|
|
310
|
+
id: string;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Result of `ctx.waitForToken()`.
|
|
314
|
+
*/
|
|
315
|
+
type WaitTokenResult<T = any> = {
|
|
316
|
+
ok: true;
|
|
317
|
+
output: T;
|
|
318
|
+
} | {
|
|
319
|
+
ok: false;
|
|
320
|
+
error: string;
|
|
321
|
+
};
|
|
322
|
+
/**
|
|
323
|
+
* Internal signal thrown by wait methods to pause handler execution.
|
|
324
|
+
* This is not a real error -- the processor catches it and transitions the job to 'waiting' status.
|
|
325
|
+
*/
|
|
326
|
+
declare class WaitSignal extends Error {
|
|
327
|
+
readonly type: 'duration' | 'date' | 'token';
|
|
328
|
+
readonly waitUntil: Date | undefined;
|
|
329
|
+
readonly tokenId: string | undefined;
|
|
330
|
+
readonly stepData: Record<string, any>;
|
|
331
|
+
readonly isWaitSignal = true;
|
|
332
|
+
constructor(type: 'duration' | 'date' | 'token', waitUntil: Date | undefined, tokenId: string | undefined, stepData: Record<string, any>);
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Status of a waitpoint token.
|
|
336
|
+
*/
|
|
337
|
+
type WaitpointStatus = 'waiting' | 'completed' | 'timed_out';
|
|
338
|
+
/**
|
|
339
|
+
* A waitpoint record from the database.
|
|
340
|
+
*/
|
|
341
|
+
interface WaitpointRecord {
|
|
342
|
+
id: string;
|
|
343
|
+
jobId: number | null;
|
|
344
|
+
status: WaitpointStatus;
|
|
345
|
+
output: any;
|
|
346
|
+
timeoutAt: Date | null;
|
|
347
|
+
createdAt: Date;
|
|
348
|
+
completedAt: Date | null;
|
|
349
|
+
tags: string[] | null;
|
|
350
|
+
}
|
|
351
|
+
type JobHandler<PayloadMap, T extends keyof PayloadMap> = (payload: PayloadMap[T], signal: AbortSignal, ctx: JobContext) => Promise<void>;
|
|
93
352
|
type JobHandlers<PayloadMap> = {
|
|
94
353
|
[K in keyof PayloadMap]: JobHandler<PayloadMap, K>;
|
|
95
354
|
};
|
|
@@ -133,8 +392,18 @@ interface Processor {
|
|
|
133
392
|
startInBackground: () => void;
|
|
134
393
|
/**
|
|
135
394
|
* Stop the job processor that runs in the background.
|
|
395
|
+
* Does not wait for in-flight jobs to complete.
|
|
136
396
|
*/
|
|
137
397
|
stop: () => void;
|
|
398
|
+
/**
|
|
399
|
+
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
400
|
+
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
401
|
+
* No new batches will be started after calling this method.
|
|
402
|
+
*
|
|
403
|
+
* @param timeoutMs - Maximum time to wait for in-flight jobs (default: 30000ms).
|
|
404
|
+
* If jobs don't complete within this time, the promise resolves anyway.
|
|
405
|
+
*/
|
|
406
|
+
stopAndDrain: (timeoutMs?: number) => Promise<void>;
|
|
138
407
|
/**
|
|
139
408
|
* Check if the job processor is running.
|
|
140
409
|
*/
|
|
@@ -165,7 +434,12 @@ interface DatabaseSSLConfig {
|
|
|
165
434
|
*/
|
|
166
435
|
rejectUnauthorized?: boolean;
|
|
167
436
|
}
|
|
168
|
-
|
|
437
|
+
/**
|
|
438
|
+
* Configuration for PostgreSQL backend (default).
|
|
439
|
+
* Backward-compatible: omitting `backend` defaults to 'postgres'.
|
|
440
|
+
*/
|
|
441
|
+
interface PostgresJobQueueConfig {
|
|
442
|
+
backend?: 'postgres';
|
|
169
443
|
databaseConfig: {
|
|
170
444
|
connectionString?: string;
|
|
171
445
|
host?: string;
|
|
@@ -177,6 +451,44 @@ interface JobQueueConfig {
|
|
|
177
451
|
};
|
|
178
452
|
verbose?: boolean;
|
|
179
453
|
}
|
|
454
|
+
/**
|
|
455
|
+
* TLS configuration for the Redis connection.
|
|
456
|
+
*/
|
|
457
|
+
interface RedisTLSConfig {
|
|
458
|
+
ca?: string;
|
|
459
|
+
cert?: string;
|
|
460
|
+
key?: string;
|
|
461
|
+
rejectUnauthorized?: boolean;
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Configuration for Redis backend.
|
|
465
|
+
*/
|
|
466
|
+
interface RedisJobQueueConfig {
|
|
467
|
+
backend: 'redis';
|
|
468
|
+
redisConfig: {
|
|
469
|
+
/** Redis URL (e.g. redis://localhost:6379) */
|
|
470
|
+
url?: string;
|
|
471
|
+
host?: string;
|
|
472
|
+
port?: number;
|
|
473
|
+
password?: string;
|
|
474
|
+
/** Redis database number (default: 0) */
|
|
475
|
+
db?: number;
|
|
476
|
+
tls?: RedisTLSConfig;
|
|
477
|
+
/**
|
|
478
|
+
* Key prefix for all Redis keys (default: 'dq:').
|
|
479
|
+
* Useful to namespace multiple queues in the same Redis instance.
|
|
480
|
+
*/
|
|
481
|
+
keyPrefix?: string;
|
|
482
|
+
};
|
|
483
|
+
verbose?: boolean;
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Job queue configuration — discriminated union.
|
|
487
|
+
* If `backend` is omitted, PostgreSQL is used.
|
|
488
|
+
*/
|
|
489
|
+
type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
|
|
490
|
+
/** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
|
|
491
|
+
type JobQueueConfigLegacy = PostgresJobQueueConfig;
|
|
180
492
|
type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
|
|
181
493
|
interface JobQueue<PayloadMap> {
|
|
182
494
|
/**
|
|
@@ -209,9 +521,10 @@ interface JobQueue<PayloadMap> {
|
|
|
209
521
|
*/
|
|
210
522
|
getAllJobs: <T extends JobType<PayloadMap>>(limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
211
523
|
/**
|
|
212
|
-
* Get jobs by filters.
|
|
213
|
-
|
|
214
|
-
*
|
|
524
|
+
* Get jobs by filters, with pagination support.
|
|
525
|
+
* - Use `cursor` for efficient keyset pagination (recommended for large datasets).
|
|
526
|
+
* - Use `limit` and `offset` for traditional pagination.
|
|
527
|
+
* - Do not combine `cursor` with `offset`.
|
|
215
528
|
*/
|
|
216
529
|
getJobs: <T extends JobType<PayloadMap>>(filters?: {
|
|
217
530
|
jobType?: string;
|
|
@@ -227,7 +540,9 @@ interface JobQueue<PayloadMap> {
|
|
|
227
540
|
values: string[];
|
|
228
541
|
mode?: TagQueryMode;
|
|
229
542
|
};
|
|
230
|
-
|
|
543
|
+
/** Cursor for keyset pagination. Only return jobs with id < cursor. */
|
|
544
|
+
cursor?: number;
|
|
545
|
+
}, limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
231
546
|
/**
|
|
232
547
|
* Retry a job given its ID.
|
|
233
548
|
* - This will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
|
|
@@ -237,11 +552,51 @@ interface JobQueue<PayloadMap> {
|
|
|
237
552
|
* Cleanup jobs that are older than the specified number of days.
|
|
238
553
|
*/
|
|
239
554
|
cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
|
|
555
|
+
/**
|
|
556
|
+
* Cleanup job events that are older than the specified number of days.
|
|
557
|
+
*/
|
|
558
|
+
cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
|
|
240
559
|
/**
|
|
241
560
|
* Cancel a job given its ID.
|
|
242
561
|
* - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
|
|
243
562
|
*/
|
|
244
563
|
cancelJob: (jobId: number) => Promise<void>;
|
|
564
|
+
/**
|
|
565
|
+
* Edit a pending job given its ID.
|
|
566
|
+
* - Only works for jobs with status 'pending'. Silently fails for other statuses.
|
|
567
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
568
|
+
* - jobType cannot be changed.
|
|
569
|
+
* - Records an 'edited' event with the updated fields in metadata.
|
|
570
|
+
*/
|
|
571
|
+
editJob: <T extends JobType<PayloadMap>>(jobId: number, updates: EditJobOptions<PayloadMap, T>) => Promise<void>;
|
|
572
|
+
/**
|
|
573
|
+
* Edit all pending jobs that match the filters.
|
|
574
|
+
* - Only works for jobs with status 'pending'. Non-pending jobs are not affected.
|
|
575
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
576
|
+
* - jobType cannot be changed.
|
|
577
|
+
* - Records an 'edited' event with the updated fields in metadata for each affected job.
|
|
578
|
+
* - Returns the number of jobs that were edited.
|
|
579
|
+
* - The filters are:
|
|
580
|
+
* - jobType: The job type to edit.
|
|
581
|
+
* - priority: The priority of the job to edit.
|
|
582
|
+
* - runAt: The time the job is scheduled to run at (now supports gt/gte/lt/lte/eq).
|
|
583
|
+
* - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based editing.
|
|
584
|
+
*/
|
|
585
|
+
editAllPendingJobs: <T extends JobType<PayloadMap>>(filters: {
|
|
586
|
+
jobType?: string;
|
|
587
|
+
priority?: number;
|
|
588
|
+
runAt?: Date | {
|
|
589
|
+
gt?: Date;
|
|
590
|
+
gte?: Date;
|
|
591
|
+
lt?: Date;
|
|
592
|
+
lte?: Date;
|
|
593
|
+
eq?: Date;
|
|
594
|
+
};
|
|
595
|
+
tags?: {
|
|
596
|
+
values: string[];
|
|
597
|
+
mode?: TagQueryMode;
|
|
598
|
+
};
|
|
599
|
+
} | undefined, updates: EditJobOptions<PayloadMap, T>) => Promise<number>;
|
|
245
600
|
/**
|
|
246
601
|
* Reclaim stuck jobs.
|
|
247
602
|
* - If a process (e.g., API route or worker) crashes after marking a job as 'processing' but before completing it, the job can remain stuck in the 'processing' state indefinitely. This can happen if the process is killed or encounters an unhandled error after updating the job status but before marking it as 'completed' or 'failed'.
|
|
@@ -283,14 +638,236 @@ interface JobQueue<PayloadMap> {
|
|
|
283
638
|
*/
|
|
284
639
|
getJobEvents: (jobId: number) => Promise<JobEvent[]>;
|
|
285
640
|
/**
|
|
286
|
-
*
|
|
641
|
+
* Create a waitpoint token.
|
|
642
|
+
* Tokens can be completed externally to resume a waiting job.
|
|
643
|
+
* Can be called outside of handlers (e.g., from an API route).
|
|
644
|
+
*
|
|
645
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
646
|
+
*
|
|
647
|
+
* @param options - Optional token configuration (timeout, tags).
|
|
648
|
+
* @returns A token object with `id`.
|
|
649
|
+
*/
|
|
650
|
+
createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
|
|
651
|
+
/**
|
|
652
|
+
* Complete a waitpoint token, resuming the associated waiting job.
|
|
653
|
+
* Can be called from anywhere (API routes, external services, etc.).
|
|
654
|
+
*
|
|
655
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
656
|
+
*
|
|
657
|
+
* @param tokenId - The ID of the token to complete.
|
|
658
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
659
|
+
*/
|
|
660
|
+
completeToken: (tokenId: string, data?: any) => Promise<void>;
|
|
661
|
+
/**
|
|
662
|
+
* Retrieve a waitpoint token by its ID.
|
|
663
|
+
*
|
|
664
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
665
|
+
*
|
|
666
|
+
* @param tokenId - The ID of the token to retrieve.
|
|
667
|
+
* @returns The token record, or null if not found.
|
|
287
668
|
*/
|
|
288
|
-
|
|
669
|
+
getToken: (tokenId: string) => Promise<WaitpointRecord | null>;
|
|
670
|
+
/**
|
|
671
|
+
* Expire timed-out waitpoint tokens and resume their associated jobs.
|
|
672
|
+
* Call this periodically (e.g., alongside `reclaimStuckJobs`).
|
|
673
|
+
*
|
|
674
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
675
|
+
*
|
|
676
|
+
* @returns The number of tokens that were expired.
|
|
677
|
+
*/
|
|
678
|
+
expireTimedOutTokens: () => Promise<number>;
|
|
679
|
+
/**
|
|
680
|
+
* Get the PostgreSQL database pool.
|
|
681
|
+
* Throws if the backend is not PostgreSQL.
|
|
682
|
+
*/
|
|
683
|
+
getPool: () => pg.Pool;
|
|
684
|
+
/**
|
|
685
|
+
* Get the Redis client instance (ioredis).
|
|
686
|
+
* Throws if the backend is not Redis.
|
|
687
|
+
*/
|
|
688
|
+
getRedisClient: () => unknown;
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Filter options used by getJobs, cancelAllUpcomingJobs, editAllPendingJobs
|
|
693
|
+
*/
|
|
694
|
+
interface JobFilters {
|
|
695
|
+
jobType?: string;
|
|
696
|
+
priority?: number;
|
|
697
|
+
runAt?: Date | {
|
|
698
|
+
gt?: Date;
|
|
699
|
+
gte?: Date;
|
|
700
|
+
lt?: Date;
|
|
701
|
+
lte?: Date;
|
|
702
|
+
eq?: Date;
|
|
703
|
+
};
|
|
704
|
+
tags?: {
|
|
705
|
+
values: string[];
|
|
706
|
+
mode?: TagQueryMode;
|
|
707
|
+
};
|
|
708
|
+
/**
|
|
709
|
+
* Cursor for keyset pagination. When provided, only return jobs with id < cursor.
|
|
710
|
+
* This is more efficient than OFFSET for large datasets.
|
|
711
|
+
* Cannot be used together with offset.
|
|
712
|
+
*/
|
|
713
|
+
cursor?: number;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Fields that can be updated on a job
|
|
717
|
+
*/
|
|
718
|
+
interface JobUpdates {
|
|
719
|
+
payload?: any;
|
|
720
|
+
maxAttempts?: number;
|
|
721
|
+
priority?: number;
|
|
722
|
+
runAt?: Date | null;
|
|
723
|
+
timeoutMs?: number | null;
|
|
724
|
+
tags?: string[] | null;
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Abstract backend interface that both PostgreSQL and Redis implement.
|
|
728
|
+
* All storage operations go through this interface so the processor
|
|
729
|
+
* and public API are backend-agnostic.
|
|
730
|
+
*/
|
|
731
|
+
interface QueueBackend {
|
|
732
|
+
/** Add a job and return its numeric ID. */
|
|
733
|
+
addJob<PayloadMap, T extends JobType<PayloadMap>>(job: JobOptions<PayloadMap, T>): Promise<number>;
|
|
734
|
+
/** Get a single job by ID, or null if not found. */
|
|
735
|
+
getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
|
|
736
|
+
/** Get jobs filtered by status, ordered by createdAt DESC. */
|
|
737
|
+
getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
738
|
+
/** Get all jobs, ordered by createdAt DESC. */
|
|
739
|
+
getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
740
|
+
/** Get jobs matching arbitrary filters, ordered by createdAt DESC. */
|
|
741
|
+
getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
742
|
+
/** Get jobs by tag(s) with query mode. */
|
|
743
|
+
getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
744
|
+
/**
|
|
745
|
+
* Atomically claim a batch of ready jobs for the given worker.
|
|
746
|
+
* Equivalent to SELECT … FOR UPDATE SKIP LOCKED in Postgres.
|
|
747
|
+
*/
|
|
748
|
+
getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
|
|
749
|
+
/** Mark a job as completed. */
|
|
750
|
+
completeJob(jobId: number): Promise<void>;
|
|
751
|
+
/** Mark a job as failed with error info and schedule retry. */
|
|
752
|
+
failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
|
|
753
|
+
/** Update locked_at to keep the job alive (heartbeat). */
|
|
754
|
+
prolongJob(jobId: number): Promise<void>;
|
|
755
|
+
/** Retry a failed/cancelled job immediately. */
|
|
756
|
+
retryJob(jobId: number): Promise<void>;
|
|
757
|
+
/** Cancel a pending job. */
|
|
758
|
+
cancelJob(jobId: number): Promise<void>;
|
|
759
|
+
/** Cancel all pending jobs matching optional filters. Returns count. */
|
|
760
|
+
cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
|
|
761
|
+
/** Edit a single pending job. */
|
|
762
|
+
editJob(jobId: number, updates: JobUpdates): Promise<void>;
|
|
763
|
+
/** Edit all pending jobs matching filters. Returns count. */
|
|
764
|
+
editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
|
|
765
|
+
/** Delete completed jobs older than N days. Returns count deleted. */
|
|
766
|
+
cleanupOldJobs(daysToKeep?: number): Promise<number>;
|
|
767
|
+
/** Delete job events older than N days. Returns count deleted. */
|
|
768
|
+
cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
|
|
769
|
+
/** Reclaim jobs stuck in 'processing' for too long. Returns count. */
|
|
770
|
+
reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
|
|
771
|
+
/** Update the progress percentage (0-100) for a job. */
|
|
772
|
+
updateProgress(jobId: number, progress: number): Promise<void>;
|
|
773
|
+
/** Record a job event. Should not throw. */
|
|
774
|
+
recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
|
|
775
|
+
/** Get all events for a job, ordered by createdAt ASC. */
|
|
776
|
+
getJobEvents(jobId: number): Promise<JobEvent[]>;
|
|
777
|
+
/** Set a pending reason for unpicked jobs of a given type. */
|
|
778
|
+
setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
|
|
289
779
|
}
|
|
290
780
|
|
|
781
|
+
declare class PostgresBackend implements QueueBackend {
|
|
782
|
+
private pool;
|
|
783
|
+
constructor(pool: Pool);
|
|
784
|
+
/** Expose the raw pool for advanced usage. */
|
|
785
|
+
getPool(): Pool;
|
|
786
|
+
recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
|
|
787
|
+
getJobEvents(jobId: number): Promise<JobEvent[]>;
|
|
788
|
+
addJob<PayloadMap, T extends JobType<PayloadMap>>({ jobType, payload, maxAttempts, priority, runAt, timeoutMs, forceKillOnTimeout, tags, idempotencyKey, }: JobOptions<PayloadMap, T>): Promise<number>;
|
|
789
|
+
getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
|
|
790
|
+
getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
791
|
+
getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
792
|
+
getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
793
|
+
getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
794
|
+
getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
|
|
795
|
+
completeJob(jobId: number): Promise<void>;
|
|
796
|
+
failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
|
|
797
|
+
prolongJob(jobId: number): Promise<void>;
|
|
798
|
+
updateProgress(jobId: number, progress: number): Promise<void>;
|
|
799
|
+
retryJob(jobId: number): Promise<void>;
|
|
800
|
+
cancelJob(jobId: number): Promise<void>;
|
|
801
|
+
cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
|
|
802
|
+
editJob(jobId: number, updates: JobUpdates): Promise<void>;
|
|
803
|
+
editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
|
|
804
|
+
cleanupOldJobs(daysToKeep?: number): Promise<number>;
|
|
805
|
+
cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
|
|
806
|
+
reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
|
|
807
|
+
/**
|
|
808
|
+
* Batch-insert multiple job events in a single query.
|
|
809
|
+
* More efficient than individual recordJobEvent calls.
|
|
810
|
+
*/
|
|
811
|
+
private recordJobEventsBatch;
|
|
812
|
+
setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Validates that a job handler can be serialized for use with forceKillOnTimeout.
|
|
817
|
+
*
|
|
818
|
+
* This function checks if a handler can be safely serialized and executed in a worker thread.
|
|
819
|
+
* Use this function during development to catch serialization issues early.
|
|
820
|
+
*
|
|
821
|
+
* @param handler - The job handler function to validate
|
|
822
|
+
* @param jobType - Optional job type name for better error messages
|
|
823
|
+
* @returns An object with `isSerializable` boolean and optional `error` message
|
|
824
|
+
*
|
|
825
|
+
* @example
|
|
826
|
+
* ```ts
|
|
827
|
+
* const handler = async (payload, signal) => {
|
|
828
|
+
* await doSomething(payload);
|
|
829
|
+
* };
|
|
830
|
+
*
|
|
831
|
+
* const result = validateHandlerSerializable(handler, 'myJob');
|
|
832
|
+
* if (!result.isSerializable) {
|
|
833
|
+
* console.error('Handler is not serializable:', result.error);
|
|
834
|
+
* }
|
|
835
|
+
* ```
|
|
836
|
+
*/
|
|
837
|
+
declare function validateHandlerSerializable<PayloadMap, T extends keyof PayloadMap & string>(handler: JobHandler<PayloadMap, T>, jobType?: string): {
|
|
838
|
+
isSerializable: boolean;
|
|
839
|
+
error?: string;
|
|
840
|
+
};
|
|
841
|
+
/**
|
|
842
|
+
* Test if a handler can be serialized and executed in a worker thread.
|
|
843
|
+
* This is a more thorough check that actually attempts to serialize and deserialize the handler.
|
|
844
|
+
*
|
|
845
|
+
* @param handler - The job handler function to test
|
|
846
|
+
* @param jobType - Optional job type name for better error messages
|
|
847
|
+
* @returns Promise that resolves to validation result
|
|
848
|
+
*
|
|
849
|
+
* @example
|
|
850
|
+
* ```ts
|
|
851
|
+
* const handler = async (payload, signal) => {
|
|
852
|
+
* await doSomething(payload);
|
|
853
|
+
* };
|
|
854
|
+
*
|
|
855
|
+
* const result = await testHandlerSerialization(handler, 'myJob');
|
|
856
|
+
* if (!result.isSerializable) {
|
|
857
|
+
* console.error('Handler failed serialization test:', result.error);
|
|
858
|
+
* }
|
|
859
|
+
* ```
|
|
860
|
+
*/
|
|
861
|
+
declare function testHandlerSerialization<PayloadMap, T extends keyof PayloadMap & string>(handler: JobHandler<PayloadMap, T>, jobType?: string): Promise<{
|
|
862
|
+
isSerializable: boolean;
|
|
863
|
+
error?: string;
|
|
864
|
+
}>;
|
|
865
|
+
|
|
291
866
|
/**
|
|
292
|
-
* Initialize the job queue system
|
|
867
|
+
* Initialize the job queue system.
|
|
868
|
+
*
|
|
869
|
+
* Defaults to PostgreSQL when `backend` is omitted.
|
|
293
870
|
*/
|
|
294
871
|
declare const initJobQueue: <PayloadMap = any>(config: JobQueueConfig) => JobQueue<PayloadMap>;
|
|
295
872
|
|
|
296
|
-
export { type DatabaseSSLConfig, FailureReason, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobRecord, type JobStatus, type JobType, type Processor, type ProcessorOptions, type TagQueryMode, initJobQueue };
|
|
873
|
+
export { type CreateTokenOptions, type DatabaseSSLConfig, type EditJobOptions, FailureReason, type JobContext, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobQueueConfigLegacy, type JobRecord, type JobStatus, type JobType, type OnTimeoutCallback, PostgresBackend, type PostgresJobQueueConfig, type Processor, type ProcessorOptions, type QueueBackend, type RedisJobQueueConfig, type RedisTLSConfig, type TagQueryMode, type WaitDuration, WaitSignal, type WaitToken, type WaitTokenResult, type WaitpointRecord, type WaitpointStatus, initJobQueue, testHandlerSerialization, validateHandlerSerializable };
|