@nicnocquee/dataqueue 1.26.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +88 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.cts +12 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +81 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +3968 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +1088 -0
- package/dist/index.d.ts +1088 -0
- package/dist/index.js +3953 -0
- package/dist/index.js.map +1 -0
- package/migrations/1781200000004_create_cron_schedules_table.sql +33 -0
- package/package.json +15 -18
- package/src/backend.ts +69 -0
- package/src/backends/postgres.ts +331 -1
- package/src/backends/redis.test.ts +350 -0
- package/src/backends/redis.ts +389 -1
- package/src/cron.test.ts +126 -0
- package/src/cron.ts +40 -0
- package/src/index.test.ts +361 -0
- package/src/index.ts +157 -4
- package/src/processor.ts +22 -4
- package/src/types.ts +149 -0
- package/LICENSE +0 -21
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,1088 @@
|
|
|
1
|
+
import * as pg from 'pg';
|
|
2
|
+
import { Pool } from 'pg';
|
|
3
|
+
import { Cron } from 'croner';
|
|
4
|
+
|
|
5
|
+
type JobType<PayloadMap> = keyof PayloadMap & string;
|
|
6
|
+
interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
7
|
+
jobType: T;
|
|
8
|
+
payload: PayloadMap[T];
|
|
9
|
+
maxAttempts?: number;
|
|
10
|
+
priority?: number;
|
|
11
|
+
runAt?: Date | null;
|
|
12
|
+
/**
|
|
13
|
+
* Timeout for this job in milliseconds. If not set, uses the processor default or unlimited.
|
|
14
|
+
*/
|
|
15
|
+
timeoutMs?: number;
|
|
16
|
+
/**
|
|
17
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
18
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
19
|
+
*
|
|
20
|
+
* **⚠️ RUNTIME REQUIREMENTS**: This option requires **Node.js** and uses the `worker_threads` module.
|
|
21
|
+
* It will **not work** in Bun or other runtimes that don't support Node.js worker threads.
|
|
22
|
+
*
|
|
23
|
+
* **IMPORTANT**: When `forceKillOnTimeout` is true, the handler must be serializable. This means:
|
|
24
|
+
* - The handler should be a standalone function (not a closure over external variables)
|
|
25
|
+
* - It should not capture variables from outer scopes that reference external dependencies
|
|
26
|
+
* - It should not use 'this' context unless it's a bound method
|
|
27
|
+
* - All dependencies must be importable in the worker thread context
|
|
28
|
+
*
|
|
29
|
+
* **Examples of serializable handlers:**
|
|
30
|
+
* ```ts
|
|
31
|
+
* // ✅ Good - standalone function
|
|
32
|
+
* const handler = async (payload, signal) => {
|
|
33
|
+
* await doSomething(payload);
|
|
34
|
+
* };
|
|
35
|
+
*
|
|
36
|
+
* // ✅ Good - function that imports dependencies
|
|
37
|
+
* const handler = async (payload, signal) => {
|
|
38
|
+
* const { api } = await import('./api');
|
|
39
|
+
* await api.call(payload);
|
|
40
|
+
* };
|
|
41
|
+
*
|
|
42
|
+
* // ❌ Bad - closure over external variable
|
|
43
|
+
* const db = getDatabase();
|
|
44
|
+
* const handler = async (payload, signal) => {
|
|
45
|
+
* await db.query(payload); // 'db' is captured from closure
|
|
46
|
+
* };
|
|
47
|
+
*
|
|
48
|
+
* // ❌ Bad - uses 'this' context
|
|
49
|
+
* class MyHandler {
|
|
50
|
+
* async handle(payload, signal) {
|
|
51
|
+
* await this.doSomething(payload); // 'this' won't work
|
|
52
|
+
* }
|
|
53
|
+
* }
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* If your handler doesn't meet these requirements, use `forceKillOnTimeout: false` (default)
|
|
57
|
+
* and ensure your handler checks `signal.aborted` to exit gracefully.
|
|
58
|
+
*
|
|
59
|
+
* Note: forceKillOnTimeout requires timeoutMs to be set.
|
|
60
|
+
*/
|
|
61
|
+
forceKillOnTimeout?: boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Tags for this job. Used for grouping, searching, or batch operations.
|
|
64
|
+
*/
|
|
65
|
+
tags?: string[];
|
|
66
|
+
/**
|
|
67
|
+
* Optional idempotency key. When provided, ensures that only one job exists for a given key.
|
|
68
|
+
* If a job with the same idempotency key already exists, `addJob` returns the existing job's ID
|
|
69
|
+
* instead of creating a duplicate.
|
|
70
|
+
*
|
|
71
|
+
* Useful for preventing duplicate jobs caused by retries, double-clicks, webhook replays,
|
|
72
|
+
* or serverless function re-invocations.
|
|
73
|
+
*
|
|
74
|
+
* The key is unique across the entire `job_queue` table regardless of job status.
|
|
75
|
+
* Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
|
|
76
|
+
*/
|
|
77
|
+
idempotencyKey?: string;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Options for editing a pending job.
|
|
81
|
+
* All fields are optional and only provided fields will be updated.
|
|
82
|
+
* Note: jobType cannot be changed.
|
|
83
|
+
* timeoutMs and tags can be set to null to clear them.
|
|
84
|
+
*/
|
|
85
|
+
type EditJobOptions<PayloadMap, T extends JobType<PayloadMap>> = Partial<Omit<JobOptions<PayloadMap, T>, 'jobType'>> & {
|
|
86
|
+
timeoutMs?: number | null;
|
|
87
|
+
tags?: string[] | null;
|
|
88
|
+
};
|
|
89
|
+
declare enum JobEventType {
|
|
90
|
+
Added = "added",
|
|
91
|
+
Processing = "processing",
|
|
92
|
+
Completed = "completed",
|
|
93
|
+
Failed = "failed",
|
|
94
|
+
Cancelled = "cancelled",
|
|
95
|
+
Retried = "retried",
|
|
96
|
+
Edited = "edited",
|
|
97
|
+
Prolonged = "prolonged",
|
|
98
|
+
Waiting = "waiting"
|
|
99
|
+
}
|
|
100
|
+
interface JobEvent {
|
|
101
|
+
id: number;
|
|
102
|
+
jobId: number;
|
|
103
|
+
eventType: JobEventType;
|
|
104
|
+
createdAt: Date;
|
|
105
|
+
metadata: any;
|
|
106
|
+
}
|
|
107
|
+
declare enum FailureReason {
|
|
108
|
+
Timeout = "timeout",
|
|
109
|
+
HandlerError = "handler_error",
|
|
110
|
+
NoHandler = "no_handler"
|
|
111
|
+
}
|
|
112
|
+
type JobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled' | 'waiting';
|
|
113
|
+
interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
114
|
+
id: number;
|
|
115
|
+
jobType: T;
|
|
116
|
+
payload: PayloadMap[T];
|
|
117
|
+
status: JobStatus;
|
|
118
|
+
createdAt: Date;
|
|
119
|
+
updatedAt: Date;
|
|
120
|
+
lockedAt: Date | null;
|
|
121
|
+
lockedBy: string | null;
|
|
122
|
+
attempts: number;
|
|
123
|
+
maxAttempts: number;
|
|
124
|
+
nextAttemptAt: Date | null;
|
|
125
|
+
priority: number;
|
|
126
|
+
runAt: Date;
|
|
127
|
+
pendingReason?: string | null;
|
|
128
|
+
errorHistory?: {
|
|
129
|
+
message: string;
|
|
130
|
+
timestamp: string;
|
|
131
|
+
}[];
|
|
132
|
+
/**
|
|
133
|
+
* Timeout for this job in milliseconds (null means no timeout).
|
|
134
|
+
*/
|
|
135
|
+
timeoutMs?: number | null;
|
|
136
|
+
/**
|
|
137
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
138
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
139
|
+
*/
|
|
140
|
+
forceKillOnTimeout?: boolean | null;
|
|
141
|
+
/**
|
|
142
|
+
* The reason for the last failure, if any.
|
|
143
|
+
*/
|
|
144
|
+
failureReason?: FailureReason | null;
|
|
145
|
+
/**
|
|
146
|
+
* The time the job was completed, if completed.
|
|
147
|
+
*/
|
|
148
|
+
completedAt: Date | null;
|
|
149
|
+
/**
|
|
150
|
+
* The time the job was first picked up for processing.
|
|
151
|
+
*/
|
|
152
|
+
startedAt: Date | null;
|
|
153
|
+
/**
|
|
154
|
+
* The time the job was last retried.
|
|
155
|
+
*/
|
|
156
|
+
lastRetriedAt: Date | null;
|
|
157
|
+
/**
|
|
158
|
+
* The time the job last failed.
|
|
159
|
+
*/
|
|
160
|
+
lastFailedAt: Date | null;
|
|
161
|
+
/**
|
|
162
|
+
* The time the job was last cancelled.
|
|
163
|
+
*/
|
|
164
|
+
lastCancelledAt: Date | null;
|
|
165
|
+
/**
|
|
166
|
+
* Tags for this job. Used for grouping, searching, or batch operations.
|
|
167
|
+
*/
|
|
168
|
+
tags?: string[];
|
|
169
|
+
/**
|
|
170
|
+
* The idempotency key for this job, if one was provided when the job was created.
|
|
171
|
+
*/
|
|
172
|
+
idempotencyKey?: string | null;
|
|
173
|
+
/**
|
|
174
|
+
* The time the job is waiting until (for time-based waits).
|
|
175
|
+
*/
|
|
176
|
+
waitUntil?: Date | null;
|
|
177
|
+
/**
|
|
178
|
+
* The waitpoint token ID the job is waiting for (for token-based waits).
|
|
179
|
+
*/
|
|
180
|
+
waitTokenId?: string | null;
|
|
181
|
+
/**
|
|
182
|
+
* Step data for the job. Stores completed step results for replay on re-invocation.
|
|
183
|
+
*/
|
|
184
|
+
stepData?: Record<string, any>;
|
|
185
|
+
/**
|
|
186
|
+
* Progress percentage for the job (0-100), or null if no progress has been reported.
|
|
187
|
+
* Updated by the handler via `ctx.setProgress(percent)`.
|
|
188
|
+
*/
|
|
189
|
+
progress?: number | null;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Callback registered via `onTimeout`. Invoked when the timeout fires, before the AbortSignal is triggered.
|
|
193
|
+
* Return a number (ms) to extend the timeout, or return nothing to let the timeout proceed.
|
|
194
|
+
*/
|
|
195
|
+
type OnTimeoutCallback = () => number | void | undefined;
|
|
196
|
+
/**
|
|
197
|
+
* Context object passed to job handlers as the third argument.
|
|
198
|
+
* Provides mechanisms to extend the job's timeout while it's running,
|
|
199
|
+
* as well as step tracking and wait capabilities.
|
|
200
|
+
*/
|
|
201
|
+
interface JobContext {
|
|
202
|
+
/**
|
|
203
|
+
* Proactively reset the timeout deadline.
|
|
204
|
+
* - If `ms` is provided, sets the deadline to `ms` milliseconds from now.
|
|
205
|
+
* - If omitted, resets the deadline to the original `timeoutMs` from now (heartbeat-style).
|
|
206
|
+
* - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
|
|
207
|
+
*/
|
|
208
|
+
prolong: (ms?: number) => void;
|
|
209
|
+
/**
|
|
210
|
+
* Register a callback that is invoked when the timeout fires, **before** the AbortSignal is triggered.
|
|
211
|
+
* - If the callback returns a number > 0, the timeout is reset to that many ms from now.
|
|
212
|
+
* - If the callback returns `undefined`, `null`, `0`, or a negative number, the timeout proceeds normally.
|
|
213
|
+
* - The callback may be invoked multiple times if the job keeps extending.
|
|
214
|
+
* - Only one callback can be registered; subsequent calls replace the previous one.
|
|
215
|
+
* - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
|
|
216
|
+
*/
|
|
217
|
+
onTimeout: (callback: OnTimeoutCallback) => void;
|
|
218
|
+
/**
|
|
219
|
+
* Execute a named step with memoization. If the step was already completed
|
|
220
|
+
* in a previous invocation (e.g., before a wait), the cached result is returned
|
|
221
|
+
* without re-executing the function.
|
|
222
|
+
*
|
|
223
|
+
* Step names must be unique within a handler and stable across re-invocations.
|
|
224
|
+
*
|
|
225
|
+
* @param stepName - A unique identifier for this step.
|
|
226
|
+
* @param fn - The function to execute. Its return value is cached.
|
|
227
|
+
* @returns The result of the step (from cache or fresh execution).
|
|
228
|
+
*/
|
|
229
|
+
run: <T>(stepName: string, fn: () => Promise<T>) => Promise<T>;
|
|
230
|
+
/**
|
|
231
|
+
* Wait for a specified duration before continuing execution.
|
|
232
|
+
* The job will be paused and resumed after the duration elapses.
|
|
233
|
+
*
|
|
234
|
+
* When this is called, the handler throws a WaitSignal internally.
|
|
235
|
+
* The job is set to 'waiting' status and will be re-invoked after the
|
|
236
|
+
* specified duration. All steps completed via `ctx.run()` before this
|
|
237
|
+
* call will be replayed from cache on re-invocation.
|
|
238
|
+
*
|
|
239
|
+
* @param duration - The duration to wait (e.g., `{ hours: 1 }`, `{ days: 7 }`).
|
|
240
|
+
*/
|
|
241
|
+
waitFor: (duration: WaitDuration) => Promise<void>;
|
|
242
|
+
/**
|
|
243
|
+
* Wait until a specific date/time before continuing execution.
|
|
244
|
+
* The job will be paused and resumed at (or after) the specified date.
|
|
245
|
+
*
|
|
246
|
+
* @param date - The date to wait until.
|
|
247
|
+
*/
|
|
248
|
+
waitUntil: (date: Date) => Promise<void>;
|
|
249
|
+
/**
|
|
250
|
+
* Create a waitpoint token. The token can be completed externally
|
|
251
|
+
* (by calling `jobQueue.completeToken()`) to resume a waiting job.
|
|
252
|
+
*
|
|
253
|
+
* Tokens can be created inside handlers or outside (via `jobQueue.createToken()`).
|
|
254
|
+
*
|
|
255
|
+
* @param options - Optional token configuration (timeout, tags).
|
|
256
|
+
* @returns A token object with `id` that can be passed to `waitForToken()`.
|
|
257
|
+
*/
|
|
258
|
+
createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
|
|
259
|
+
/**
|
|
260
|
+
* Wait for a waitpoint token to be completed by an external signal.
|
|
261
|
+
* The job will be paused until `jobQueue.completeToken(tokenId, data)` is called
|
|
262
|
+
* or the token times out.
|
|
263
|
+
*
|
|
264
|
+
* @param tokenId - The ID of the token to wait for.
|
|
265
|
+
* @returns A result object indicating success or timeout.
|
|
266
|
+
*/
|
|
267
|
+
waitForToken: <T = any>(tokenId: string) => Promise<WaitTokenResult<T>>;
|
|
268
|
+
/**
|
|
269
|
+
* Report progress for this job (0-100).
|
|
270
|
+
* The value is persisted to the database and can be read by clients
|
|
271
|
+
* via `getJob()` or the React SDK's `useJob()` hook.
|
|
272
|
+
*
|
|
273
|
+
* @param percent - Progress percentage (0-100). Values are rounded to the nearest integer.
|
|
274
|
+
* @throws If percent is outside the 0-100 range.
|
|
275
|
+
*/
|
|
276
|
+
setProgress: (percent: number) => Promise<void>;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Duration specification for `ctx.waitFor()`.
|
|
280
|
+
* At least one field must be provided. Fields are additive.
|
|
281
|
+
*/
|
|
282
|
+
interface WaitDuration {
|
|
283
|
+
seconds?: number;
|
|
284
|
+
minutes?: number;
|
|
285
|
+
hours?: number;
|
|
286
|
+
days?: number;
|
|
287
|
+
weeks?: number;
|
|
288
|
+
months?: number;
|
|
289
|
+
years?: number;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Options for creating a waitpoint token.
|
|
293
|
+
*/
|
|
294
|
+
interface CreateTokenOptions {
|
|
295
|
+
/**
|
|
296
|
+
* Maximum time to wait for the token to be completed.
|
|
297
|
+
* Accepts a duration string like '10m', '1h', '24h', '7d'.
|
|
298
|
+
* If not provided, the token has no timeout.
|
|
299
|
+
*/
|
|
300
|
+
timeout?: string;
|
|
301
|
+
/**
|
|
302
|
+
* Tags to attach to the token for filtering.
|
|
303
|
+
*/
|
|
304
|
+
tags?: string[];
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* A waitpoint token returned by `ctx.createToken()`.
|
|
308
|
+
*/
|
|
309
|
+
interface WaitToken {
|
|
310
|
+
/** The unique token ID. */
|
|
311
|
+
id: string;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Result of `ctx.waitForToken()`.
|
|
315
|
+
*/
|
|
316
|
+
type WaitTokenResult<T = any> = {
|
|
317
|
+
ok: true;
|
|
318
|
+
output: T;
|
|
319
|
+
} | {
|
|
320
|
+
ok: false;
|
|
321
|
+
error: string;
|
|
322
|
+
};
|
|
323
|
+
/**
|
|
324
|
+
* Internal signal thrown by wait methods to pause handler execution.
|
|
325
|
+
* This is not a real error -- the processor catches it and transitions the job to 'waiting' status.
|
|
326
|
+
*/
|
|
327
|
+
declare class WaitSignal extends Error {
|
|
328
|
+
readonly type: 'duration' | 'date' | 'token';
|
|
329
|
+
readonly waitUntil: Date | undefined;
|
|
330
|
+
readonly tokenId: string | undefined;
|
|
331
|
+
readonly stepData: Record<string, any>;
|
|
332
|
+
readonly isWaitSignal = true;
|
|
333
|
+
constructor(type: 'duration' | 'date' | 'token', waitUntil: Date | undefined, tokenId: string | undefined, stepData: Record<string, any>);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Status of a waitpoint token.
|
|
337
|
+
*/
|
|
338
|
+
type WaitpointStatus = 'waiting' | 'completed' | 'timed_out';
|
|
339
|
+
/**
|
|
340
|
+
* A waitpoint record from the database.
|
|
341
|
+
*/
|
|
342
|
+
interface WaitpointRecord {
|
|
343
|
+
id: string;
|
|
344
|
+
jobId: number | null;
|
|
345
|
+
status: WaitpointStatus;
|
|
346
|
+
output: any;
|
|
347
|
+
timeoutAt: Date | null;
|
|
348
|
+
createdAt: Date;
|
|
349
|
+
completedAt: Date | null;
|
|
350
|
+
tags: string[] | null;
|
|
351
|
+
}
|
|
352
|
+
type JobHandler<PayloadMap, T extends keyof PayloadMap> = (payload: PayloadMap[T], signal: AbortSignal, ctx: JobContext) => Promise<void>;
|
|
353
|
+
type JobHandlers<PayloadMap> = {
|
|
354
|
+
[K in keyof PayloadMap]: JobHandler<PayloadMap, K>;
|
|
355
|
+
};
|
|
356
|
+
interface ProcessorOptions {
|
|
357
|
+
workerId?: string;
|
|
358
|
+
/**
|
|
359
|
+
* The number of jobs to process at a time.
|
|
360
|
+
* - If not provided, the processor will process 10 jobs at a time.
|
|
361
|
+
* - In serverless functions, it's better to process less jobs at a time since serverless functions are charged by the second and have a timeout.
|
|
362
|
+
*/
|
|
363
|
+
batchSize?: number;
|
|
364
|
+
/**
|
|
365
|
+
* The maximum number of jobs to process in parallel per batch.
|
|
366
|
+
* - If not provided, all jobs in the batch are processed in parallel.
|
|
367
|
+
* - Set to 1 to process jobs sequentially.
|
|
368
|
+
* - Set to a lower value to avoid resource exhaustion.
|
|
369
|
+
*/
|
|
370
|
+
concurrency?: number;
|
|
371
|
+
/**
|
|
372
|
+
* The interval in milliseconds to poll for new jobs.
|
|
373
|
+
* - If not provided, the processor will process jobs every 5 seconds when startInBackground is called.
|
|
374
|
+
* - In serverless functions, it's better to leave this empty.
|
|
375
|
+
* - If you call start instead of startInBackground, the pollInterval is ignored.
|
|
376
|
+
*/
|
|
377
|
+
pollInterval?: number;
|
|
378
|
+
onError?: (error: Error) => void;
|
|
379
|
+
verbose?: boolean;
|
|
380
|
+
/**
|
|
381
|
+
* Only process jobs with this job type (string or array of strings). If omitted, all job types are processed.
|
|
382
|
+
*/
|
|
383
|
+
jobType?: string | string[];
|
|
384
|
+
}
|
|
385
|
+
interface Processor {
|
|
386
|
+
/**
|
|
387
|
+
* Start the job processor in the background.
|
|
388
|
+
* - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs (as many as batchSize) as they become available.
|
|
389
|
+
* - **You have to call the stop method to stop the processor.**
|
|
390
|
+
* - Handlers are provided per-processor when calling createProcessor.
|
|
391
|
+
* - In serverless functions, it's recommended to call start instead and await it to finish.
|
|
392
|
+
*/
|
|
393
|
+
startInBackground: () => void;
|
|
394
|
+
/**
|
|
395
|
+
* Stop the job processor that runs in the background.
|
|
396
|
+
* Does not wait for in-flight jobs to complete.
|
|
397
|
+
*/
|
|
398
|
+
stop: () => void;
|
|
399
|
+
/**
|
|
400
|
+
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
401
|
+
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
402
|
+
* No new batches will be started after calling this method.
|
|
403
|
+
*
|
|
404
|
+
* @param timeoutMs - Maximum time to wait for in-flight jobs (default: 30000ms).
|
|
405
|
+
* If jobs don't complete within this time, the promise resolves anyway.
|
|
406
|
+
*/
|
|
407
|
+
stopAndDrain: (timeoutMs?: number) => Promise<void>;
|
|
408
|
+
/**
|
|
409
|
+
* Check if the job processor is running.
|
|
410
|
+
*/
|
|
411
|
+
isRunning: () => boolean;
|
|
412
|
+
/**
|
|
413
|
+
* Start the job processor synchronously.
|
|
414
|
+
* - This will process jobs (as many as batchSize) immediately and then stop. The pollInterval is ignored.
|
|
415
|
+
* - In serverless functions, it's recommended to use this instead of startInBackground.
|
|
416
|
+
* - Returns the number of jobs processed.
|
|
417
|
+
*/
|
|
418
|
+
start: () => Promise<number>;
|
|
419
|
+
}
|
|
420
|
+
interface DatabaseSSLConfig {
|
|
421
|
+
/**
|
|
422
|
+
* CA certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
|
|
423
|
+
*/
|
|
424
|
+
ca?: string;
|
|
425
|
+
/**
|
|
426
|
+
* Client certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
|
|
427
|
+
*/
|
|
428
|
+
cert?: string;
|
|
429
|
+
/**
|
|
430
|
+
* Client private key as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
|
|
431
|
+
*/
|
|
432
|
+
key?: string;
|
|
433
|
+
/**
|
|
434
|
+
* Whether to reject unauthorized certificates (default: true)
|
|
435
|
+
*/
|
|
436
|
+
rejectUnauthorized?: boolean;
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Configuration for PostgreSQL backend (default).
|
|
440
|
+
* Backward-compatible: omitting `backend` defaults to 'postgres'.
|
|
441
|
+
*/
|
|
442
|
+
interface PostgresJobQueueConfig {
|
|
443
|
+
backend?: 'postgres';
|
|
444
|
+
databaseConfig: {
|
|
445
|
+
connectionString?: string;
|
|
446
|
+
host?: string;
|
|
447
|
+
port?: number;
|
|
448
|
+
database?: string;
|
|
449
|
+
user?: string;
|
|
450
|
+
password?: string;
|
|
451
|
+
ssl?: DatabaseSSLConfig;
|
|
452
|
+
};
|
|
453
|
+
verbose?: boolean;
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* TLS configuration for the Redis connection.
|
|
457
|
+
*/
|
|
458
|
+
interface RedisTLSConfig {
|
|
459
|
+
ca?: string;
|
|
460
|
+
cert?: string;
|
|
461
|
+
key?: string;
|
|
462
|
+
rejectUnauthorized?: boolean;
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Configuration for Redis backend.
|
|
466
|
+
*/
|
|
467
|
+
interface RedisJobQueueConfig {
|
|
468
|
+
backend: 'redis';
|
|
469
|
+
redisConfig: {
|
|
470
|
+
/** Redis URL (e.g. redis://localhost:6379) */
|
|
471
|
+
url?: string;
|
|
472
|
+
host?: string;
|
|
473
|
+
port?: number;
|
|
474
|
+
password?: string;
|
|
475
|
+
/** Redis database number (default: 0) */
|
|
476
|
+
db?: number;
|
|
477
|
+
tls?: RedisTLSConfig;
|
|
478
|
+
/**
|
|
479
|
+
* Key prefix for all Redis keys (default: 'dq:').
|
|
480
|
+
* Useful to namespace multiple queues in the same Redis instance.
|
|
481
|
+
*/
|
|
482
|
+
keyPrefix?: string;
|
|
483
|
+
};
|
|
484
|
+
verbose?: boolean;
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Job queue configuration — discriminated union.
|
|
488
|
+
* If `backend` is omitted, PostgreSQL is used.
|
|
489
|
+
*/
|
|
490
|
+
type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
|
|
491
|
+
/** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
|
|
492
|
+
type JobQueueConfigLegacy = PostgresJobQueueConfig;
|
|
493
|
+
type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
|
|
494
|
+
/**
|
|
495
|
+
* Status of a cron schedule.
|
|
496
|
+
*/
|
|
497
|
+
type CronScheduleStatus = 'active' | 'paused';
|
|
498
|
+
/**
|
|
499
|
+
* Options for creating a recurring cron schedule.
|
|
500
|
+
* Each schedule defines a recurring job that is automatically enqueued
|
|
501
|
+
* when its cron expression matches.
|
|
502
|
+
*/
|
|
503
|
+
interface CronScheduleOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
504
|
+
/** Unique human-readable name for the schedule. */
|
|
505
|
+
scheduleName: string;
|
|
506
|
+
/** Standard cron expression (5 fields, e.g. "0 * * * *"). */
|
|
507
|
+
cronExpression: string;
|
|
508
|
+
/** Job type from the PayloadMap. */
|
|
509
|
+
jobType: T;
|
|
510
|
+
/** Payload for each job instance. */
|
|
511
|
+
payload: PayloadMap[T];
|
|
512
|
+
/** Maximum retry attempts for each job instance (default: 3). */
|
|
513
|
+
maxAttempts?: number;
|
|
514
|
+
/** Priority for each job instance (default: 0). */
|
|
515
|
+
priority?: number;
|
|
516
|
+
/** Timeout in milliseconds for each job instance. */
|
|
517
|
+
timeoutMs?: number;
|
|
518
|
+
/** Whether to force-kill the job on timeout (default: false). */
|
|
519
|
+
forceKillOnTimeout?: boolean;
|
|
520
|
+
/** Tags for each job instance. */
|
|
521
|
+
tags?: string[];
|
|
522
|
+
/** IANA timezone string for cron evaluation (default: "UTC"). */
|
|
523
|
+
timezone?: string;
|
|
524
|
+
/**
|
|
525
|
+
* Whether to allow overlapping job instances (default: false).
|
|
526
|
+
* When false, a new job will not be enqueued if the previous instance
|
|
527
|
+
* is still pending, processing, or waiting.
|
|
528
|
+
*/
|
|
529
|
+
allowOverlap?: boolean;
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* A persisted cron schedule record.
|
|
533
|
+
*/
|
|
534
|
+
interface CronScheduleRecord {
|
|
535
|
+
id: number;
|
|
536
|
+
scheduleName: string;
|
|
537
|
+
cronExpression: string;
|
|
538
|
+
jobType: string;
|
|
539
|
+
payload: any;
|
|
540
|
+
maxAttempts: number;
|
|
541
|
+
priority: number;
|
|
542
|
+
timeoutMs: number | null;
|
|
543
|
+
forceKillOnTimeout: boolean;
|
|
544
|
+
tags: string[] | undefined;
|
|
545
|
+
timezone: string;
|
|
546
|
+
allowOverlap: boolean;
|
|
547
|
+
status: CronScheduleStatus;
|
|
548
|
+
lastEnqueuedAt: Date | null;
|
|
549
|
+
lastJobId: number | null;
|
|
550
|
+
nextRunAt: Date | null;
|
|
551
|
+
createdAt: Date;
|
|
552
|
+
updatedAt: Date;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Options for editing an existing cron schedule.
|
|
556
|
+
* All fields are optional; only provided fields are updated.
|
|
557
|
+
*/
|
|
558
|
+
interface EditCronScheduleOptions {
|
|
559
|
+
cronExpression?: string;
|
|
560
|
+
payload?: any;
|
|
561
|
+
maxAttempts?: number;
|
|
562
|
+
priority?: number;
|
|
563
|
+
timeoutMs?: number | null;
|
|
564
|
+
forceKillOnTimeout?: boolean;
|
|
565
|
+
tags?: string[] | null;
|
|
566
|
+
timezone?: string;
|
|
567
|
+
allowOverlap?: boolean;
|
|
568
|
+
}
|
|
569
|
+
interface JobQueue<PayloadMap> {
|
|
570
|
+
/**
|
|
571
|
+
* Add a job to the job queue.
|
|
572
|
+
*/
|
|
573
|
+
addJob: <T extends JobType<PayloadMap>>(job: JobOptions<PayloadMap, T>) => Promise<number>;
|
|
574
|
+
/**
|
|
575
|
+
* Get a job by its ID.
|
|
576
|
+
*/
|
|
577
|
+
getJob: <T extends JobType<PayloadMap>>(id: number) => Promise<JobRecord<PayloadMap, T> | null>;
|
|
578
|
+
/**
|
|
579
|
+
* Get jobs by their status, with pagination.
|
|
580
|
+
* - If no limit is provided, all jobs are returned.
|
|
581
|
+
* - If no offset is provided, the first page is returned.
|
|
582
|
+
* - The jobs are returned in descending order of createdAt.
|
|
583
|
+
*/
|
|
584
|
+
getJobsByStatus: <T extends JobType<PayloadMap>>(status: JobStatus, limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
585
|
+
/**
|
|
586
|
+
* Get jobs by tag(s).
|
|
587
|
+
* - Modes:
|
|
588
|
+
* - 'exact': Jobs with exactly the same tags (no more, no less)
|
|
589
|
+
* - 'all': Jobs that have all the given tags (can have more)
|
|
590
|
+
* - 'any': Jobs that have at least one of the given tags
|
|
591
|
+
* - 'none': Jobs that have none of the given tags
|
|
592
|
+
* - Default mode is 'all'.
|
|
593
|
+
*/
|
|
594
|
+
getJobsByTags: <T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
595
|
+
/**
|
|
596
|
+
* Get all jobs.
|
|
597
|
+
*/
|
|
598
|
+
getAllJobs: <T extends JobType<PayloadMap>>(limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
599
|
+
/**
|
|
600
|
+
* Get jobs by filters, with pagination support.
|
|
601
|
+
* - Use `cursor` for efficient keyset pagination (recommended for large datasets).
|
|
602
|
+
* - Use `limit` and `offset` for traditional pagination.
|
|
603
|
+
* - Do not combine `cursor` with `offset`.
|
|
604
|
+
*/
|
|
605
|
+
getJobs: <T extends JobType<PayloadMap>>(filters?: {
|
|
606
|
+
jobType?: string;
|
|
607
|
+
priority?: number;
|
|
608
|
+
runAt?: Date | {
|
|
609
|
+
gt?: Date;
|
|
610
|
+
gte?: Date;
|
|
611
|
+
lt?: Date;
|
|
612
|
+
lte?: Date;
|
|
613
|
+
eq?: Date;
|
|
614
|
+
};
|
|
615
|
+
tags?: {
|
|
616
|
+
values: string[];
|
|
617
|
+
mode?: TagQueryMode;
|
|
618
|
+
};
|
|
619
|
+
/** Cursor for keyset pagination. Only return jobs with id < cursor. */
|
|
620
|
+
cursor?: number;
|
|
621
|
+
}, limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
|
|
622
|
+
/**
|
|
623
|
+
* Retry a job given its ID.
|
|
624
|
+
* - This will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
|
|
625
|
+
*/
|
|
626
|
+
retryJob: (jobId: number) => Promise<void>;
|
|
627
|
+
/**
|
|
628
|
+
* Cleanup jobs that are older than the specified number of days.
|
|
629
|
+
*/
|
|
630
|
+
cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
|
|
631
|
+
/**
|
|
632
|
+
* Cleanup job events that are older than the specified number of days.
|
|
633
|
+
*/
|
|
634
|
+
cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
|
|
635
|
+
/**
|
|
636
|
+
* Cancel a job given its ID.
|
|
637
|
+
* - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
|
|
638
|
+
*/
|
|
639
|
+
cancelJob: (jobId: number) => Promise<void>;
|
|
640
|
+
/**
|
|
641
|
+
* Edit a pending job given its ID.
|
|
642
|
+
* - Only works for jobs with status 'pending'. Silently fails for other statuses.
|
|
643
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
644
|
+
* - jobType cannot be changed.
|
|
645
|
+
* - Records an 'edited' event with the updated fields in metadata.
|
|
646
|
+
*/
|
|
647
|
+
editJob: <T extends JobType<PayloadMap>>(jobId: number, updates: EditJobOptions<PayloadMap, T>) => Promise<void>;
|
|
648
|
+
/**
|
|
649
|
+
* Edit all pending jobs that match the filters.
|
|
650
|
+
* - Only works for jobs with status 'pending'. Non-pending jobs are not affected.
|
|
651
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
652
|
+
* - jobType cannot be changed.
|
|
653
|
+
* - Records an 'edited' event with the updated fields in metadata for each affected job.
|
|
654
|
+
* - Returns the number of jobs that were edited.
|
|
655
|
+
* - The filters are:
|
|
656
|
+
* - jobType: The job type to edit.
|
|
657
|
+
* - priority: The priority of the job to edit.
|
|
658
|
+
* - runAt: The time the job is scheduled to run at (now supports gt/gte/lt/lte/eq).
|
|
659
|
+
* - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based editing.
|
|
660
|
+
*/
|
|
661
|
+
editAllPendingJobs: <T extends JobType<PayloadMap>>(filters: {
|
|
662
|
+
jobType?: string;
|
|
663
|
+
priority?: number;
|
|
664
|
+
runAt?: Date | {
|
|
665
|
+
gt?: Date;
|
|
666
|
+
gte?: Date;
|
|
667
|
+
lt?: Date;
|
|
668
|
+
lte?: Date;
|
|
669
|
+
eq?: Date;
|
|
670
|
+
};
|
|
671
|
+
tags?: {
|
|
672
|
+
values: string[];
|
|
673
|
+
mode?: TagQueryMode;
|
|
674
|
+
};
|
|
675
|
+
} | undefined, updates: EditJobOptions<PayloadMap, T>) => Promise<number>;
|
|
676
|
+
/**
|
|
677
|
+
* Reclaim stuck jobs.
|
|
678
|
+
* - If a process (e.g., API route or worker) crashes after marking a job as 'processing' but before completing it, the job can remain stuck in the 'processing' state indefinitely. This can happen if the process is killed or encounters an unhandled error after updating the job status but before marking it as 'completed' or 'failed'.
|
|
679
|
+
* - This function will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
|
|
680
|
+
* - The default max processing time is 10 minutes.
|
|
681
|
+
*/
|
|
682
|
+
reclaimStuckJobs: (maxProcessingTimeMinutes?: number) => Promise<number>;
|
|
683
|
+
/**
|
|
684
|
+
* Cancel all upcoming jobs that match the filters.
|
|
685
|
+
* - If no filters are provided, all upcoming jobs are cancelled.
|
|
686
|
+
* - If filters are provided, only jobs that match the filters are cancelled.
|
|
687
|
+
* - The filters are:
|
|
688
|
+
* - jobType: The job type to cancel.
|
|
689
|
+
* - priority: The priority of the job to cancel.
|
|
690
|
+
* - runAt: The time the job is scheduled to run at (now supports gt/gte/lt/lte/eq).
|
|
691
|
+
* - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based cancellation.
|
|
692
|
+
*/
|
|
693
|
+
cancelAllUpcomingJobs: (filters?: {
|
|
694
|
+
jobType?: string;
|
|
695
|
+
priority?: number;
|
|
696
|
+
runAt?: Date | {
|
|
697
|
+
gt?: Date;
|
|
698
|
+
gte?: Date;
|
|
699
|
+
lt?: Date;
|
|
700
|
+
lte?: Date;
|
|
701
|
+
eq?: Date;
|
|
702
|
+
};
|
|
703
|
+
tags?: {
|
|
704
|
+
values: string[];
|
|
705
|
+
mode?: TagQueryMode;
|
|
706
|
+
};
|
|
707
|
+
}) => Promise<number>;
|
|
708
|
+
/**
|
|
709
|
+
* Create a job processor. Handlers must be provided per-processor.
|
|
710
|
+
*/
|
|
711
|
+
createProcessor: (handlers: JobHandlers<PayloadMap>, options?: ProcessorOptions) => Processor;
|
|
712
|
+
/**
|
|
713
|
+
* Get the job events for a job.
|
|
714
|
+
*/
|
|
715
|
+
getJobEvents: (jobId: number) => Promise<JobEvent[]>;
|
|
716
|
+
/**
|
|
717
|
+
* Create a waitpoint token.
|
|
718
|
+
* Tokens can be completed externally to resume a waiting job.
|
|
719
|
+
* Can be called outside of handlers (e.g., from an API route).
|
|
720
|
+
*
|
|
721
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
722
|
+
*
|
|
723
|
+
* @param options - Optional token configuration (timeout, tags).
|
|
724
|
+
* @returns A token object with `id`.
|
|
725
|
+
*/
|
|
726
|
+
createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
|
|
727
|
+
/**
|
|
728
|
+
* Complete a waitpoint token, resuming the associated waiting job.
|
|
729
|
+
* Can be called from anywhere (API routes, external services, etc.).
|
|
730
|
+
*
|
|
731
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
732
|
+
*
|
|
733
|
+
* @param tokenId - The ID of the token to complete.
|
|
734
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
735
|
+
*/
|
|
736
|
+
completeToken: (tokenId: string, data?: any) => Promise<void>;
|
|
737
|
+
/**
|
|
738
|
+
* Retrieve a waitpoint token by its ID.
|
|
739
|
+
*
|
|
740
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
741
|
+
*
|
|
742
|
+
* @param tokenId - The ID of the token to retrieve.
|
|
743
|
+
* @returns The token record, or null if not found.
|
|
744
|
+
*/
|
|
745
|
+
getToken: (tokenId: string) => Promise<WaitpointRecord | null>;
|
|
746
|
+
/**
|
|
747
|
+
* Expire timed-out waitpoint tokens and resume their associated jobs.
|
|
748
|
+
* Call this periodically (e.g., alongside `reclaimStuckJobs`).
|
|
749
|
+
*
|
|
750
|
+
* **PostgreSQL backend only.** Throws if the backend is Redis.
|
|
751
|
+
*
|
|
752
|
+
* @returns The number of tokens that were expired.
|
|
753
|
+
*/
|
|
754
|
+
expireTimedOutTokens: () => Promise<number>;
|
|
755
|
+
/**
|
|
756
|
+
* Add a recurring cron schedule. The processor automatically enqueues
|
|
757
|
+
* due cron jobs before each batch, so no manual triggering is needed.
|
|
758
|
+
*
|
|
759
|
+
* @returns The ID of the created schedule.
|
|
760
|
+
* @throws If the cron expression is invalid or the schedule name is already taken.
|
|
761
|
+
*/
|
|
762
|
+
addCronJob: <T extends JobType<PayloadMap>>(options: CronScheduleOptions<PayloadMap, T>) => Promise<number>;
|
|
763
|
+
/**
|
|
764
|
+
* Get a cron schedule by its ID.
|
|
765
|
+
*/
|
|
766
|
+
getCronJob: (id: number) => Promise<CronScheduleRecord | null>;
|
|
767
|
+
/**
|
|
768
|
+
* Get a cron schedule by its unique name.
|
|
769
|
+
*/
|
|
770
|
+
getCronJobByName: (name: string) => Promise<CronScheduleRecord | null>;
|
|
771
|
+
/**
|
|
772
|
+
* List all cron schedules, optionally filtered by status.
|
|
773
|
+
*/
|
|
774
|
+
listCronJobs: (status?: CronScheduleStatus) => Promise<CronScheduleRecord[]>;
|
|
775
|
+
/**
|
|
776
|
+
* Remove a cron schedule by its ID. Does not cancel any already-enqueued jobs.
|
|
777
|
+
*/
|
|
778
|
+
removeCronJob: (id: number) => Promise<void>;
|
|
779
|
+
/**
|
|
780
|
+
* Pause a cron schedule. Paused schedules are skipped by `enqueueDueCronJobs()`.
|
|
781
|
+
*/
|
|
782
|
+
pauseCronJob: (id: number) => Promise<void>;
|
|
783
|
+
/**
|
|
784
|
+
* Resume a paused cron schedule.
|
|
785
|
+
*/
|
|
786
|
+
resumeCronJob: (id: number) => Promise<void>;
|
|
787
|
+
/**
|
|
788
|
+
* Edit an existing cron schedule. Only provided fields are updated.
|
|
789
|
+
* If `cronExpression` or `timezone` changes, `nextRunAt` is recalculated.
|
|
790
|
+
*/
|
|
791
|
+
editCronJob: (id: number, updates: EditCronScheduleOptions) => Promise<void>;
|
|
792
|
+
/**
|
|
793
|
+
* Check all active cron schedules and enqueue jobs for any whose
|
|
794
|
+
* `nextRunAt` has passed. When `allowOverlap` is false (the default),
|
|
795
|
+
* a new job is not enqueued if the previous instance is still
|
|
796
|
+
* pending, processing, or waiting.
|
|
797
|
+
*
|
|
798
|
+
* **Note:** The processor calls this automatically before each batch,
|
|
799
|
+
* so you typically do not need to call it yourself. It is exposed for
|
|
800
|
+
* manual use in tests or one-off scripts.
|
|
801
|
+
*
|
|
802
|
+
* @returns The number of jobs that were enqueued.
|
|
803
|
+
*/
|
|
804
|
+
enqueueDueCronJobs: () => Promise<number>;
|
|
805
|
+
/**
|
|
806
|
+
* Get the PostgreSQL database pool.
|
|
807
|
+
* Throws if the backend is not PostgreSQL.
|
|
808
|
+
*/
|
|
809
|
+
getPool: () => pg.Pool;
|
|
810
|
+
/**
|
|
811
|
+
* Get the Redis client instance (ioredis).
|
|
812
|
+
* Throws if the backend is not Redis.
|
|
813
|
+
*/
|
|
814
|
+
getRedisClient: () => unknown;
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* Filter options used by getJobs, cancelAllUpcomingJobs, editAllPendingJobs
|
|
819
|
+
*/
|
|
820
|
+
interface JobFilters {
|
|
821
|
+
jobType?: string;
|
|
822
|
+
priority?: number;
|
|
823
|
+
runAt?: Date | {
|
|
824
|
+
gt?: Date;
|
|
825
|
+
gte?: Date;
|
|
826
|
+
lt?: Date;
|
|
827
|
+
lte?: Date;
|
|
828
|
+
eq?: Date;
|
|
829
|
+
};
|
|
830
|
+
tags?: {
|
|
831
|
+
values: string[];
|
|
832
|
+
mode?: TagQueryMode;
|
|
833
|
+
};
|
|
834
|
+
/**
|
|
835
|
+
* Cursor for keyset pagination. When provided, only return jobs with id < cursor.
|
|
836
|
+
* This is more efficient than OFFSET for large datasets.
|
|
837
|
+
* Cannot be used together with offset.
|
|
838
|
+
*/
|
|
839
|
+
cursor?: number;
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* Fields that can be updated on a job
|
|
843
|
+
*/
|
|
844
|
+
interface JobUpdates {
|
|
845
|
+
payload?: any;
|
|
846
|
+
maxAttempts?: number;
|
|
847
|
+
priority?: number;
|
|
848
|
+
runAt?: Date | null;
|
|
849
|
+
timeoutMs?: number | null;
|
|
850
|
+
tags?: string[] | null;
|
|
851
|
+
}
|
|
852
|
+
/**
|
|
853
|
+
* Input shape for creating a cron schedule in the backend.
|
|
854
|
+
* This is the backend-level version of CronScheduleOptions.
|
|
855
|
+
*/
|
|
856
|
+
interface CronScheduleInput {
|
|
857
|
+
scheduleName: string;
|
|
858
|
+
cronExpression: string;
|
|
859
|
+
jobType: string;
|
|
860
|
+
payload: any;
|
|
861
|
+
maxAttempts: number;
|
|
862
|
+
priority: number;
|
|
863
|
+
timeoutMs: number | null;
|
|
864
|
+
forceKillOnTimeout: boolean;
|
|
865
|
+
tags: string[] | undefined;
|
|
866
|
+
timezone: string;
|
|
867
|
+
allowOverlap: boolean;
|
|
868
|
+
nextRunAt: Date | null;
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Abstract backend interface that both PostgreSQL and Redis implement.
|
|
872
|
+
* All storage operations go through this interface so the processor
|
|
873
|
+
* and public API are backend-agnostic.
|
|
874
|
+
*/
|
|
875
|
+
interface QueueBackend {
|
|
876
|
+
/** Add a job and return its numeric ID. */
|
|
877
|
+
addJob<PayloadMap, T extends JobType<PayloadMap>>(job: JobOptions<PayloadMap, T>): Promise<number>;
|
|
878
|
+
/** Get a single job by ID, or null if not found. */
|
|
879
|
+
getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
|
|
880
|
+
/** Get jobs filtered by status, ordered by createdAt DESC. */
|
|
881
|
+
getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
882
|
+
/** Get all jobs, ordered by createdAt DESC. */
|
|
883
|
+
getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
884
|
+
/** Get jobs matching arbitrary filters, ordered by createdAt DESC. */
|
|
885
|
+
getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
886
|
+
/** Get jobs by tag(s) with query mode. */
|
|
887
|
+
getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
888
|
+
/**
|
|
889
|
+
* Atomically claim a batch of ready jobs for the given worker.
|
|
890
|
+
* Equivalent to SELECT … FOR UPDATE SKIP LOCKED in Postgres.
|
|
891
|
+
*/
|
|
892
|
+
getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
|
|
893
|
+
/** Mark a job as completed. */
|
|
894
|
+
completeJob(jobId: number): Promise<void>;
|
|
895
|
+
/** Mark a job as failed with error info and schedule retry. */
|
|
896
|
+
failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
|
|
897
|
+
/** Update locked_at to keep the job alive (heartbeat). */
|
|
898
|
+
prolongJob(jobId: number): Promise<void>;
|
|
899
|
+
/** Retry a failed/cancelled job immediately. */
|
|
900
|
+
retryJob(jobId: number): Promise<void>;
|
|
901
|
+
/** Cancel a pending job. */
|
|
902
|
+
cancelJob(jobId: number): Promise<void>;
|
|
903
|
+
/** Cancel all pending jobs matching optional filters. Returns count. */
|
|
904
|
+
cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
|
|
905
|
+
/** Edit a single pending job. */
|
|
906
|
+
editJob(jobId: number, updates: JobUpdates): Promise<void>;
|
|
907
|
+
/** Edit all pending jobs matching filters. Returns count. */
|
|
908
|
+
editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
|
|
909
|
+
/** Delete completed jobs older than N days. Returns count deleted. */
|
|
910
|
+
cleanupOldJobs(daysToKeep?: number): Promise<number>;
|
|
911
|
+
/** Delete job events older than N days. Returns count deleted. */
|
|
912
|
+
cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
|
|
913
|
+
/** Reclaim jobs stuck in 'processing' for too long. Returns count. */
|
|
914
|
+
reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
|
|
915
|
+
/** Update the progress percentage (0-100) for a job. */
|
|
916
|
+
updateProgress(jobId: number, progress: number): Promise<void>;
|
|
917
|
+
/** Record a job event. Should not throw. */
|
|
918
|
+
recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
|
|
919
|
+
/** Get all events for a job, ordered by createdAt ASC. */
|
|
920
|
+
getJobEvents(jobId: number): Promise<JobEvent[]>;
|
|
921
|
+
/** Create a cron schedule and return its ID. */
|
|
922
|
+
addCronSchedule(input: CronScheduleInput): Promise<number>;
|
|
923
|
+
/** Get a cron schedule by ID, or null if not found. */
|
|
924
|
+
getCronSchedule(id: number): Promise<CronScheduleRecord | null>;
|
|
925
|
+
/** Get a cron schedule by its unique name, or null if not found. */
|
|
926
|
+
getCronScheduleByName(name: string): Promise<CronScheduleRecord | null>;
|
|
927
|
+
/** List cron schedules, optionally filtered by status. */
|
|
928
|
+
listCronSchedules(status?: CronScheduleStatus): Promise<CronScheduleRecord[]>;
|
|
929
|
+
/** Delete a cron schedule by ID. */
|
|
930
|
+
removeCronSchedule(id: number): Promise<void>;
|
|
931
|
+
/** Pause a cron schedule. */
|
|
932
|
+
pauseCronSchedule(id: number): Promise<void>;
|
|
933
|
+
/** Resume a cron schedule. */
|
|
934
|
+
resumeCronSchedule(id: number): Promise<void>;
|
|
935
|
+
/** Edit a cron schedule. */
|
|
936
|
+
editCronSchedule(id: number, updates: EditCronScheduleOptions, nextRunAt?: Date | null): Promise<void>;
|
|
937
|
+
/**
|
|
938
|
+
* Atomically fetch all active cron schedules whose nextRunAt <= now.
|
|
939
|
+
* In PostgreSQL this uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing.
|
|
940
|
+
*/
|
|
941
|
+
getDueCronSchedules(): Promise<CronScheduleRecord[]>;
|
|
942
|
+
/**
|
|
943
|
+
* Update a cron schedule after a job has been enqueued.
|
|
944
|
+
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
945
|
+
*/
|
|
946
|
+
updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
|
|
947
|
+
/** Set a pending reason for unpicked jobs of a given type. */
|
|
948
|
+
setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
declare class PostgresBackend implements QueueBackend {
|
|
952
|
+
private pool;
|
|
953
|
+
constructor(pool: Pool);
|
|
954
|
+
/** Expose the raw pool for advanced usage. */
|
|
955
|
+
getPool(): Pool;
|
|
956
|
+
recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
|
|
957
|
+
getJobEvents(jobId: number): Promise<JobEvent[]>;
|
|
958
|
+
addJob<PayloadMap, T extends JobType<PayloadMap>>({ jobType, payload, maxAttempts, priority, runAt, timeoutMs, forceKillOnTimeout, tags, idempotencyKey, }: JobOptions<PayloadMap, T>): Promise<number>;
|
|
959
|
+
getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
|
|
960
|
+
getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
961
|
+
getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
962
|
+
getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
963
|
+
getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
|
|
964
|
+
getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
|
|
965
|
+
completeJob(jobId: number): Promise<void>;
|
|
966
|
+
failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
|
|
967
|
+
prolongJob(jobId: number): Promise<void>;
|
|
968
|
+
updateProgress(jobId: number, progress: number): Promise<void>;
|
|
969
|
+
retryJob(jobId: number): Promise<void>;
|
|
970
|
+
cancelJob(jobId: number): Promise<void>;
|
|
971
|
+
cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
|
|
972
|
+
editJob(jobId: number, updates: JobUpdates): Promise<void>;
|
|
973
|
+
editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
|
|
974
|
+
cleanupOldJobs(daysToKeep?: number): Promise<number>;
|
|
975
|
+
cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
|
|
976
|
+
reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
|
|
977
|
+
/**
|
|
978
|
+
* Batch-insert multiple job events in a single query.
|
|
979
|
+
* More efficient than individual recordJobEvent calls.
|
|
980
|
+
*/
|
|
981
|
+
private recordJobEventsBatch;
|
|
982
|
+
/** Create a cron schedule and return its ID. */
|
|
983
|
+
addCronSchedule(input: CronScheduleInput): Promise<number>;
|
|
984
|
+
/** Get a cron schedule by ID. */
|
|
985
|
+
getCronSchedule(id: number): Promise<CronScheduleRecord | null>;
|
|
986
|
+
/** Get a cron schedule by its unique name. */
|
|
987
|
+
getCronScheduleByName(name: string): Promise<CronScheduleRecord | null>;
|
|
988
|
+
/** List cron schedules, optionally filtered by status. */
|
|
989
|
+
listCronSchedules(status?: CronScheduleStatus): Promise<CronScheduleRecord[]>;
|
|
990
|
+
/** Delete a cron schedule by ID. */
|
|
991
|
+
removeCronSchedule(id: number): Promise<void>;
|
|
992
|
+
/** Pause a cron schedule. */
|
|
993
|
+
pauseCronSchedule(id: number): Promise<void>;
|
|
994
|
+
/** Resume a paused cron schedule. */
|
|
995
|
+
resumeCronSchedule(id: number): Promise<void>;
|
|
996
|
+
/** Edit a cron schedule. */
|
|
997
|
+
editCronSchedule(id: number, updates: EditCronScheduleOptions, nextRunAt?: Date | null): Promise<void>;
|
|
998
|
+
/**
|
|
999
|
+
* Atomically fetch all active cron schedules whose nextRunAt <= NOW().
|
|
1000
|
+
* Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
|
|
1001
|
+
*/
|
|
1002
|
+
getDueCronSchedules(): Promise<CronScheduleRecord[]>;
|
|
1003
|
+
/**
|
|
1004
|
+
* Update a cron schedule after a job has been enqueued.
|
|
1005
|
+
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
1006
|
+
*/
|
|
1007
|
+
updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
|
|
1008
|
+
setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
/**
|
|
1012
|
+
* Validates that a job handler can be serialized for use with forceKillOnTimeout.
|
|
1013
|
+
*
|
|
1014
|
+
* This function checks if a handler can be safely serialized and executed in a worker thread.
|
|
1015
|
+
* Use this function during development to catch serialization issues early.
|
|
1016
|
+
*
|
|
1017
|
+
* @param handler - The job handler function to validate
|
|
1018
|
+
* @param jobType - Optional job type name for better error messages
|
|
1019
|
+
* @returns An object with `isSerializable` boolean and optional `error` message
|
|
1020
|
+
*
|
|
1021
|
+
* @example
|
|
1022
|
+
* ```ts
|
|
1023
|
+
* const handler = async (payload, signal) => {
|
|
1024
|
+
* await doSomething(payload);
|
|
1025
|
+
* };
|
|
1026
|
+
*
|
|
1027
|
+
* const result = validateHandlerSerializable(handler, 'myJob');
|
|
1028
|
+
* if (!result.isSerializable) {
|
|
1029
|
+
* console.error('Handler is not serializable:', result.error);
|
|
1030
|
+
* }
|
|
1031
|
+
* ```
|
|
1032
|
+
*/
|
|
1033
|
+
declare function validateHandlerSerializable<PayloadMap, T extends keyof PayloadMap & string>(handler: JobHandler<PayloadMap, T>, jobType?: string): {
|
|
1034
|
+
isSerializable: boolean;
|
|
1035
|
+
error?: string;
|
|
1036
|
+
};
|
|
1037
|
+
/**
|
|
1038
|
+
* Test if a handler can be serialized and executed in a worker thread.
|
|
1039
|
+
* This is a more thorough check that actually attempts to serialize and deserialize the handler.
|
|
1040
|
+
*
|
|
1041
|
+
* @param handler - The job handler function to test
|
|
1042
|
+
* @param jobType - Optional job type name for better error messages
|
|
1043
|
+
* @returns Promise that resolves to validation result
|
|
1044
|
+
*
|
|
1045
|
+
* @example
|
|
1046
|
+
* ```ts
|
|
1047
|
+
* const handler = async (payload, signal) => {
|
|
1048
|
+
* await doSomething(payload);
|
|
1049
|
+
* };
|
|
1050
|
+
*
|
|
1051
|
+
* const result = await testHandlerSerialization(handler, 'myJob');
|
|
1052
|
+
* if (!result.isSerializable) {
|
|
1053
|
+
* console.error('Handler failed serialization test:', result.error);
|
|
1054
|
+
* }
|
|
1055
|
+
* ```
|
|
1056
|
+
*/
|
|
1057
|
+
declare function testHandlerSerialization<PayloadMap, T extends keyof PayloadMap & string>(handler: JobHandler<PayloadMap, T>, jobType?: string): Promise<{
|
|
1058
|
+
isSerializable: boolean;
|
|
1059
|
+
error?: string;
|
|
1060
|
+
}>;
|
|
1061
|
+
|
|
1062
|
+
/**
|
|
1063
|
+
* Calculate the next occurrence of a cron expression after a given date.
|
|
1064
|
+
*
|
|
1065
|
+
* @param cronExpression - A standard cron expression (5 fields, e.g. "0 * * * *").
|
|
1066
|
+
* @param timezone - IANA timezone string (default: "UTC").
|
|
1067
|
+
* @param after - The reference date to compute the next run from (default: now).
|
|
1068
|
+
* @param CronImpl - Cron class for dependency injection (default: croner's Cron).
|
|
1069
|
+
* @returns The next occurrence as a Date, or null if the expression will never fire again.
|
|
1070
|
+
*/
|
|
1071
|
+
declare function getNextCronOccurrence(cronExpression: string, timezone?: string, after?: Date, CronImpl?: typeof Cron): Date | null;
|
|
1072
|
+
/**
|
|
1073
|
+
* Validate whether a string is a syntactically correct cron expression.
|
|
1074
|
+
*
|
|
1075
|
+
* @param cronExpression - The cron expression to validate.
|
|
1076
|
+
* @param CronImpl - Cron class for dependency injection (default: croner's Cron).
|
|
1077
|
+
* @returns True if the expression is valid, false otherwise.
|
|
1078
|
+
*/
|
|
1079
|
+
declare function validateCronExpression(cronExpression: string, CronImpl?: typeof Cron): boolean;
|
|
1080
|
+
|
|
1081
|
+
/**
|
|
1082
|
+
* Initialize the job queue system.
|
|
1083
|
+
*
|
|
1084
|
+
* Defaults to PostgreSQL when `backend` is omitted.
|
|
1085
|
+
*/
|
|
1086
|
+
declare const initJobQueue: <PayloadMap = any>(config: JobQueueConfig) => JobQueue<PayloadMap>;
|
|
1087
|
+
|
|
1088
|
+
export { type CreateTokenOptions, type CronScheduleInput, type CronScheduleOptions, type CronScheduleRecord, type CronScheduleStatus, type DatabaseSSLConfig, type EditCronScheduleOptions, type EditJobOptions, FailureReason, type JobContext, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobQueueConfigLegacy, type JobRecord, type JobStatus, type JobType, type OnTimeoutCallback, PostgresBackend, type PostgresJobQueueConfig, type Processor, type ProcessorOptions, type QueueBackend, type RedisJobQueueConfig, type RedisTLSConfig, type TagQueryMode, type WaitDuration, WaitSignal, type WaitToken, type WaitTokenResult, type WaitpointRecord, type WaitpointStatus, getNextCronOccurrence, initJobQueue, testHandlerSerialization, validateCronExpression, validateHandlerSerializable };
|