@nicnocquee/dataqueue 1.22.0 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +486 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +151 -2
- package/dist/index.d.ts +151 -2
- package/dist/index.js +485 -30
- package/dist/index.js.map +1 -1
- package/migrations/1765809419000_add_force_kill_on_timeout_to_job_queue.sql +6 -0
- package/package.json +1 -1
- package/src/handler-validation.test.ts +414 -0
- package/src/handler-validation.ts +168 -0
- package/src/index.test.ts +224 -0
- package/src/index.ts +33 -0
- package/src/processor.test.ts +55 -0
- package/src/processor.ts +261 -17
- package/src/queue.test.ts +522 -0
- package/src/queue.ts +286 -9
- package/src/types.ts +102 -0
package/src/queue.ts
CHANGED
|
@@ -44,6 +44,7 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
44
44
|
priority = 0,
|
|
45
45
|
runAt = null,
|
|
46
46
|
timeoutMs = undefined,
|
|
47
|
+
forceKillOnTimeout = false,
|
|
47
48
|
tags = undefined,
|
|
48
49
|
}: JobOptions<PayloadMap, T>,
|
|
49
50
|
): Promise<number> => {
|
|
@@ -53,8 +54,8 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
53
54
|
if (runAt) {
|
|
54
55
|
result = await client.query(
|
|
55
56
|
`INSERT INTO job_queue
|
|
56
|
-
(job_type, payload, max_attempts, priority, run_at, timeout_ms, tags)
|
|
57
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
57
|
+
(job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags)
|
|
58
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
58
59
|
RETURNING id`,
|
|
59
60
|
[
|
|
60
61
|
jobType,
|
|
@@ -63,6 +64,7 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
63
64
|
priority,
|
|
64
65
|
runAt,
|
|
65
66
|
timeoutMs ?? null,
|
|
67
|
+
forceKillOnTimeout ?? false,
|
|
66
68
|
tags ?? null,
|
|
67
69
|
],
|
|
68
70
|
);
|
|
@@ -72,8 +74,8 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
72
74
|
} else {
|
|
73
75
|
result = await client.query(
|
|
74
76
|
`INSERT INTO job_queue
|
|
75
|
-
(job_type, payload, max_attempts, priority, timeout_ms, tags)
|
|
76
|
-
VALUES ($1, $2, $3, $4, $5, $6)
|
|
77
|
+
(job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags)
|
|
78
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
77
79
|
RETURNING id`,
|
|
78
80
|
[
|
|
79
81
|
jobType,
|
|
@@ -81,6 +83,7 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
81
83
|
maxAttempts,
|
|
82
84
|
priority,
|
|
83
85
|
timeoutMs ?? null,
|
|
86
|
+
forceKillOnTimeout ?? false,
|
|
84
87
|
tags ?? null,
|
|
85
88
|
],
|
|
86
89
|
);
|
|
@@ -112,7 +115,7 @@ export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
112
115
|
const client = await pool.connect();
|
|
113
116
|
try {
|
|
114
117
|
const result = await client.query(
|
|
115
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue WHERE id = $1`,
|
|
118
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags FROM job_queue WHERE id = $1`,
|
|
116
119
|
[id],
|
|
117
120
|
);
|
|
118
121
|
|
|
@@ -129,6 +132,7 @@ export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
129
132
|
...job,
|
|
130
133
|
payload: job.payload,
|
|
131
134
|
timeoutMs: job.timeoutMs,
|
|
135
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
132
136
|
failureReason: job.failureReason,
|
|
133
137
|
};
|
|
134
138
|
} catch (error) {
|
|
@@ -154,7 +158,7 @@ export const getJobsByStatus = async <
|
|
|
154
158
|
const client = await pool.connect();
|
|
155
159
|
try {
|
|
156
160
|
const result = await client.query(
|
|
157
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
|
|
161
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
|
|
158
162
|
[status, limit, offset],
|
|
159
163
|
);
|
|
160
164
|
|
|
@@ -164,6 +168,7 @@ export const getJobsByStatus = async <
|
|
|
164
168
|
...job,
|
|
165
169
|
payload: job.payload,
|
|
166
170
|
timeoutMs: job.timeoutMs,
|
|
171
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
167
172
|
failureReason: job.failureReason,
|
|
168
173
|
}));
|
|
169
174
|
} catch (error) {
|
|
@@ -230,7 +235,7 @@ export const getNextBatch = async <
|
|
|
230
235
|
LIMIT $2
|
|
231
236
|
FOR UPDATE SKIP LOCKED
|
|
232
237
|
)
|
|
233
|
-
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason"
|
|
238
|
+
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason"
|
|
234
239
|
`,
|
|
235
240
|
params,
|
|
236
241
|
);
|
|
@@ -249,6 +254,7 @@ export const getNextBatch = async <
|
|
|
249
254
|
...job,
|
|
250
255
|
payload: job.payload,
|
|
251
256
|
timeoutMs: job.timeoutMs,
|
|
257
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
252
258
|
}));
|
|
253
259
|
} catch (error) {
|
|
254
260
|
log(`Error getting next batch: ${error}`);
|
|
@@ -413,6 +419,274 @@ export const cancelJob = async (pool: Pool, jobId: number): Promise<void> => {
|
|
|
413
419
|
}
|
|
414
420
|
};
|
|
415
421
|
|
|
422
|
+
/**
|
|
423
|
+
* Edit a pending job (only if still pending)
|
|
424
|
+
*/
|
|
425
|
+
export const editJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
426
|
+
pool: Pool,
|
|
427
|
+
jobId: number,
|
|
428
|
+
updates: {
|
|
429
|
+
payload?: PayloadMap[T];
|
|
430
|
+
maxAttempts?: number;
|
|
431
|
+
priority?: number;
|
|
432
|
+
runAt?: Date | null;
|
|
433
|
+
timeoutMs?: number | null;
|
|
434
|
+
tags?: string[] | null;
|
|
435
|
+
},
|
|
436
|
+
): Promise<void> => {
|
|
437
|
+
const client = await pool.connect();
|
|
438
|
+
try {
|
|
439
|
+
const updateFields: string[] = [];
|
|
440
|
+
const params: any[] = [];
|
|
441
|
+
let paramIdx = 1;
|
|
442
|
+
|
|
443
|
+
// Build dynamic UPDATE query based on provided fields
|
|
444
|
+
if (updates.payload !== undefined) {
|
|
445
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
446
|
+
params.push(updates.payload);
|
|
447
|
+
}
|
|
448
|
+
if (updates.maxAttempts !== undefined) {
|
|
449
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
450
|
+
params.push(updates.maxAttempts);
|
|
451
|
+
}
|
|
452
|
+
if (updates.priority !== undefined) {
|
|
453
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
454
|
+
params.push(updates.priority);
|
|
455
|
+
}
|
|
456
|
+
if (updates.runAt !== undefined) {
|
|
457
|
+
if (updates.runAt === null) {
|
|
458
|
+
// null means run now (use current timestamp)
|
|
459
|
+
updateFields.push(`run_at = NOW()`);
|
|
460
|
+
} else {
|
|
461
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
462
|
+
params.push(updates.runAt);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
if (updates.timeoutMs !== undefined) {
|
|
466
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
467
|
+
params.push(updates.timeoutMs ?? null);
|
|
468
|
+
}
|
|
469
|
+
if (updates.tags !== undefined) {
|
|
470
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
471
|
+
params.push(updates.tags ?? null);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// If no fields to update, return early
|
|
475
|
+
if (updateFields.length === 0) {
|
|
476
|
+
log(`No fields to update for job ${jobId}`);
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// Always update updated_at timestamp
|
|
481
|
+
updateFields.push(`updated_at = NOW()`);
|
|
482
|
+
|
|
483
|
+
// Add jobId as the last parameter for WHERE clause
|
|
484
|
+
params.push(jobId);
|
|
485
|
+
|
|
486
|
+
const query = `
|
|
487
|
+
UPDATE job_queue
|
|
488
|
+
SET ${updateFields.join(', ')}
|
|
489
|
+
WHERE id = $${paramIdx} AND status = 'pending'
|
|
490
|
+
`;
|
|
491
|
+
|
|
492
|
+
await client.query(query, params);
|
|
493
|
+
|
|
494
|
+
// Record edit event with metadata containing updated fields
|
|
495
|
+
const metadata: any = {};
|
|
496
|
+
if (updates.payload !== undefined) metadata.payload = updates.payload;
|
|
497
|
+
if (updates.maxAttempts !== undefined)
|
|
498
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
499
|
+
if (updates.priority !== undefined) metadata.priority = updates.priority;
|
|
500
|
+
if (updates.runAt !== undefined) metadata.runAt = updates.runAt;
|
|
501
|
+
if (updates.timeoutMs !== undefined) metadata.timeoutMs = updates.timeoutMs;
|
|
502
|
+
if (updates.tags !== undefined) metadata.tags = updates.tags;
|
|
503
|
+
|
|
504
|
+
await recordJobEvent(pool, jobId, JobEventType.Edited, metadata);
|
|
505
|
+
log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
|
|
506
|
+
} catch (error) {
|
|
507
|
+
log(`Error editing job ${jobId}: ${error}`);
|
|
508
|
+
throw error;
|
|
509
|
+
} finally {
|
|
510
|
+
client.release();
|
|
511
|
+
}
|
|
512
|
+
};
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Edit all pending jobs matching the filters
|
|
516
|
+
*/
|
|
517
|
+
export const editAllPendingJobs = async <
|
|
518
|
+
PayloadMap,
|
|
519
|
+
T extends keyof PayloadMap & string,
|
|
520
|
+
>(
|
|
521
|
+
pool: Pool,
|
|
522
|
+
filters:
|
|
523
|
+
| {
|
|
524
|
+
jobType?: string;
|
|
525
|
+
priority?: number;
|
|
526
|
+
runAt?:
|
|
527
|
+
| Date
|
|
528
|
+
| { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
|
|
529
|
+
tags?: { values: string[]; mode?: TagQueryMode };
|
|
530
|
+
}
|
|
531
|
+
| undefined = undefined,
|
|
532
|
+
updates: {
|
|
533
|
+
payload?: PayloadMap[T];
|
|
534
|
+
maxAttempts?: number;
|
|
535
|
+
priority?: number;
|
|
536
|
+
runAt?: Date | null;
|
|
537
|
+
timeoutMs?: number;
|
|
538
|
+
tags?: string[];
|
|
539
|
+
},
|
|
540
|
+
): Promise<number> => {
|
|
541
|
+
const client = await pool.connect();
|
|
542
|
+
try {
|
|
543
|
+
// Build SET clause from updates
|
|
544
|
+
const updateFields: string[] = [];
|
|
545
|
+
const params: any[] = [];
|
|
546
|
+
let paramIdx = 1;
|
|
547
|
+
|
|
548
|
+
if (updates.payload !== undefined) {
|
|
549
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
550
|
+
params.push(updates.payload);
|
|
551
|
+
}
|
|
552
|
+
if (updates.maxAttempts !== undefined) {
|
|
553
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
554
|
+
params.push(updates.maxAttempts);
|
|
555
|
+
}
|
|
556
|
+
if (updates.priority !== undefined) {
|
|
557
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
558
|
+
params.push(updates.priority);
|
|
559
|
+
}
|
|
560
|
+
if (updates.runAt !== undefined) {
|
|
561
|
+
if (updates.runAt === null) {
|
|
562
|
+
// null means run now (use current timestamp)
|
|
563
|
+
updateFields.push(`run_at = NOW()`);
|
|
564
|
+
} else {
|
|
565
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
566
|
+
params.push(updates.runAt);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
if (updates.timeoutMs !== undefined) {
|
|
570
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
571
|
+
params.push(updates.timeoutMs ?? null);
|
|
572
|
+
}
|
|
573
|
+
if (updates.tags !== undefined) {
|
|
574
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
575
|
+
params.push(updates.tags ?? null);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// If no fields to update, return early
|
|
579
|
+
if (updateFields.length === 0) {
|
|
580
|
+
log(`No fields to update for batch edit`);
|
|
581
|
+
return 0;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// Always update updated_at timestamp
|
|
585
|
+
updateFields.push(`updated_at = NOW()`);
|
|
586
|
+
|
|
587
|
+
// Build WHERE clause from filters
|
|
588
|
+
let query = `
|
|
589
|
+
UPDATE job_queue
|
|
590
|
+
SET ${updateFields.join(', ')}
|
|
591
|
+
WHERE status = 'pending'`;
|
|
592
|
+
|
|
593
|
+
if (filters) {
|
|
594
|
+
if (filters.jobType) {
|
|
595
|
+
query += ` AND job_type = $${paramIdx++}`;
|
|
596
|
+
params.push(filters.jobType);
|
|
597
|
+
}
|
|
598
|
+
if (filters.priority !== undefined) {
|
|
599
|
+
query += ` AND priority = $${paramIdx++}`;
|
|
600
|
+
params.push(filters.priority);
|
|
601
|
+
}
|
|
602
|
+
if (filters.runAt) {
|
|
603
|
+
if (filters.runAt instanceof Date) {
|
|
604
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
605
|
+
params.push(filters.runAt);
|
|
606
|
+
} else if (typeof filters.runAt === 'object') {
|
|
607
|
+
const ops = filters.runAt;
|
|
608
|
+
if (ops.gt) {
|
|
609
|
+
query += ` AND run_at > $${paramIdx++}`;
|
|
610
|
+
params.push(ops.gt);
|
|
611
|
+
}
|
|
612
|
+
if (ops.gte) {
|
|
613
|
+
query += ` AND run_at >= $${paramIdx++}`;
|
|
614
|
+
params.push(ops.gte);
|
|
615
|
+
}
|
|
616
|
+
if (ops.lt) {
|
|
617
|
+
query += ` AND run_at < $${paramIdx++}`;
|
|
618
|
+
params.push(ops.lt);
|
|
619
|
+
}
|
|
620
|
+
if (ops.lte) {
|
|
621
|
+
query += ` AND run_at <= $${paramIdx++}`;
|
|
622
|
+
params.push(ops.lte);
|
|
623
|
+
}
|
|
624
|
+
if (ops.eq) {
|
|
625
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
626
|
+
params.push(ops.eq);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
if (
|
|
631
|
+
filters.tags &&
|
|
632
|
+
filters.tags.values &&
|
|
633
|
+
filters.tags.values.length > 0
|
|
634
|
+
) {
|
|
635
|
+
const mode = filters.tags.mode || 'all';
|
|
636
|
+
const tagValues = filters.tags.values;
|
|
637
|
+
switch (mode) {
|
|
638
|
+
case 'exact':
|
|
639
|
+
query += ` AND tags = $${paramIdx++}`;
|
|
640
|
+
params.push(tagValues);
|
|
641
|
+
break;
|
|
642
|
+
case 'all':
|
|
643
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
644
|
+
params.push(tagValues);
|
|
645
|
+
break;
|
|
646
|
+
case 'any':
|
|
647
|
+
query += ` AND tags && $${paramIdx++}`;
|
|
648
|
+
params.push(tagValues);
|
|
649
|
+
break;
|
|
650
|
+
case 'none':
|
|
651
|
+
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
652
|
+
params.push(tagValues);
|
|
653
|
+
break;
|
|
654
|
+
default:
|
|
655
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
656
|
+
params.push(tagValues);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
query += '\nRETURNING id';
|
|
661
|
+
|
|
662
|
+
const result = await client.query(query, params);
|
|
663
|
+
const editedCount = result.rowCount || 0;
|
|
664
|
+
|
|
665
|
+
// Record edit event with metadata containing updated fields for each job
|
|
666
|
+
const metadata: any = {};
|
|
667
|
+
if (updates.payload !== undefined) metadata.payload = updates.payload;
|
|
668
|
+
if (updates.maxAttempts !== undefined)
|
|
669
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
670
|
+
if (updates.priority !== undefined) metadata.priority = updates.priority;
|
|
671
|
+
if (updates.runAt !== undefined) metadata.runAt = updates.runAt;
|
|
672
|
+
if (updates.timeoutMs !== undefined) metadata.timeoutMs = updates.timeoutMs;
|
|
673
|
+
if (updates.tags !== undefined) metadata.tags = updates.tags;
|
|
674
|
+
|
|
675
|
+
// Record events for each affected job
|
|
676
|
+
for (const row of result.rows) {
|
|
677
|
+
await recordJobEvent(pool, row.id, JobEventType.Edited, metadata);
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
|
|
681
|
+
return editedCount;
|
|
682
|
+
} catch (error) {
|
|
683
|
+
log(`Error editing pending jobs: ${error}`);
|
|
684
|
+
throw error;
|
|
685
|
+
} finally {
|
|
686
|
+
client.release();
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
|
|
416
690
|
/**
|
|
417
691
|
* Cancel all upcoming jobs (pending and scheduled in the future) with optional filters
|
|
418
692
|
*/
|
|
@@ -526,7 +800,7 @@ export const getAllJobs = async <
|
|
|
526
800
|
const client = await pool.connect();
|
|
527
801
|
try {
|
|
528
802
|
const result = await client.query(
|
|
529
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
803
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
530
804
|
[limit, offset],
|
|
531
805
|
);
|
|
532
806
|
log(`Found ${result.rows.length} jobs (all)`);
|
|
@@ -534,6 +808,7 @@ export const getAllJobs = async <
|
|
|
534
808
|
...job,
|
|
535
809
|
payload: job.payload,
|
|
536
810
|
timeoutMs: job.timeoutMs,
|
|
811
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
537
812
|
}));
|
|
538
813
|
} catch (error) {
|
|
539
814
|
log(`Error getting all jobs: ${error}`);
|
|
@@ -674,6 +949,7 @@ export const getJobsByTags = async <
|
|
|
674
949
|
...job,
|
|
675
950
|
payload: job.payload,
|
|
676
951
|
timeoutMs: job.timeoutMs,
|
|
952
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
677
953
|
failureReason: job.failureReason,
|
|
678
954
|
}));
|
|
679
955
|
} catch (error) {
|
|
@@ -699,7 +975,7 @@ export const getJobs = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
699
975
|
): Promise<JobRecord<PayloadMap, T>[]> => {
|
|
700
976
|
const client = await pool.connect();
|
|
701
977
|
try {
|
|
702
|
-
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags FROM job_queue`;
|
|
978
|
+
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags FROM job_queue`;
|
|
703
979
|
const params: any[] = [];
|
|
704
980
|
let where: string[] = [];
|
|
705
981
|
let paramIdx = 1;
|
|
@@ -796,6 +1072,7 @@ export const getJobs = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
796
1072
|
...job,
|
|
797
1073
|
payload: job.payload,
|
|
798
1074
|
timeoutMs: job.timeoutMs,
|
|
1075
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
799
1076
|
failureReason: job.failureReason,
|
|
800
1077
|
}));
|
|
801
1078
|
} catch (error) {
|
package/src/types.ts
CHANGED
|
@@ -13,12 +13,71 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
13
13
|
* Timeout for this job in milliseconds. If not set, uses the processor default or unlimited.
|
|
14
14
|
*/
|
|
15
15
|
timeoutMs?: number;
|
|
16
|
+
/**
|
|
17
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
18
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
19
|
+
*
|
|
20
|
+
* **⚠️ RUNTIME REQUIREMENTS**: This option requires **Node.js** and uses the `worker_threads` module.
|
|
21
|
+
* It will **not work** in Bun or other runtimes that don't support Node.js worker threads.
|
|
22
|
+
*
|
|
23
|
+
* **IMPORTANT**: When `forceKillOnTimeout` is true, the handler must be serializable. This means:
|
|
24
|
+
* - The handler should be a standalone function (not a closure over external variables)
|
|
25
|
+
* - It should not capture variables from outer scopes that reference external dependencies
|
|
26
|
+
* - It should not use 'this' context unless it's a bound method
|
|
27
|
+
* - All dependencies must be importable in the worker thread context
|
|
28
|
+
*
|
|
29
|
+
* **Examples of serializable handlers:**
|
|
30
|
+
* ```ts
|
|
31
|
+
* // ✅ Good - standalone function
|
|
32
|
+
* const handler = async (payload, signal) => {
|
|
33
|
+
* await doSomething(payload);
|
|
34
|
+
* };
|
|
35
|
+
*
|
|
36
|
+
* // ✅ Good - function that imports dependencies
|
|
37
|
+
* const handler = async (payload, signal) => {
|
|
38
|
+
* const { api } = await import('./api');
|
|
39
|
+
* await api.call(payload);
|
|
40
|
+
* };
|
|
41
|
+
*
|
|
42
|
+
* // ❌ Bad - closure over external variable
|
|
43
|
+
* const db = getDatabase();
|
|
44
|
+
* const handler = async (payload, signal) => {
|
|
45
|
+
* await db.query(payload); // 'db' is captured from closure
|
|
46
|
+
* };
|
|
47
|
+
*
|
|
48
|
+
* // ❌ Bad - uses 'this' context
|
|
49
|
+
* class MyHandler {
|
|
50
|
+
* async handle(payload, signal) {
|
|
51
|
+
* await this.doSomething(payload); // 'this' won't work
|
|
52
|
+
* }
|
|
53
|
+
* }
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* If your handler doesn't meet these requirements, use `forceKillOnTimeout: false` (default)
|
|
57
|
+
* and ensure your handler checks `signal.aborted` to exit gracefully.
|
|
58
|
+
*
|
|
59
|
+
* Note: forceKillOnTimeout requires timeoutMs to be set.
|
|
60
|
+
*/
|
|
61
|
+
forceKillOnTimeout?: boolean;
|
|
16
62
|
/**
|
|
17
63
|
* Tags for this job. Used for grouping, searching, or batch operations.
|
|
18
64
|
*/
|
|
19
65
|
tags?: string[];
|
|
20
66
|
}
|
|
21
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Options for editing a pending job.
|
|
70
|
+
* All fields are optional and only provided fields will be updated.
|
|
71
|
+
* Note: jobType cannot be changed.
|
|
72
|
+
* timeoutMs and tags can be set to null to clear them.
|
|
73
|
+
*/
|
|
74
|
+
export type EditJobOptions<PayloadMap, T extends JobType<PayloadMap>> = Partial<
|
|
75
|
+
Omit<JobOptions<PayloadMap, T>, 'jobType'>
|
|
76
|
+
> & {
|
|
77
|
+
timeoutMs?: number | null;
|
|
78
|
+
tags?: string[] | null;
|
|
79
|
+
};
|
|
80
|
+
|
|
22
81
|
export enum JobEventType {
|
|
23
82
|
Added = 'added',
|
|
24
83
|
Processing = 'processing',
|
|
@@ -26,6 +85,7 @@ export enum JobEventType {
|
|
|
26
85
|
Failed = 'failed',
|
|
27
86
|
Cancelled = 'cancelled',
|
|
28
87
|
Retried = 'retried',
|
|
88
|
+
Edited = 'edited',
|
|
29
89
|
}
|
|
30
90
|
|
|
31
91
|
export interface JobEvent {
|
|
@@ -69,6 +129,11 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
|
|
|
69
129
|
* Timeout for this job in milliseconds (null means no timeout).
|
|
70
130
|
*/
|
|
71
131
|
timeoutMs?: number | null;
|
|
132
|
+
/**
|
|
133
|
+
* If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
|
|
134
|
+
* If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
|
|
135
|
+
*/
|
|
136
|
+
forceKillOnTimeout?: boolean | null;
|
|
72
137
|
/**
|
|
73
138
|
* The reason for the last failure, if any.
|
|
74
139
|
*/
|
|
@@ -269,6 +334,43 @@ export interface JobQueue<PayloadMap> {
|
|
|
269
334
|
* - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
|
|
270
335
|
*/
|
|
271
336
|
cancelJob: (jobId: number) => Promise<void>;
|
|
337
|
+
/**
|
|
338
|
+
* Edit a pending job given its ID.
|
|
339
|
+
* - Only works for jobs with status 'pending'. Silently fails for other statuses.
|
|
340
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
341
|
+
* - jobType cannot be changed.
|
|
342
|
+
* - Records an 'edited' event with the updated fields in metadata.
|
|
343
|
+
*/
|
|
344
|
+
editJob: <T extends JobType<PayloadMap>>(
|
|
345
|
+
jobId: number,
|
|
346
|
+
updates: EditJobOptions<PayloadMap, T>,
|
|
347
|
+
) => Promise<void>;
|
|
348
|
+
/**
|
|
349
|
+
* Edit all pending jobs that match the filters.
|
|
350
|
+
* - Only works for jobs with status 'pending'. Non-pending jobs are not affected.
|
|
351
|
+
* - All fields in EditJobOptions are optional - only provided fields will be updated.
|
|
352
|
+
* - jobType cannot be changed.
|
|
353
|
+
* - Records an 'edited' event with the updated fields in metadata for each affected job.
|
|
354
|
+
* - Returns the number of jobs that were edited.
|
|
355
|
+
* - The filters are:
|
|
356
|
+
* - jobType: The job type to edit.
|
|
357
|
+
* - priority: The priority of the job to edit.
|
|
358
|
+
* - runAt: The time the job is scheduled to run at (now supports gt/gte/lt/lte/eq).
|
|
359
|
+
* - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based editing.
|
|
360
|
+
*/
|
|
361
|
+
editAllPendingJobs: <T extends JobType<PayloadMap>>(
|
|
362
|
+
filters:
|
|
363
|
+
| {
|
|
364
|
+
jobType?: string;
|
|
365
|
+
priority?: number;
|
|
366
|
+
runAt?:
|
|
367
|
+
| Date
|
|
368
|
+
| { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
|
|
369
|
+
tags?: { values: string[]; mode?: TagQueryMode };
|
|
370
|
+
}
|
|
371
|
+
| undefined,
|
|
372
|
+
updates: EditJobOptions<PayloadMap, T>,
|
|
373
|
+
) => Promise<number>;
|
|
272
374
|
/**
|
|
273
375
|
* Reclaim stuck jobs.
|
|
274
376
|
* - If a process (e.g., API route or worker) crashes after marking a job as 'processing' but before completing it, the job can remain stuck in the 'processing' state indefinitely. This can happen if the process is killed or encounters an unhandled error after updating the job status but before marking it as 'completed' or 'failed'.
|