@nicnocquee/dataqueue 1.16.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +44 -42
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +32 -31
- package/dist/index.d.ts +32 -31
- package/dist/index.js +44 -42
- package/dist/index.js.map +1 -1
- package/package.json +4 -2
- package/src/index.test.ts +30 -30
- package/src/index.ts +1 -1
- package/src/processor.test.ts +24 -24
- package/src/processor.ts +5 -5
- package/src/queue.test.ts +74 -73
- package/src/queue.ts +43 -40
- package/src/types.ts +37 -30
- package/src/utils.ts +19 -0
package/src/queue.test.ts
CHANGED
|
@@ -2,7 +2,8 @@ import { Pool } from 'pg';
|
|
|
2
2
|
import { describe, expect, it, beforeEach, afterEach } from 'vitest';
|
|
3
3
|
import * as queue from './queue.js';
|
|
4
4
|
import { createTestDbAndPool, destroyTestDb } from './test-util.js';
|
|
5
|
-
import { JobEventType } from './types.js';
|
|
5
|
+
import { JobEvent, JobEventType } from './types.js';
|
|
6
|
+
import { objectKeysToCamelCase } from './utils.js';
|
|
6
7
|
|
|
7
8
|
// Example integration test setup
|
|
8
9
|
|
|
@@ -23,13 +24,13 @@ describe('queue integration', () => {
|
|
|
23
24
|
|
|
24
25
|
it('should add a job and retrieve it', async () => {
|
|
25
26
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
26
|
-
|
|
27
|
+
jobType: 'email',
|
|
27
28
|
payload: { to: 'test@example.com' },
|
|
28
29
|
});
|
|
29
30
|
expect(typeof jobId).toBe('number');
|
|
30
31
|
const job = await queue.getJob(pool, jobId);
|
|
31
32
|
expect(job).not.toBeNull();
|
|
32
|
-
expect(job?.
|
|
33
|
+
expect(job?.jobType).toBe('email');
|
|
33
34
|
expect(job?.payload).toEqual({ to: 'test@example.com' });
|
|
34
35
|
});
|
|
35
36
|
|
|
@@ -38,12 +39,12 @@ describe('queue integration', () => {
|
|
|
38
39
|
const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
39
40
|
pool,
|
|
40
41
|
{
|
|
41
|
-
|
|
42
|
+
jobType: 'email',
|
|
42
43
|
payload: { to: 'a@example.com' },
|
|
43
44
|
},
|
|
44
45
|
);
|
|
45
46
|
const jobId2 = await queue.addJob<{ sms: { to: string } }, 'sms'>(pool, {
|
|
46
|
-
|
|
47
|
+
jobType: 'sms',
|
|
47
48
|
payload: { to: 'b@example.com' },
|
|
48
49
|
});
|
|
49
50
|
// All jobs should be 'pending' by default
|
|
@@ -55,7 +56,7 @@ describe('queue integration', () => {
|
|
|
55
56
|
|
|
56
57
|
it('should retry a failed job', async () => {
|
|
57
58
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
58
|
-
|
|
59
|
+
jobType: 'email',
|
|
59
60
|
payload: { to: 'fail@example.com' },
|
|
60
61
|
});
|
|
61
62
|
// Mark as failed
|
|
@@ -70,7 +71,7 @@ describe('queue integration', () => {
|
|
|
70
71
|
|
|
71
72
|
it('should mark a job as completed', async () => {
|
|
72
73
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
73
|
-
|
|
74
|
+
jobType: 'email',
|
|
74
75
|
payload: { to: 'done@example.com' },
|
|
75
76
|
});
|
|
76
77
|
await queue.completeJob(pool, jobId);
|
|
@@ -79,18 +80,18 @@ describe('queue integration', () => {
|
|
|
79
80
|
});
|
|
80
81
|
|
|
81
82
|
it('should get the next batch of jobs to process', async () => {
|
|
82
|
-
// Add jobs (do not set
|
|
83
|
+
// Add jobs (do not set runAt, use DB default)
|
|
83
84
|
const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
84
85
|
pool,
|
|
85
86
|
{
|
|
86
|
-
|
|
87
|
+
jobType: 'email',
|
|
87
88
|
payload: { to: 'batch1@example.com' },
|
|
88
89
|
},
|
|
89
90
|
);
|
|
90
91
|
const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
91
92
|
pool,
|
|
92
93
|
{
|
|
93
|
-
|
|
94
|
+
jobType: 'email',
|
|
94
95
|
payload: { to: 'batch2@example.com' },
|
|
95
96
|
},
|
|
96
97
|
);
|
|
@@ -107,9 +108,9 @@ describe('queue integration', () => {
|
|
|
107
108
|
// Add a job scheduled 1 day in the future
|
|
108
109
|
const futureDate = new Date(Date.now() + 24 * 60 * 60 * 1000);
|
|
109
110
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
110
|
-
|
|
111
|
+
jobType: 'email',
|
|
111
112
|
payload: { to: 'future@example.com' },
|
|
112
|
-
|
|
113
|
+
runAt: futureDate,
|
|
113
114
|
});
|
|
114
115
|
const jobs = await queue.getNextBatch(pool, 'worker-2', 1);
|
|
115
116
|
const ids = jobs.map((j) => j.id);
|
|
@@ -122,7 +123,7 @@ describe('queue integration', () => {
|
|
|
122
123
|
it('should cleanup old completed jobs', async () => {
|
|
123
124
|
// Add and complete a job
|
|
124
125
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
125
|
-
|
|
126
|
+
jobType: 'email',
|
|
126
127
|
payload: { to: 'cleanup@example.com' },
|
|
127
128
|
});
|
|
128
129
|
await queue.completeJob(pool, jobId);
|
|
@@ -140,7 +141,7 @@ describe('queue integration', () => {
|
|
|
140
141
|
|
|
141
142
|
it('should cancel a scheduled job', async () => {
|
|
142
143
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
143
|
-
|
|
144
|
+
jobType: 'email',
|
|
144
145
|
payload: { to: 'cancelme@example.com' },
|
|
145
146
|
});
|
|
146
147
|
await queue.cancelJob(pool, jobId);
|
|
@@ -151,7 +152,7 @@ describe('queue integration', () => {
|
|
|
151
152
|
const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
152
153
|
pool,
|
|
153
154
|
{
|
|
154
|
-
|
|
155
|
+
jobType: 'email',
|
|
155
156
|
payload: { to: 'done@example.com' },
|
|
156
157
|
},
|
|
157
158
|
);
|
|
@@ -166,21 +167,21 @@ describe('queue integration', () => {
|
|
|
166
167
|
const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
167
168
|
pool,
|
|
168
169
|
{
|
|
169
|
-
|
|
170
|
+
jobType: 'email',
|
|
170
171
|
payload: { to: 'cancelall1@example.com' },
|
|
171
172
|
},
|
|
172
173
|
);
|
|
173
174
|
const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
174
175
|
pool,
|
|
175
176
|
{
|
|
176
|
-
|
|
177
|
+
jobType: 'email',
|
|
177
178
|
payload: { to: 'cancelall2@example.com' },
|
|
178
179
|
},
|
|
179
180
|
);
|
|
180
181
|
const jobId3 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
181
182
|
pool,
|
|
182
183
|
{
|
|
183
|
-
|
|
184
|
+
jobType: 'email',
|
|
184
185
|
payload: { to: 'cancelall3@example.com' },
|
|
185
186
|
},
|
|
186
187
|
);
|
|
@@ -188,7 +189,7 @@ describe('queue integration', () => {
|
|
|
188
189
|
const jobId4 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
189
190
|
pool,
|
|
190
191
|
{
|
|
191
|
-
|
|
192
|
+
jobType: 'email',
|
|
192
193
|
payload: { to: 'done@example.com' },
|
|
193
194
|
},
|
|
194
195
|
);
|
|
@@ -211,17 +212,17 @@ describe('queue integration', () => {
|
|
|
211
212
|
expect(completedJob?.status).toBe('completed');
|
|
212
213
|
});
|
|
213
214
|
|
|
214
|
-
it('should store and retrieve
|
|
215
|
+
it('should store and retrieve runAt in UTC without timezone shift', async () => {
|
|
215
216
|
const utcDate = new Date(Date.UTC(2030, 0, 1, 12, 0, 0, 0)); // 2030-01-01T12:00:00.000Z
|
|
216
217
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
217
|
-
|
|
218
|
+
jobType: 'email',
|
|
218
219
|
payload: { to: 'utc@example.com' },
|
|
219
|
-
|
|
220
|
+
runAt: utcDate,
|
|
220
221
|
});
|
|
221
222
|
const job = await queue.getJob(pool, jobId);
|
|
222
223
|
expect(job).not.toBeNull();
|
|
223
|
-
// The
|
|
224
|
-
expect(job?.
|
|
224
|
+
// The runAt value should match exactly (toISOString) what we inserted
|
|
225
|
+
expect(job?.runAt.toISOString()).toBe(utcDate.toISOString());
|
|
225
226
|
});
|
|
226
227
|
|
|
227
228
|
it('should get all jobs', async () => {
|
|
@@ -229,16 +230,16 @@ describe('queue integration', () => {
|
|
|
229
230
|
const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
|
|
230
231
|
pool,
|
|
231
232
|
{
|
|
232
|
-
|
|
233
|
+
jobType: 'email',
|
|
233
234
|
payload: { to: 'all1@example.com' },
|
|
234
235
|
},
|
|
235
236
|
);
|
|
236
237
|
const jobId2 = await queue.addJob<{ sms: { to: string } }, 'sms'>(pool, {
|
|
237
|
-
|
|
238
|
+
jobType: 'sms',
|
|
238
239
|
payload: { to: 'all2@example.com' },
|
|
239
240
|
});
|
|
240
241
|
const jobId3 = await queue.addJob<{ push: { to: string } }, 'push'>(pool, {
|
|
241
|
-
|
|
242
|
+
jobType: 'push',
|
|
242
243
|
payload: { to: 'all3@example.com' },
|
|
243
244
|
});
|
|
244
245
|
// Get all jobs
|
|
@@ -249,26 +250,26 @@ describe('queue integration', () => {
|
|
|
249
250
|
expect(ids).toContain(jobId3);
|
|
250
251
|
// Should return correct job data
|
|
251
252
|
const job1 = jobs.find((j) => j.id === jobId1);
|
|
252
|
-
expect(job1?.
|
|
253
|
+
expect(job1?.jobType).toBe('email');
|
|
253
254
|
expect(job1?.payload).toEqual({ to: 'all1@example.com' });
|
|
254
255
|
});
|
|
255
256
|
|
|
256
257
|
it('should support pagination in getAllJobs', async () => {
|
|
257
258
|
// Add four jobs
|
|
258
259
|
await queue.addJob<{ a: { n: number } }, 'a'>(pool, {
|
|
259
|
-
|
|
260
|
+
jobType: 'a',
|
|
260
261
|
payload: { n: 1 },
|
|
261
262
|
});
|
|
262
263
|
await queue.addJob<{ b: { n: number } }, 'b'>(pool, {
|
|
263
|
-
|
|
264
|
+
jobType: 'b',
|
|
264
265
|
payload: { n: 2 },
|
|
265
266
|
});
|
|
266
267
|
await queue.addJob<{ c: { n: number } }, 'c'>(pool, {
|
|
267
|
-
|
|
268
|
+
jobType: 'c',
|
|
268
269
|
payload: { n: 3 },
|
|
269
270
|
});
|
|
270
271
|
await queue.addJob<{ d: { n: number } }, 'd'>(pool, {
|
|
271
|
-
|
|
272
|
+
jobType: 'd',
|
|
272
273
|
payload: { n: 4 },
|
|
273
274
|
});
|
|
274
275
|
// Get first two jobs
|
|
@@ -285,7 +286,7 @@ describe('queue integration', () => {
|
|
|
285
286
|
|
|
286
287
|
it('should track error history for failed jobs', async () => {
|
|
287
288
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
288
|
-
|
|
289
|
+
jobType: 'email',
|
|
289
290
|
payload: { to: 'failhistory@example.com' },
|
|
290
291
|
});
|
|
291
292
|
// Fail the job twice with different errors
|
|
@@ -293,18 +294,18 @@ describe('queue integration', () => {
|
|
|
293
294
|
await queue.failJob(pool, jobId, new Error('second error'));
|
|
294
295
|
const job = await queue.getJob(pool, jobId);
|
|
295
296
|
expect(job?.status).toBe('failed');
|
|
296
|
-
expect(Array.isArray(job?.
|
|
297
|
-
expect(job?.
|
|
298
|
-
expect(job?.
|
|
299
|
-
expect(job?.
|
|
300
|
-
expect(typeof job?.
|
|
301
|
-
expect(typeof job?.
|
|
297
|
+
expect(Array.isArray(job?.errorHistory)).toBe(true);
|
|
298
|
+
expect(job?.errorHistory?.length).toBeGreaterThanOrEqual(2);
|
|
299
|
+
expect(job?.errorHistory?.[0].message).toBe('first error');
|
|
300
|
+
expect(job?.errorHistory?.[1].message).toBe('second error');
|
|
301
|
+
expect(typeof job?.errorHistory?.[0].timestamp).toBe('string');
|
|
302
|
+
expect(typeof job?.errorHistory?.[1].timestamp).toBe('string');
|
|
302
303
|
});
|
|
303
304
|
|
|
304
305
|
it('should reclaim stuck processing jobs', async () => {
|
|
305
306
|
// Add a job and set it to processing with an old locked_at
|
|
306
307
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
307
|
-
|
|
308
|
+
jobType: 'email',
|
|
308
309
|
payload: { to: 'stuck@example.com' },
|
|
309
310
|
});
|
|
310
311
|
await pool.query(
|
|
@@ -319,8 +320,8 @@ describe('queue integration', () => {
|
|
|
319
320
|
expect(reclaimed).toBeGreaterThanOrEqual(1);
|
|
320
321
|
job = await queue.getJob(pool, jobId);
|
|
321
322
|
expect(job?.status).toBe('pending');
|
|
322
|
-
expect(job?.
|
|
323
|
-
expect(job?.
|
|
323
|
+
expect(job?.lockedAt).toBeNull();
|
|
324
|
+
expect(job?.lockedBy).toBeNull();
|
|
324
325
|
});
|
|
325
326
|
});
|
|
326
327
|
|
|
@@ -344,18 +345,18 @@ describe('job event tracking', () => {
|
|
|
344
345
|
'SELECT * FROM job_events WHERE job_id = $1 ORDER BY created_at ASC',
|
|
345
346
|
[jobId],
|
|
346
347
|
);
|
|
347
|
-
return res.rows;
|
|
348
|
+
return res.rows.map((row) => objectKeysToCamelCase(row) as JobEvent);
|
|
348
349
|
}
|
|
349
350
|
|
|
350
351
|
it('records added and processing events', async () => {
|
|
351
352
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
352
|
-
|
|
353
|
+
jobType: 'email',
|
|
353
354
|
payload: { to: 'event1@example.com' },
|
|
354
355
|
});
|
|
355
356
|
// Pick up for processing
|
|
356
357
|
await queue.getNextBatch(pool, 'worker-evt', 1);
|
|
357
358
|
const events = await getEvents(jobId);
|
|
358
|
-
expect(events.map((e) => e.
|
|
359
|
+
expect(events.map((e) => e.eventType)).toEqual([
|
|
359
360
|
JobEventType.Added,
|
|
360
361
|
JobEventType.Processing,
|
|
361
362
|
]);
|
|
@@ -363,39 +364,39 @@ describe('job event tracking', () => {
|
|
|
363
364
|
|
|
364
365
|
it('records completed event', async () => {
|
|
365
366
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
366
|
-
|
|
367
|
+
jobType: 'email',
|
|
367
368
|
payload: { to: 'event2@example.com' },
|
|
368
369
|
});
|
|
369
370
|
await queue.getNextBatch(pool, 'worker-evt', 1);
|
|
370
371
|
await queue.completeJob(pool, jobId);
|
|
371
372
|
const events = await getEvents(jobId);
|
|
372
|
-
expect(events.map((e) => e.
|
|
373
|
+
expect(events.map((e) => e.eventType)).toContain(JobEventType.Completed);
|
|
373
374
|
});
|
|
374
375
|
|
|
375
376
|
it('records failed and retried events', async () => {
|
|
376
377
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
377
|
-
|
|
378
|
+
jobType: 'email',
|
|
378
379
|
payload: { to: 'event3@example.com' },
|
|
379
380
|
});
|
|
380
381
|
await queue.getNextBatch(pool, 'worker-evt', 1);
|
|
381
382
|
await queue.failJob(pool, jobId, new Error('fail for event'));
|
|
382
383
|
await queue.retryJob(pool, jobId);
|
|
383
384
|
const events = await getEvents(jobId);
|
|
384
|
-
expect(events.map((e) => e.
|
|
385
|
+
expect(events.map((e) => e.eventType)).toEqual(
|
|
385
386
|
expect.arrayContaining([JobEventType.Failed, JobEventType.Retried]),
|
|
386
387
|
);
|
|
387
|
-
const failEvent = events.find((e) => e.
|
|
388
|
-
expect(failEvent
|
|
388
|
+
const failEvent = events.find((e) => e.eventType === JobEventType.Failed);
|
|
389
|
+
expect(failEvent?.metadata).toMatchObject({ message: 'fail for event' });
|
|
389
390
|
});
|
|
390
391
|
|
|
391
392
|
it('records cancelled event', async () => {
|
|
392
393
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
393
|
-
|
|
394
|
+
jobType: 'email',
|
|
394
395
|
payload: { to: 'event4@example.com' },
|
|
395
396
|
});
|
|
396
397
|
await queue.cancelJob(pool, jobId);
|
|
397
398
|
const events = await getEvents(jobId);
|
|
398
|
-
expect(events.map((e) => e.
|
|
399
|
+
expect(events.map((e) => e.eventType)).toContain(JobEventType.Cancelled);
|
|
399
400
|
});
|
|
400
401
|
});
|
|
401
402
|
|
|
@@ -421,41 +422,41 @@ describe('job lifecycle timestamp columns', () => {
|
|
|
421
422
|
return res.rows[0];
|
|
422
423
|
}
|
|
423
424
|
|
|
424
|
-
it('sets
|
|
425
|
+
it('sets startedAt when job is picked up for processing', async () => {
|
|
425
426
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
426
|
-
|
|
427
|
+
jobType: 'email',
|
|
427
428
|
payload: { to: 'ts1@example.com' },
|
|
428
429
|
});
|
|
429
430
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
430
431
|
const job = await getJobRow(jobId);
|
|
431
|
-
expect(job.
|
|
432
|
+
expect(job.startedAt).not.toBeNull();
|
|
432
433
|
});
|
|
433
434
|
|
|
434
|
-
it('sets
|
|
435
|
+
it('sets completedAt when job is completed', async () => {
|
|
435
436
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
436
|
-
|
|
437
|
+
jobType: 'email',
|
|
437
438
|
payload: { to: 'ts2@example.com' },
|
|
438
439
|
});
|
|
439
440
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
440
441
|
await queue.completeJob(pool, jobId);
|
|
441
442
|
const job = await getJobRow(jobId);
|
|
442
|
-
expect(job.
|
|
443
|
+
expect(job.completedAt).not.toBeNull();
|
|
443
444
|
});
|
|
444
445
|
|
|
445
|
-
it('sets
|
|
446
|
+
it('sets lastFailedAt when job fails', async () => {
|
|
446
447
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
447
|
-
|
|
448
|
+
jobType: 'email',
|
|
448
449
|
payload: { to: 'ts3@example.com' },
|
|
449
450
|
});
|
|
450
451
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
451
452
|
await queue.failJob(pool, jobId, new Error('fail for ts'));
|
|
452
453
|
const job = await getJobRow(jobId);
|
|
453
|
-
expect(job.
|
|
454
|
+
expect(job.lastFailedAt).not.toBeNull();
|
|
454
455
|
});
|
|
455
456
|
|
|
456
|
-
it('sets
|
|
457
|
+
it('sets lastRetriedAt when job is retried', async () => {
|
|
457
458
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
458
|
-
|
|
459
|
+
jobType: 'email',
|
|
459
460
|
payload: { to: 'ts4@example.com' },
|
|
460
461
|
});
|
|
461
462
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
@@ -465,25 +466,25 @@ describe('job lifecycle timestamp columns', () => {
|
|
|
465
466
|
'UPDATE job_queue SET next_attempt_at = NOW() WHERE id = $1',
|
|
466
467
|
[jobId],
|
|
467
468
|
);
|
|
468
|
-
// Pick up for processing again (should increment attempts and set
|
|
469
|
+
// Pick up for processing again (should increment attempts and set lastRetriedAt)
|
|
469
470
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
470
471
|
const job = await getJobRow(jobId);
|
|
471
|
-
expect(job.
|
|
472
|
+
expect(job.lastRetriedAt).not.toBeNull();
|
|
472
473
|
});
|
|
473
474
|
|
|
474
|
-
it('sets
|
|
475
|
+
it('sets lastCancelledAt when job is cancelled', async () => {
|
|
475
476
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
476
|
-
|
|
477
|
+
jobType: 'email',
|
|
477
478
|
payload: { to: 'ts5@example.com' },
|
|
478
479
|
});
|
|
479
480
|
await queue.cancelJob(pool, jobId);
|
|
480
481
|
const job = await getJobRow(jobId);
|
|
481
|
-
expect(job.
|
|
482
|
+
expect(job.lastCancelledAt).not.toBeNull();
|
|
482
483
|
});
|
|
483
484
|
|
|
484
|
-
it('sets
|
|
485
|
+
it('sets lastRetriedAt when job is picked up for processing again (attempts > 0)', async () => {
|
|
485
486
|
const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
|
|
486
|
-
|
|
487
|
+
jobType: 'email',
|
|
487
488
|
payload: { to: 'ts6@example.com' },
|
|
488
489
|
});
|
|
489
490
|
// First pick up and fail the job
|
|
@@ -494,9 +495,9 @@ describe('job lifecycle timestamp columns', () => {
|
|
|
494
495
|
'UPDATE job_queue SET next_attempt_at = NOW() WHERE id = $1',
|
|
495
496
|
[jobId],
|
|
496
497
|
);
|
|
497
|
-
// Pick up for processing again (should increment attempts and set
|
|
498
|
+
// Pick up for processing again (should increment attempts and set lastRetriedAt)
|
|
498
499
|
await queue.getNextBatch(pool, 'worker-ts', 1);
|
|
499
500
|
const job = await getJobRow(jobId);
|
|
500
|
-
expect(job.
|
|
501
|
+
expect(job.lastRetriedAt).not.toBeNull();
|
|
501
502
|
});
|
|
502
503
|
});
|
package/src/queue.ts
CHANGED
|
@@ -37,27 +37,27 @@ export const recordJobEvent = async (
|
|
|
37
37
|
export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
38
38
|
pool: Pool,
|
|
39
39
|
{
|
|
40
|
-
|
|
40
|
+
jobType,
|
|
41
41
|
payload,
|
|
42
|
-
|
|
42
|
+
maxAttempts = 3,
|
|
43
43
|
priority = 0,
|
|
44
|
-
|
|
44
|
+
runAt = null,
|
|
45
45
|
timeoutMs = undefined,
|
|
46
46
|
}: JobOptions<PayloadMap, T>,
|
|
47
47
|
): Promise<number> => {
|
|
48
48
|
const client = await pool.connect();
|
|
49
49
|
try {
|
|
50
50
|
let result;
|
|
51
|
-
if (
|
|
51
|
+
if (runAt) {
|
|
52
52
|
result = await client.query(
|
|
53
53
|
`INSERT INTO job_queue
|
|
54
54
|
(job_type, payload, max_attempts, priority, run_at, timeout_ms)
|
|
55
55
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
56
56
|
RETURNING id`,
|
|
57
|
-
[
|
|
57
|
+
[jobType, payload, maxAttempts, priority, runAt, timeoutMs ?? null],
|
|
58
58
|
);
|
|
59
59
|
log(
|
|
60
|
-
`Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)},
|
|
60
|
+
`Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, runAt ${runAt.toISOString()}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}`,
|
|
61
61
|
);
|
|
62
62
|
} else {
|
|
63
63
|
result = await client.query(
|
|
@@ -65,14 +65,14 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
65
65
|
(job_type, payload, max_attempts, priority, timeout_ms)
|
|
66
66
|
VALUES ($1, $2, $3, $4, $5)
|
|
67
67
|
RETURNING id`,
|
|
68
|
-
[
|
|
68
|
+
[jobType, payload, maxAttempts, priority, timeoutMs ?? null],
|
|
69
69
|
);
|
|
70
70
|
log(
|
|
71
|
-
`Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, priority ${priority},
|
|
71
|
+
`Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}`,
|
|
72
72
|
);
|
|
73
73
|
}
|
|
74
74
|
await recordJobEvent(pool, result.rows[0].id, JobEventType.Added, {
|
|
75
|
-
|
|
75
|
+
jobType,
|
|
76
76
|
payload,
|
|
77
77
|
});
|
|
78
78
|
return result.rows[0].id;
|
|
@@ -93,9 +93,10 @@ export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
93
93
|
): Promise<JobRecord<PayloadMap, T> | null> => {
|
|
94
94
|
const client = await pool.connect();
|
|
95
95
|
try {
|
|
96
|
-
const result = await client.query(
|
|
97
|
-
id,
|
|
98
|
-
|
|
96
|
+
const result = await client.query(
|
|
97
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue WHERE id = $1`,
|
|
98
|
+
[id],
|
|
99
|
+
);
|
|
99
100
|
|
|
100
101
|
if (result.rows.length === 0) {
|
|
101
102
|
log(`Job ${id} not found`);
|
|
@@ -104,11 +105,13 @@ export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
|
|
|
104
105
|
|
|
105
106
|
log(`Found job ${id}`);
|
|
106
107
|
|
|
108
|
+
const job = result.rows[0] as JobRecord<PayloadMap, T>;
|
|
109
|
+
|
|
107
110
|
return {
|
|
108
|
-
...
|
|
109
|
-
payload:
|
|
110
|
-
|
|
111
|
-
|
|
111
|
+
...job,
|
|
112
|
+
payload: job.payload,
|
|
113
|
+
timeoutMs: job.timeoutMs,
|
|
114
|
+
failureReason: job.failureReason,
|
|
112
115
|
};
|
|
113
116
|
} catch (error) {
|
|
114
117
|
log(`Error getting job ${id}: ${error}`);
|
|
@@ -133,17 +136,17 @@ export const getJobsByStatus = async <
|
|
|
133
136
|
const client = await pool.connect();
|
|
134
137
|
try {
|
|
135
138
|
const result = await client.query(
|
|
136
|
-
|
|
139
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
|
|
137
140
|
[status, limit, offset],
|
|
138
141
|
);
|
|
139
142
|
|
|
140
143
|
log(`Found ${result.rows.length} jobs by status ${status}`);
|
|
141
144
|
|
|
142
|
-
return result.rows.map((
|
|
143
|
-
...
|
|
144
|
-
payload:
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
return result.rows.map((job) => ({
|
|
146
|
+
...job,
|
|
147
|
+
payload: job.payload,
|
|
148
|
+
timeoutMs: job.timeoutMs,
|
|
149
|
+
failureReason: job.failureReason,
|
|
147
150
|
}));
|
|
148
151
|
} catch (error) {
|
|
149
152
|
log(`Error getting jobs by status ${status}: ${error}`);
|
|
@@ -209,7 +212,7 @@ export const getNextBatch = async <
|
|
|
209
212
|
LIMIT $2
|
|
210
213
|
FOR UPDATE SKIP LOCKED
|
|
211
214
|
)
|
|
212
|
-
RETURNING
|
|
215
|
+
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason"
|
|
213
216
|
`,
|
|
214
217
|
params,
|
|
215
218
|
);
|
|
@@ -224,10 +227,10 @@ export const getNextBatch = async <
|
|
|
224
227
|
await recordJobEvent(pool, row.id, JobEventType.Processing);
|
|
225
228
|
}
|
|
226
229
|
|
|
227
|
-
return result.rows.map((
|
|
228
|
-
...
|
|
229
|
-
payload:
|
|
230
|
-
|
|
230
|
+
return result.rows.map((job) => ({
|
|
231
|
+
...job,
|
|
232
|
+
payload: job.payload,
|
|
233
|
+
timeoutMs: job.timeoutMs,
|
|
231
234
|
}));
|
|
232
235
|
} catch (error) {
|
|
233
236
|
log(`Error getting next batch: ${error}`);
|
|
@@ -397,7 +400,7 @@ export const cancelJob = async (pool: Pool, jobId: number): Promise<void> => {
|
|
|
397
400
|
*/
|
|
398
401
|
export const cancelAllUpcomingJobs = async (
|
|
399
402
|
pool: Pool,
|
|
400
|
-
filters?: {
|
|
403
|
+
filters?: { jobType?: string; priority?: number; runAt?: Date },
|
|
401
404
|
): Promise<number> => {
|
|
402
405
|
const client = await pool.connect();
|
|
403
406
|
try {
|
|
@@ -408,17 +411,17 @@ export const cancelAllUpcomingJobs = async (
|
|
|
408
411
|
const params: any[] = [];
|
|
409
412
|
let paramIdx = 1;
|
|
410
413
|
if (filters) {
|
|
411
|
-
if (filters.
|
|
414
|
+
if (filters.jobType) {
|
|
412
415
|
query += ` AND job_type = $${paramIdx++}`;
|
|
413
|
-
params.push(filters.
|
|
416
|
+
params.push(filters.jobType);
|
|
414
417
|
}
|
|
415
418
|
if (filters.priority !== undefined) {
|
|
416
419
|
query += ` AND priority = $${paramIdx++}`;
|
|
417
420
|
params.push(filters.priority);
|
|
418
421
|
}
|
|
419
|
-
if (filters.
|
|
422
|
+
if (filters.runAt) {
|
|
420
423
|
query += ` AND run_at = $${paramIdx++}`;
|
|
421
|
-
params.push(filters.
|
|
424
|
+
params.push(filters.runAt);
|
|
422
425
|
}
|
|
423
426
|
}
|
|
424
427
|
query += '\nRETURNING id';
|
|
@@ -447,14 +450,14 @@ export const getAllJobs = async <
|
|
|
447
450
|
const client = await pool.connect();
|
|
448
451
|
try {
|
|
449
452
|
const result = await client.query(
|
|
450
|
-
|
|
453
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason" FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
451
454
|
[limit, offset],
|
|
452
455
|
);
|
|
453
456
|
log(`Found ${result.rows.length} jobs (all)`);
|
|
454
|
-
return result.rows.map((
|
|
455
|
-
...
|
|
456
|
-
payload:
|
|
457
|
-
|
|
457
|
+
return result.rows.map((job) => ({
|
|
458
|
+
...job,
|
|
459
|
+
payload: job.payload,
|
|
460
|
+
timeoutMs: job.timeoutMs,
|
|
458
461
|
}));
|
|
459
462
|
} catch (error) {
|
|
460
463
|
log(`Error getting all jobs: ${error}`);
|
|
@@ -528,7 +531,7 @@ export const reclaimStuckJobs = async (
|
|
|
528
531
|
};
|
|
529
532
|
|
|
530
533
|
/**
|
|
531
|
-
* Get all events for a job, ordered by
|
|
534
|
+
* Get all events for a job, ordered by createdAt ascending
|
|
532
535
|
*/
|
|
533
536
|
export const getJobEvents = async (
|
|
534
537
|
pool: Pool,
|
|
@@ -537,10 +540,10 @@ export const getJobEvents = async (
|
|
|
537
540
|
const client = await pool.connect();
|
|
538
541
|
try {
|
|
539
542
|
const res = await client.query(
|
|
540
|
-
|
|
543
|
+
`SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
|
|
541
544
|
[jobId],
|
|
542
545
|
);
|
|
543
|
-
return res.rows;
|
|
546
|
+
return res.rows as JobEvent[];
|
|
544
547
|
} finally {
|
|
545
548
|
client.release();
|
|
546
549
|
}
|