@open-mercato/queue 0.6.8-develop.7016.1.4ed7b1e49d → 0.6.8-develop.7019.1.f4c01c4b5c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/factory.js +2 -1
- package/dist/factory.js.map +2 -2
- package/dist/strategies/async.js +111 -4
- package/dist/strategies/async.js.map +3 -3
- package/dist/worker/runner.js +3 -1
- package/dist/worker/runner.js.map +2 -2
- package/package.json +3 -3
- package/src/__tests__/abandoned-job.test.ts +494 -0
- package/src/__tests__/bullmq-abandoned-reasons.test.ts +42 -0
- package/src/__tests__/worker-abandoned-job.test.ts +101 -0
- package/src/factory.ts +7 -1
- package/src/strategies/async.ts +206 -2
- package/src/types.ts +47 -0
- package/src/worker/runner.ts +4 -0
|
@@ -0,0 +1,494 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { createModuleQueue } from '../factory'
|
|
6
|
+
import { ABANDONED_JOB_DRAIN_TIMEOUT_MS, ABANDONED_JOB_SWEEP_INTERVAL_MS } from '../strategies/async'
|
|
7
|
+
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
8
|
+
import type { QueuedJob } from '../types'
|
|
9
|
+
|
|
10
|
+
type WorkerListener = (...args: unknown[]) => void
|
|
11
|
+
|
|
12
|
+
let capturedProcessor: ((job: { id?: string; data: unknown; attemptsMade: number }) => Promise<void>) | null = null
|
|
13
|
+
const capturedListeners = new Map<string, WorkerListener[]>()
|
|
14
|
+
|
|
15
|
+
function emit(event: string, ...args: unknown[]): void {
|
|
16
|
+
for (const listener of capturedListeners.get(event) ?? []) listener(...args)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
20
|
+
getRedisUrlOrThrow: jest.fn(),
|
|
21
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
22
|
+
REDIS_WIRE_PROTOCOL: jest.requireActual('@open-mercato/shared/lib/redis/connection').REDIS_WIRE_PROTOCOL,
|
|
23
|
+
}))
|
|
24
|
+
|
|
25
|
+
const mockQueueGetJobs = jest.fn(async (): Promise<unknown[]> => [])
|
|
26
|
+
|
|
27
|
+
jest.mock('bullmq', () => {
|
|
28
|
+
class MockQueue<T> {
|
|
29
|
+
constructor(_name: string, _opts: unknown) {}
|
|
30
|
+
add = jest.fn(async () => ({ id: 'bull-job-id' }))
|
|
31
|
+
close = jest.fn(async () => {})
|
|
32
|
+
obliterate = jest.fn(async () => {})
|
|
33
|
+
getJobCounts = jest.fn(async () => ({ waiting: 0, active: 0, completed: 0, failed: 0 }))
|
|
34
|
+
getJobs = mockQueueGetJobs
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
class MockWorker<T> {
|
|
38
|
+
constructor(
|
|
39
|
+
_name: string,
|
|
40
|
+
processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
|
|
41
|
+
_opts: unknown,
|
|
42
|
+
) {
|
|
43
|
+
capturedProcessor = processor as (job: { id?: string; data: unknown; attemptsMade: number }) => Promise<void>
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
on = (event: string, listener: WorkerListener) => {
|
|
47
|
+
const existing = capturedListeners.get(event) ?? []
|
|
48
|
+
existing.push(listener)
|
|
49
|
+
capturedListeners.set(event, existing)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
close = jest.fn(async () => {})
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return { Queue: MockQueue, Worker: MockWorker }
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
type Payload = { runId: string }
|
|
59
|
+
|
|
60
|
+
function bullJob(id: string, payload: Payload): { id: string; data: QueuedJob<Payload>; attemptsMade: number } {
|
|
61
|
+
return {
|
|
62
|
+
id,
|
|
63
|
+
data: { id, payload, createdAt: new Date(0).toISOString() },
|
|
64
|
+
attemptsMade: 0,
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** A delivery BullMQ did not put its own id on — the payload still carries the id we minted. */
|
|
69
|
+
function bullJobWithoutId(
|
|
70
|
+
payloadId: string,
|
|
71
|
+
payload: Payload,
|
|
72
|
+
): { id?: string; data: QueuedJob<Payload>; attemptsMade: number } {
|
|
73
|
+
return {
|
|
74
|
+
id: undefined,
|
|
75
|
+
data: { id: payloadId, payload, createdAt: new Date(0).toISOString() },
|
|
76
|
+
attemptsMade: 0,
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
type FailedSetJob = {
|
|
81
|
+
id: string
|
|
82
|
+
data: QueuedJob<Payload>
|
|
83
|
+
failedReason: string
|
|
84
|
+
remove: jest.Mock
|
|
85
|
+
updateData: jest.Mock
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** A job as the sweep sees it in the failed set. `updateData` persists like the real driver's. */
|
|
89
|
+
function failedSetJob(
|
|
90
|
+
id: string,
|
|
91
|
+
payload: Payload,
|
|
92
|
+
failedReason: string,
|
|
93
|
+
metadata?: Record<string, unknown>,
|
|
94
|
+
): FailedSetJob {
|
|
95
|
+
const record: FailedSetJob = {
|
|
96
|
+
id,
|
|
97
|
+
data: { id, payload, createdAt: new Date(0).toISOString(), ...(metadata ? { metadata } : {}) },
|
|
98
|
+
failedReason,
|
|
99
|
+
remove: jest.fn(async () => {}),
|
|
100
|
+
updateData: jest.fn(async (data: QueuedJob<Payload>) => {
|
|
101
|
+
record.data = data
|
|
102
|
+
}),
|
|
103
|
+
}
|
|
104
|
+
return record
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function flushAsync(turns = 12): Promise<void> {
|
|
108
|
+
for (let index = 0; index < turns; index += 1) {
|
|
109
|
+
await Promise.resolve()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
describe('onJobAbandoned', () => {
|
|
114
|
+
const getRedisUrlOrThrowMock = getRedisUrlOrThrow as jest.MockedFunction<typeof getRedisUrlOrThrow>
|
|
115
|
+
const originalStrategy = process.env.QUEUE_STRATEGY
|
|
116
|
+
|
|
117
|
+
beforeEach(() => {
|
|
118
|
+
jest.clearAllMocks()
|
|
119
|
+
capturedProcessor = null
|
|
120
|
+
capturedListeners.clear()
|
|
121
|
+
getRedisUrlOrThrowMock.mockReturnValue('redis://localhost:6379')
|
|
122
|
+
mockQueueGetJobs.mockResolvedValue([])
|
|
123
|
+
process.env.QUEUE_STRATEGY = 'async'
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
afterEach(() => {
|
|
127
|
+
if (originalStrategy === undefined) delete process.env.QUEUE_STRATEGY
|
|
128
|
+
else process.env.QUEUE_STRATEGY = originalStrategy
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
it('fires with the job payload when the queue fails a job it never handed to the handler', async () => {
|
|
132
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
133
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
134
|
+
await queue.process(async () => {})
|
|
135
|
+
|
|
136
|
+
const job = bullJob('job-1', { runId: 'run-1' })
|
|
137
|
+
emit('failed', job, new Error('job stalled more than allowable limit'))
|
|
138
|
+
await Promise.resolve()
|
|
139
|
+
|
|
140
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
141
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(job.data, {
|
|
142
|
+
jobId: 'job-1',
|
|
143
|
+
reason: 'job stalled more than allowable limit',
|
|
144
|
+
})
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
it('does not fire when the handler ran and threw — that failure is the handler\'s own', async () => {
|
|
148
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
149
|
+
const handlerError = new Error('import batch blew up')
|
|
150
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
151
|
+
await queue.process(async () => {
|
|
152
|
+
throw handlerError
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
const job = bullJob('job-1', { runId: 'run-1' })
|
|
156
|
+
await expect(capturedProcessor!(job)).rejects.toThrow(handlerError)
|
|
157
|
+
emit('failed', job, handlerError)
|
|
158
|
+
await Promise.resolve()
|
|
159
|
+
|
|
160
|
+
expect(onJobAbandoned).not.toHaveBeenCalled()
|
|
161
|
+
})
|
|
162
|
+
|
|
163
|
+
it('fires for the abandonment reason BullMQ writes when a job outruns its started limit', async () => {
|
|
164
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
165
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
166
|
+
await queue.process(async () => {})
|
|
167
|
+
|
|
168
|
+
const job = bullJob('job-1', { runId: 'run-1' })
|
|
169
|
+
emit('failed', job, new Error('job started more than allowable limit'))
|
|
170
|
+
await Promise.resolve()
|
|
171
|
+
|
|
172
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(job.data, {
|
|
173
|
+
jobId: 'job-1',
|
|
174
|
+
reason: 'job started more than allowable limit',
|
|
175
|
+
})
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
it('fires when the same worker earlier ran an attempt of the job that stalled out', async () => {
|
|
179
|
+
// The zombie case: this process entered the processor for an attempt that hung past the lock
|
|
180
|
+
// duration, and the final, abandoned delivery lands back in the same process. Classification
|
|
181
|
+
// must not depend on what this worker remembers doing.
|
|
182
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
183
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned, concurrency: 2 })
|
|
184
|
+
let releaseZombie = () => {}
|
|
185
|
+
await queue.process(async () => {
|
|
186
|
+
await new Promise<void>((resolve) => {
|
|
187
|
+
releaseZombie = resolve
|
|
188
|
+
})
|
|
189
|
+
})
|
|
190
|
+
|
|
191
|
+
const job = bullJob('job-1', { runId: 'run-1' })
|
|
192
|
+
void capturedProcessor!(job) // still hanging, exactly like the stalled attempt
|
|
193
|
+
emit('failed', job, new Error('job stalled more than allowable limit'))
|
|
194
|
+
await Promise.resolve()
|
|
195
|
+
|
|
196
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
197
|
+
releaseZombie()
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
it('reports each abandoned job when several failures arrive together', async () => {
|
|
201
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
202
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned, concurrency: 2 })
|
|
203
|
+
await queue.process(async () => {})
|
|
204
|
+
|
|
205
|
+
const handlerFailure = bullJob('job-threw', { runId: 'run-1' })
|
|
206
|
+
const abandoned = bullJob('job-abandoned', { runId: 'run-2' })
|
|
207
|
+
emit('failed', handlerFailure, new Error('import batch blew up'))
|
|
208
|
+
emit('failed', abandoned, new Error('job stalled more than allowable limit'))
|
|
209
|
+
await Promise.resolve()
|
|
210
|
+
|
|
211
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
212
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(abandoned.data, {
|
|
213
|
+
jobId: 'job-abandoned',
|
|
214
|
+
reason: 'job stalled more than allowable limit',
|
|
215
|
+
})
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
it('reports an abandoned job that carries no job id of its own', async () => {
|
|
219
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
220
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
221
|
+
await queue.process(async () => {})
|
|
222
|
+
|
|
223
|
+
const job = bullJobWithoutId('payload-1', { runId: 'run-1' })
|
|
224
|
+
emit('failed', job, new Error('job stalled more than allowable limit'))
|
|
225
|
+
await Promise.resolve()
|
|
226
|
+
|
|
227
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(job.data, {
|
|
228
|
+
jobId: null,
|
|
229
|
+
reason: 'job stalled more than allowable limit',
|
|
230
|
+
})
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
it('stays quiet when the queue cannot produce the job at all', async () => {
|
|
234
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
235
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
236
|
+
await queue.process(async () => {})
|
|
237
|
+
|
|
238
|
+
emit('failed', undefined, new Error('job stalled more than allowable limit'))
|
|
239
|
+
emit('failed', { id: 'job-1' }, new Error('job stalled more than allowable limit'))
|
|
240
|
+
await Promise.resolve()
|
|
241
|
+
|
|
242
|
+
expect(onJobAbandoned).not.toHaveBeenCalled()
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
it('drains an in-flight report on close so a shutdown cannot truncate a repair', async () => {
|
|
246
|
+
let finishReport = () => {}
|
|
247
|
+
const reportFinished = jest.fn()
|
|
248
|
+
const onJobAbandoned = jest.fn(
|
|
249
|
+
() =>
|
|
250
|
+
new Promise<void>((resolve) => {
|
|
251
|
+
finishReport = () => {
|
|
252
|
+
reportFinished()
|
|
253
|
+
resolve()
|
|
254
|
+
}
|
|
255
|
+
}),
|
|
256
|
+
)
|
|
257
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
258
|
+
await queue.process(async () => {})
|
|
259
|
+
|
|
260
|
+
emit('failed', bullJob('job-1', { runId: 'run-1' }), new Error('job stalled more than allowable limit'))
|
|
261
|
+
await Promise.resolve()
|
|
262
|
+
expect(reportFinished).not.toHaveBeenCalled()
|
|
263
|
+
|
|
264
|
+
const closed = queue.close()
|
|
265
|
+
let closedEarly = false
|
|
266
|
+
void closed.then(() => {
|
|
267
|
+
closedEarly = !reportFinished.mock.calls.length
|
|
268
|
+
})
|
|
269
|
+
await Promise.resolve()
|
|
270
|
+
expect(closedEarly).toBe(false)
|
|
271
|
+
|
|
272
|
+
finishReport()
|
|
273
|
+
await closed
|
|
274
|
+
expect(reportFinished).toHaveBeenCalledTimes(1)
|
|
275
|
+
})
|
|
276
|
+
|
|
277
|
+
it('swallows a throwing hook so the reporting cannot kill the worker', async () => {
|
|
278
|
+
const onJobAbandoned = jest.fn(async () => {
|
|
279
|
+
throw new Error('reporting failed')
|
|
280
|
+
})
|
|
281
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
282
|
+
await queue.process(async () => {})
|
|
283
|
+
|
|
284
|
+
const job = bullJob('job-1', { runId: 'run-1' })
|
|
285
|
+
expect(() => emit('failed', job, new Error('job stalled more than allowable limit'))).not.toThrow()
|
|
286
|
+
await Promise.resolve()
|
|
287
|
+
await Promise.resolve()
|
|
288
|
+
|
|
289
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
290
|
+
})
|
|
291
|
+
|
|
292
|
+
it('sweeps the failed set on worker start and delivers reports nobody was alive to send', async () => {
|
|
293
|
+
const abandoned = failedSetJob('job-a', { runId: 'run-a' }, 'job stalled more than allowable limit')
|
|
294
|
+
const handlerFailure = failedSetJob('job-b', { runId: 'run-b' }, 'import batch blew up')
|
|
295
|
+
const alreadyReported = failedSetJob('job-c', { runId: 'run-c' }, 'job stalled more than allowable limit', {
|
|
296
|
+
abandonReportedAt: '2026-01-01T00:00:00.000Z',
|
|
297
|
+
})
|
|
298
|
+
mockQueueGetJobs.mockResolvedValue([abandoned, handlerFailure, alreadyReported])
|
|
299
|
+
|
|
300
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
301
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
302
|
+
await queue.process(async () => {})
|
|
303
|
+
await flushAsync()
|
|
304
|
+
|
|
305
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
306
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(expect.objectContaining({ id: 'job-a' }), {
|
|
307
|
+
jobId: 'job-a',
|
|
308
|
+
reason: 'job stalled more than allowable limit',
|
|
309
|
+
})
|
|
310
|
+
expect(abandoned.data.metadata?.abandonReportedAt).toEqual(expect.any(String))
|
|
311
|
+
expect(handlerFailure.updateData).not.toHaveBeenCalled()
|
|
312
|
+
expect(alreadyReported.updateData).not.toHaveBeenCalled()
|
|
313
|
+
expect(abandoned.remove).not.toHaveBeenCalled()
|
|
314
|
+
await queue.close()
|
|
315
|
+
})
|
|
316
|
+
|
|
317
|
+
it('retries an undelivered report on the next sweep instead of losing it', async () => {
|
|
318
|
+
jest.useFakeTimers()
|
|
319
|
+
try {
|
|
320
|
+
const abandoned = failedSetJob('job-a', { runId: 'run-a' }, 'job stalled more than allowable limit')
|
|
321
|
+
mockQueueGetJobs.mockResolvedValue([abandoned])
|
|
322
|
+
|
|
323
|
+
const onJobAbandoned = jest
|
|
324
|
+
.fn<Promise<void>, [unknown, unknown]>()
|
|
325
|
+
.mockRejectedValueOnce(new Error('db down'))
|
|
326
|
+
.mockResolvedValue(undefined)
|
|
327
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
328
|
+
await queue.process(async () => {})
|
|
329
|
+
await flushAsync()
|
|
330
|
+
|
|
331
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
332
|
+
expect(abandoned.updateData).not.toHaveBeenCalled()
|
|
333
|
+
|
|
334
|
+
jest.advanceTimersByTime(ABANDONED_JOB_SWEEP_INTERVAL_MS)
|
|
335
|
+
await flushAsync()
|
|
336
|
+
|
|
337
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(2)
|
|
338
|
+
expect(abandoned.updateData).toHaveBeenCalledTimes(1)
|
|
339
|
+
await queue.close()
|
|
340
|
+
} finally {
|
|
341
|
+
jest.useRealTimers()
|
|
342
|
+
}
|
|
343
|
+
})
|
|
344
|
+
|
|
345
|
+
it('acknowledges a delivered report so later sweeps do not repeat it', async () => {
|
|
346
|
+
jest.useFakeTimers()
|
|
347
|
+
try {
|
|
348
|
+
const job = failedSetJob('job-1', { runId: 'run-1' }, 'job stalled more than allowable limit')
|
|
349
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
350
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
351
|
+
await queue.process(async () => {})
|
|
352
|
+
await flushAsync()
|
|
353
|
+
|
|
354
|
+
emit('failed', job, new Error('job stalled more than allowable limit'))
|
|
355
|
+
await flushAsync()
|
|
356
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
357
|
+
expect(job.data.metadata?.abandonReportedAt).toEqual(expect.any(String))
|
|
358
|
+
|
|
359
|
+
mockQueueGetJobs.mockResolvedValue([job])
|
|
360
|
+
jest.advanceTimersByTime(ABANDONED_JOB_SWEEP_INTERVAL_MS)
|
|
361
|
+
await flushAsync()
|
|
362
|
+
|
|
363
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
364
|
+
await queue.close()
|
|
365
|
+
} finally {
|
|
366
|
+
jest.useRealTimers()
|
|
367
|
+
}
|
|
368
|
+
})
|
|
369
|
+
|
|
370
|
+
it('does not report a job twice while its first report is still in flight', async () => {
|
|
371
|
+
const job = failedSetJob('job-1', { runId: 'run-1' }, 'job stalled more than allowable limit')
|
|
372
|
+
mockQueueGetJobs.mockResolvedValue([job])
|
|
373
|
+
|
|
374
|
+
let finishReport = () => {}
|
|
375
|
+
const onJobAbandoned = jest.fn(
|
|
376
|
+
() =>
|
|
377
|
+
new Promise<void>((resolve) => {
|
|
378
|
+
finishReport = resolve
|
|
379
|
+
}),
|
|
380
|
+
)
|
|
381
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
382
|
+
await queue.process(async () => {})
|
|
383
|
+
await flushAsync()
|
|
384
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
385
|
+
|
|
386
|
+
emit('failed', job, new Error('job stalled more than allowable limit'))
|
|
387
|
+
await flushAsync()
|
|
388
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
389
|
+
|
|
390
|
+
finishReport()
|
|
391
|
+
await flushAsync()
|
|
392
|
+
expect(job.updateData).toHaveBeenCalledTimes(1)
|
|
393
|
+
await queue.close()
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
it('gives up on a hanging report at shutdown instead of blocking it forever', async () => {
|
|
397
|
+
jest.useFakeTimers()
|
|
398
|
+
try {
|
|
399
|
+
const onJobAbandoned = jest.fn(() => new Promise<void>(() => {})) // never settles
|
|
400
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
401
|
+
await queue.process(async () => {})
|
|
402
|
+
|
|
403
|
+
emit('failed', bullJob('job-1', { runId: 'run-1' }), new Error('job stalled more than allowable limit'))
|
|
404
|
+
await flushAsync()
|
|
405
|
+
expect(onJobAbandoned).toHaveBeenCalledTimes(1)
|
|
406
|
+
|
|
407
|
+
let closed = false
|
|
408
|
+
const closePromise = queue.close().then(() => {
|
|
409
|
+
closed = true
|
|
410
|
+
})
|
|
411
|
+
await flushAsync()
|
|
412
|
+
expect(closed).toBe(false)
|
|
413
|
+
|
|
414
|
+
jest.advanceTimersByTime(ABANDONED_JOB_DRAIN_TIMEOUT_MS)
|
|
415
|
+
await closePromise
|
|
416
|
+
expect(closed).toBe(true)
|
|
417
|
+
} finally {
|
|
418
|
+
jest.useRealTimers()
|
|
419
|
+
}
|
|
420
|
+
})
|
|
421
|
+
|
|
422
|
+
it('does not start a report from a sweep that was already running when close began', async () => {
|
|
423
|
+
let releaseGetJobs = (_jobs: unknown[]) => {}
|
|
424
|
+
mockQueueGetJobs.mockImplementation(
|
|
425
|
+
() =>
|
|
426
|
+
new Promise((resolve) => {
|
|
427
|
+
releaseGetJobs = resolve as (jobs: unknown[]) => void
|
|
428
|
+
}),
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
432
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
433
|
+
await queue.process(async () => {})
|
|
434
|
+
await flushAsync()
|
|
435
|
+
|
|
436
|
+
// The start-up sweep is parked inside getJobs; shutdown begins before it returns.
|
|
437
|
+
const closePromise = queue.close()
|
|
438
|
+
releaseGetJobs([failedSetJob('job-1', { runId: 'run-1' }, 'job stalled more than allowable limit')])
|
|
439
|
+
await flushAsync()
|
|
440
|
+
await closePromise
|
|
441
|
+
|
|
442
|
+
expect(onJobAbandoned).not.toHaveBeenCalled()
|
|
443
|
+
})
|
|
444
|
+
|
|
445
|
+
it('honours QUEUE_ABANDONED_SWEEP_INTERVAL_MS for the sweep cadence', async () => {
|
|
446
|
+
jest.useFakeTimers()
|
|
447
|
+
process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS = '1000'
|
|
448
|
+
try {
|
|
449
|
+
mockQueueGetJobs.mockResolvedValue([])
|
|
450
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
451
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
452
|
+
await queue.process(async () => {})
|
|
453
|
+
await flushAsync()
|
|
454
|
+
expect(mockQueueGetJobs).toHaveBeenCalledTimes(1)
|
|
455
|
+
|
|
456
|
+
jest.advanceTimersByTime(1000)
|
|
457
|
+
await flushAsync()
|
|
458
|
+
|
|
459
|
+
expect(mockQueueGetJobs).toHaveBeenCalledTimes(2)
|
|
460
|
+
await queue.close()
|
|
461
|
+
} finally {
|
|
462
|
+
delete process.env.QUEUE_ABANDONED_SWEEP_INTERVAL_MS
|
|
463
|
+
jest.useRealTimers()
|
|
464
|
+
}
|
|
465
|
+
})
|
|
466
|
+
|
|
467
|
+
it('is not forwarded to the local strategy, which cannot abandon a job', async () => {
|
|
468
|
+
process.env.QUEUE_STRATEGY = 'local'
|
|
469
|
+
const originalCwd = process.cwd()
|
|
470
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-abandoned-'))
|
|
471
|
+
process.chdir(tmp)
|
|
472
|
+
|
|
473
|
+
try {
|
|
474
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
475
|
+
const queue = createModuleQueue<Payload>('test-queue', { onJobAbandoned })
|
|
476
|
+
expect(queue.strategy).toBe('local')
|
|
477
|
+
|
|
478
|
+
await queue.enqueue({ runId: 'run-1' })
|
|
479
|
+
const result = await queue.process(
|
|
480
|
+
async () => {
|
|
481
|
+
throw new Error('handler failed')
|
|
482
|
+
},
|
|
483
|
+
{ limit: 1 },
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
expect(result.failed).toBe(1)
|
|
487
|
+
expect(onJobAbandoned).not.toHaveBeenCalled()
|
|
488
|
+
await queue.close()
|
|
489
|
+
} finally {
|
|
490
|
+
process.chdir(originalCwd)
|
|
491
|
+
fs.rmSync(tmp, { recursive: true, force: true })
|
|
492
|
+
}
|
|
493
|
+
})
|
|
494
|
+
})
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { ABANDONED_JOB_REASONS } from '../strategies/async'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* `onJobAbandoned` classifies a failure by matching the reason BullMQ records when it destroys a job
|
|
8
|
+
* without calling the processor. That is a string comparison against another package's internals, so
|
|
9
|
+
* an upgrade could rename one and silently switch the hook off — the queue would go back to failing
|
|
10
|
+
* jobs nobody hears about, with every test still green.
|
|
11
|
+
*
|
|
12
|
+
* This asserts the strings are still there. If it fails after a BullMQ bump, read the new source and
|
|
13
|
+
* update `ABANDONED_JOB_REASONS` — do not delete the assertion.
|
|
14
|
+
*/
|
|
15
|
+
function readBullmqSource(): string {
|
|
16
|
+
const entry = require.resolve('bullmq')
|
|
17
|
+
const root = path.join(entry.slice(0, entry.lastIndexOf(`${path.sep}dist${path.sep}`)), 'dist')
|
|
18
|
+
const chunks: string[] = []
|
|
19
|
+
|
|
20
|
+
const walk = (dir: string) => {
|
|
21
|
+
for (const item of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
22
|
+
const full = path.join(dir, item.name)
|
|
23
|
+
if (item.isDirectory()) walk(full)
|
|
24
|
+
else if (item.name.endsWith('.js') || item.name.endsWith('.lua')) chunks.push(fs.readFileSync(full, 'utf8'))
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
walk(root)
|
|
29
|
+
return chunks.join('\n')
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe('BullMQ abandonment reasons', () => {
|
|
33
|
+
const source = readBullmqSource()
|
|
34
|
+
|
|
35
|
+
it('reads a non-empty BullMQ source tree', () => {
|
|
36
|
+
expect(source.length).toBeGreaterThan(0)
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
it.each(ABANDONED_JOB_REASONS)('still emits %p', (reason) => {
|
|
40
|
+
expect(source).toContain(reason)
|
|
41
|
+
})
|
|
42
|
+
})
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { runWorker } from '../worker/runner'
|
|
2
|
+
import type { WorkerDescriptor } from '../types'
|
|
3
|
+
|
|
4
|
+
type WorkerListener = (...args: unknown[]) => void
|
|
5
|
+
|
|
6
|
+
const capturedListeners = new Map<string, WorkerListener[]>()
|
|
7
|
+
|
|
8
|
+
function emit(event: string, ...args: unknown[]): void {
|
|
9
|
+
for (const listener of capturedListeners.get(event) ?? []) listener(...args)
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
13
|
+
getRedisUrlOrThrow: jest.fn(() => 'redis://localhost:6379'),
|
|
14
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
15
|
+
REDIS_WIRE_PROTOCOL: jest.requireActual('@open-mercato/shared/lib/redis/connection').REDIS_WIRE_PROTOCOL,
|
|
16
|
+
}))
|
|
17
|
+
|
|
18
|
+
jest.mock('bullmq', () => {
|
|
19
|
+
class MockQueue<T> {
|
|
20
|
+
constructor(_name: string, _opts: unknown) {}
|
|
21
|
+
add = jest.fn(async () => ({ id: 'bull-job-id' }))
|
|
22
|
+
close = jest.fn(async () => {})
|
|
23
|
+
obliterate = jest.fn(async () => {})
|
|
24
|
+
getJobCounts = jest.fn(async () => ({ waiting: 0, active: 0, completed: 0, failed: 0 }))
|
|
25
|
+
getJobs = jest.fn(async () => [])
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
class MockWorker<T> {
|
|
29
|
+
constructor(_name: string, _processor: unknown, _opts: unknown) {}
|
|
30
|
+
on = (event: string, listener: WorkerListener) => {
|
|
31
|
+
const existing = capturedListeners.get(event) ?? []
|
|
32
|
+
existing.push(listener)
|
|
33
|
+
capturedListeners.set(event, existing)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
close = jest.fn(async () => {})
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return { Queue: MockQueue, Worker: MockWorker }
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Reachability, not wiring.
|
|
44
|
+
*
|
|
45
|
+
* The queue that runs jobs is built by `runWorker`, not by whoever enqueues them, so a callback
|
|
46
|
+
* attached to the enqueueing instance is never installed on the consumer — the failure this guards
|
|
47
|
+
* against is invisible to any test that asserts the option was *passed* somewhere. This one goes
|
|
48
|
+
* through the path `worker --all` uses and asserts the callback actually fires.
|
|
49
|
+
*/
|
|
50
|
+
describe('runWorker — abandoned-job reporting', () => {
|
|
51
|
+
beforeEach(() => {
|
|
52
|
+
capturedListeners.clear()
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
it('installs a worker descriptor\'s onJobAbandoned on the queue it builds', async () => {
|
|
56
|
+
const onJobAbandoned = jest.fn(async () => {})
|
|
57
|
+
const descriptor: WorkerDescriptor = {
|
|
58
|
+
id: 'test:worker',
|
|
59
|
+
queue: 'test-queue',
|
|
60
|
+
concurrency: 1,
|
|
61
|
+
handler: async () => {},
|
|
62
|
+
onJobAbandoned,
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
await runWorker({
|
|
66
|
+
queueName: descriptor.queue,
|
|
67
|
+
handler: descriptor.handler,
|
|
68
|
+
concurrency: descriptor.concurrency,
|
|
69
|
+
onJobAbandoned: descriptor.onJobAbandoned,
|
|
70
|
+
strategy: 'async',
|
|
71
|
+
gracefulShutdown: false,
|
|
72
|
+
background: true,
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
const payload = { id: 'job-1', payload: { runId: 'run-1' }, createdAt: new Date(0).toISOString() }
|
|
76
|
+
emit('failed', { id: 'job-1', data: payload }, new Error('job stalled more than allowable limit'))
|
|
77
|
+
await Promise.resolve()
|
|
78
|
+
await Promise.resolve()
|
|
79
|
+
|
|
80
|
+
expect(onJobAbandoned).toHaveBeenCalledWith(payload, {
|
|
81
|
+
jobId: 'job-1',
|
|
82
|
+
reason: 'job stalled more than allowable limit',
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
it('leaves the queue without one when no worker declares it', async () => {
|
|
87
|
+
await runWorker({
|
|
88
|
+
queueName: 'test-queue',
|
|
89
|
+
handler: async () => {},
|
|
90
|
+
strategy: 'async',
|
|
91
|
+
gracefulShutdown: false,
|
|
92
|
+
background: true,
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
// Nothing to assert a call against; the guarantee is that the 'failed' path stays inert, which
|
|
96
|
+
// would otherwise show up as a thrown error inside the listener.
|
|
97
|
+
expect(() =>
|
|
98
|
+
emit('failed', { id: 'job-1', data: { id: 'job-1', payload: {}, createdAt: '' } }, new Error('job stalled more than allowable limit')),
|
|
99
|
+
).not.toThrow()
|
|
100
|
+
})
|
|
101
|
+
})
|
package/src/factory.ts
CHANGED
|
@@ -82,7 +82,10 @@ export function resolveQueueStrategy(): QueueStrategyType {
|
|
|
82
82
|
*/
|
|
83
83
|
export function createModuleQueue<T = unknown>(
|
|
84
84
|
name: string,
|
|
85
|
-
options?: Pick<
|
|
85
|
+
options?: Pick<
|
|
86
|
+
AsyncQueueOptions,
|
|
87
|
+
'attempts' | 'concurrency' | 'lockDuration' | 'maxStalledCount' | 'onJobAbandoned'
|
|
88
|
+
>,
|
|
86
89
|
): Queue<T> {
|
|
87
90
|
const strategy = resolveQueueStrategy()
|
|
88
91
|
if (strategy === 'async') {
|
|
@@ -92,7 +95,10 @@ export function createModuleQueue<T = unknown>(
|
|
|
92
95
|
attempts: options?.attempts,
|
|
93
96
|
lockDuration: options?.lockDuration,
|
|
94
97
|
maxStalledCount: options?.maxStalledCount,
|
|
98
|
+
onJobAbandoned: options?.onJobAbandoned,
|
|
95
99
|
})
|
|
96
100
|
}
|
|
101
|
+
// The local strategy runs the handler in-process, so there is no queue that could outlive it and
|
|
102
|
+
// abandon a job — `onJobAbandoned` has nothing to report and is deliberately not forwarded.
|
|
97
103
|
return createLocalQueue<T>(name, { concurrency: options?.concurrency })
|
|
98
104
|
}
|