@open-mercato/scheduler 0.9.0 → 0.9.1-develop.7459.1.99e3accd30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/scheduler/api/jobs/[id]/executions/route.js +1 -1
- package/dist/modules/scheduler/api/jobs/[id]/executions/route.js.map +2 -2
- package/dist/modules/scheduler/api/jobs/route.js +1 -1
- package/dist/modules/scheduler/api/jobs/route.js.map +2 -2
- package/dist/modules/scheduler/cli.js +28 -6
- package/dist/modules/scheduler/cli.js.map +2 -2
- package/dist/modules/scheduler/data/validators.js +5 -5
- package/dist/modules/scheduler/data/validators.js.map +2 -2
- package/dist/modules/scheduler/lib/bullmqNextRun.js +61 -0
- package/dist/modules/scheduler/lib/bullmqNextRun.js.map +7 -0
- package/dist/modules/scheduler/lib/nextRunSync.js +80 -0
- package/dist/modules/scheduler/lib/nextRunSync.js.map +7 -0
- package/dist/modules/scheduler/workers/execute-schedule.worker.js +20 -0
- package/dist/modules/scheduler/workers/execute-schedule.worker.js.map +2 -2
- package/package.json +7 -6
- package/src/modules/scheduler/__tests__/cli.run.test.ts +135 -0
- package/src/modules/scheduler/api/jobs/[id]/executions/route.ts +1 -1
- package/src/modules/scheduler/api/jobs/route.ts +1 -1
- package/src/modules/scheduler/api/trigger/__tests__/route.test.ts +15 -0
- package/src/modules/scheduler/cli.ts +33 -7
- package/src/modules/scheduler/data/__tests__/validators.test.ts +29 -0
- package/src/modules/scheduler/data/validators.ts +5 -5
- package/src/modules/scheduler/lib/__tests__/bullmqNextRun.test.ts +200 -0
- package/src/modules/scheduler/lib/__tests__/nextRunSync.test.ts +222 -0
- package/src/modules/scheduler/lib/bullmqNextRun.ts +112 -0
- package/src/modules/scheduler/lib/nextRunSync.ts +131 -0
- package/src/modules/scheduler/workers/__tests__/execute-schedule.next-run.test.ts +468 -0
- package/src/modules/scheduler/workers/__tests__/execute-schedule.worker.test.ts +4 -0
- package/src/modules/scheduler/workers/execute-schedule.worker.ts +35 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { readBullmqNextRun } from '../bullmqNextRun'
|
|
2
|
+
|
|
3
|
+
const mockQueue = {
|
|
4
|
+
getJobScheduler: jest.fn(),
|
|
5
|
+
on: jest.fn(),
|
|
6
|
+
close: jest.fn(),
|
|
7
|
+
disconnect: jest.fn(),
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
let queueWithoutJobSchedulerLookup = false
|
|
11
|
+
|
|
12
|
+
const mockQueueConstructor = jest.fn(() => (
|
|
13
|
+
queueWithoutJobSchedulerLookup
|
|
14
|
+
? { on: mockQueue.on, close: mockQueue.close, disconnect: mockQueue.disconnect }
|
|
15
|
+
: mockQueue
|
|
16
|
+
))
|
|
17
|
+
|
|
18
|
+
jest.mock('bullmq', () => ({
|
|
19
|
+
Queue: mockQueueConstructor,
|
|
20
|
+
}))
|
|
21
|
+
|
|
22
|
+
const mockGetRedisUrlOrThrow = jest.fn(() => 'redis://localhost:6379')
|
|
23
|
+
|
|
24
|
+
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
25
|
+
getRedisUrlOrThrow: () => mockGetRedisUrlOrThrow(),
|
|
26
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
27
|
+
}))
|
|
28
|
+
|
|
29
|
+
const scheduleId = '11111111-1111-4111-8111-111111111111'
|
|
30
|
+
const nextMillis = Date.UTC(2030, 0, 1, 6, 0, 0)
|
|
31
|
+
|
|
32
|
+
describe('readBullmqNextRun', () => {
|
|
33
|
+
beforeEach(() => {
|
|
34
|
+
jest.clearAllMocks()
|
|
35
|
+
queueWithoutJobSchedulerLookup = false
|
|
36
|
+
mockQueue.on.mockReset()
|
|
37
|
+
mockQueue.close.mockResolvedValue(undefined)
|
|
38
|
+
mockQueue.disconnect.mockResolvedValue(undefined)
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
it('returns the scheduler next slot with its timing configuration and closes the queue', async () => {
|
|
42
|
+
mockQueue.getJobScheduler.mockResolvedValue({
|
|
43
|
+
key: `schedule-${scheduleId}`,
|
|
44
|
+
name: `schedule-${scheduleId}`,
|
|
45
|
+
next: nextMillis,
|
|
46
|
+
pattern: '0 6 * * *',
|
|
47
|
+
tz: 'Europe/Warsaw',
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toEqual({
|
|
51
|
+
nextRunAt: new Date(nextMillis),
|
|
52
|
+
pattern: '0 6 * * *',
|
|
53
|
+
timezone: 'Europe/Warsaw',
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
expect(mockQueueConstructor).toHaveBeenCalledWith('scheduler-execution', expect.objectContaining({
|
|
57
|
+
connection: expect.objectContaining({ host: 'localhost', port: 6379 }),
|
|
58
|
+
}))
|
|
59
|
+
expect(mockQueue.getJobScheduler).toHaveBeenCalledWith(`schedule-${scheduleId}`)
|
|
60
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
61
|
+
expect(mockQueue.disconnect).not.toHaveBeenCalled()
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('returns the interval of an every-based scheduler', async () => {
|
|
65
|
+
mockQueue.getJobScheduler.mockResolvedValue({ next: nextMillis, every: 900000, tz: 'UTC' })
|
|
66
|
+
|
|
67
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toEqual({
|
|
68
|
+
nextRunAt: new Date(nextMillis),
|
|
69
|
+
every: 900000,
|
|
70
|
+
timezone: 'UTC',
|
|
71
|
+
})
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
it('returns null and still closes the queue when the scheduler does not exist', async () => {
|
|
75
|
+
mockQueue.getJobScheduler.mockResolvedValue(undefined)
|
|
76
|
+
|
|
77
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toBeNull()
|
|
78
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
it('returns null when the scheduler has no next slot', async () => {
|
|
82
|
+
mockQueue.getJobScheduler.mockResolvedValue({ next: null, pattern: '0 6 * * *' })
|
|
83
|
+
|
|
84
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toBeNull()
|
|
85
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it('returns null and closes the queue when the installed BullMQ has no getJobScheduler', async () => {
|
|
89
|
+
queueWithoutJobSchedulerLookup = true
|
|
90
|
+
|
|
91
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toBeNull()
|
|
92
|
+
expect(mockQueue.getJobScheduler).not.toHaveBeenCalled()
|
|
93
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
it('closes the queue and rethrows when the read fails', async () => {
|
|
97
|
+
mockQueue.getJobScheduler.mockRejectedValue(new Error('redis read failed'))
|
|
98
|
+
|
|
99
|
+
await expect(readBullmqNextRun(scheduleId)).rejects.toThrow('redis read failed')
|
|
100
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
101
|
+
expect(mockQueue.disconnect).not.toHaveBeenCalled()
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
it('leaves no timer behind after a completed read', async () => {
|
|
105
|
+
jest.useFakeTimers()
|
|
106
|
+
try {
|
|
107
|
+
mockQueue.getJobScheduler.mockResolvedValue({ next: nextMillis, pattern: '0 6 * * *', tz: 'UTC' })
|
|
108
|
+
|
|
109
|
+
await readBullmqNextRun(scheduleId)
|
|
110
|
+
|
|
111
|
+
expect(jest.getTimerCount()).toBe(0)
|
|
112
|
+
} finally {
|
|
113
|
+
jest.useRealTimers()
|
|
114
|
+
}
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
it('rejects at the deadline and disconnects when the read never settles', async () => {
|
|
118
|
+
mockQueue.getJobScheduler.mockReturnValue(new Promise(() => undefined))
|
|
119
|
+
|
|
120
|
+
await expect(readBullmqNextRun(scheduleId, 20)).rejects.toThrow('Timed out after 20ms')
|
|
121
|
+
expect(mockQueue.disconnect).toHaveBeenCalledTimes(1)
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
it('rejects at the deadline when close hangs after a successful read', async () => {
|
|
125
|
+
mockQueue.getJobScheduler.mockResolvedValue({ next: nextMillis, pattern: '0 6 * * *' })
|
|
126
|
+
mockQueue.close.mockReturnValue(new Promise(() => undefined))
|
|
127
|
+
|
|
128
|
+
await expect(readBullmqNextRun(scheduleId, 20)).rejects.toThrow('Timed out after 20ms')
|
|
129
|
+
expect(mockQueue.disconnect).toHaveBeenCalledTimes(1)
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
it('keeps the value it read when close fails', async () => {
|
|
133
|
+
mockQueue.getJobScheduler.mockResolvedValue({ next: nextMillis, pattern: '0 6 * * *', tz: 'UTC' })
|
|
134
|
+
mockQueue.close.mockRejectedValue(new Error('close failed'))
|
|
135
|
+
|
|
136
|
+
await expect(readBullmqNextRun(scheduleId)).resolves.toEqual({
|
|
137
|
+
nextRunAt: new Date(nextMillis),
|
|
138
|
+
pattern: '0 6 * * *',
|
|
139
|
+
timezone: 'UTC',
|
|
140
|
+
})
|
|
141
|
+
})
|
|
142
|
+
|
|
143
|
+
it('rethrows the read error, not the close error, when both fail', async () => {
|
|
144
|
+
mockQueue.getJobScheduler.mockRejectedValue(new Error('redis read failed'))
|
|
145
|
+
mockQueue.close.mockRejectedValue(new Error('close failed'))
|
|
146
|
+
|
|
147
|
+
await expect(readBullmqNextRun(scheduleId)).rejects.toThrow('redis read failed')
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
it('tears the queue down with both close and disconnect at the deadline and handles their rejections', async () => {
|
|
151
|
+
mockQueue.getJobScheduler.mockReturnValue(new Promise(() => undefined))
|
|
152
|
+
let failClose: (error: Error) => void = () => undefined
|
|
153
|
+
let failDisconnect: (error: Error) => void = () => undefined
|
|
154
|
+
const pendingClose = new Promise<void>((_resolve, reject) => { failClose = reject })
|
|
155
|
+
const pendingDisconnect = new Promise<void>((_resolve, reject) => { failDisconnect = reject })
|
|
156
|
+
const closeHandled = jest.spyOn(pendingClose, 'catch')
|
|
157
|
+
const disconnectHandled = jest.spyOn(pendingDisconnect, 'catch')
|
|
158
|
+
mockQueue.close.mockReturnValue(pendingClose)
|
|
159
|
+
mockQueue.disconnect.mockReturnValue(pendingDisconnect)
|
|
160
|
+
|
|
161
|
+
await expect(readBullmqNextRun(scheduleId, 20)).rejects.toThrow('Timed out after 20ms')
|
|
162
|
+
|
|
163
|
+
expect(mockQueue.close).toHaveBeenCalledTimes(1)
|
|
164
|
+
expect(mockQueue.disconnect).toHaveBeenCalledTimes(1)
|
|
165
|
+
expect(closeHandled).toHaveBeenCalledTimes(1)
|
|
166
|
+
expect(disconnectHandled).toHaveBeenCalledTimes(1)
|
|
167
|
+
failClose(new Error('close failed'))
|
|
168
|
+
failDisconnect(new Error('disconnect failed'))
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
it('attaches the last connection error BullMQ emitted as the cause of the timeout', async () => {
|
|
172
|
+
mockQueue.getJobScheduler.mockReturnValue(new Promise(() => undefined))
|
|
173
|
+
const connectionError = new Error('connect ECONNREFUSED 127.0.0.1:6379')
|
|
174
|
+
mockQueue.on.mockImplementation((_event: string, listener: (error: Error) => void) => {
|
|
175
|
+
listener(connectionError)
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
await expect(readBullmqNextRun(scheduleId, 20)).rejects.toMatchObject({
|
|
179
|
+
message: expect.stringContaining('Timed out after 20ms'),
|
|
180
|
+
cause: connectionError,
|
|
181
|
+
})
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
it('registers an error listener so BullMQ does not print connection errors itself', async () => {
|
|
185
|
+
mockQueue.getJobScheduler.mockResolvedValue(undefined)
|
|
186
|
+
|
|
187
|
+
await readBullmqNextRun(scheduleId)
|
|
188
|
+
|
|
189
|
+
expect(mockQueue.on).toHaveBeenCalledWith('error', expect.any(Function))
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
it('rejects without opening a queue when Redis is not configured', async () => {
|
|
193
|
+
mockGetRedisUrlOrThrow.mockImplementationOnce(() => {
|
|
194
|
+
throw new Error('Redis URL is not configured')
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
await expect(readBullmqNextRun(scheduleId)).rejects.toThrow('Redis URL is not configured')
|
|
198
|
+
expect(mockQueueConstructor).not.toHaveBeenCalled()
|
|
199
|
+
})
|
|
200
|
+
})
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import type { EntityManager } from '@mikro-orm/core'
|
|
2
|
+
import { bullmqNextRunMatchesTiming, syncScheduleNextRunAt, type ScheduleTimingSnapshot } from '../nextRunSync'
|
|
3
|
+
|
|
4
|
+
const mockReadBullmqNextRun = jest.fn()
|
|
5
|
+
|
|
6
|
+
jest.mock('../bullmqNextRun', () => ({
|
|
7
|
+
readBullmqNextRun: (...args: unknown[]) => mockReadBullmqNextRun(...args),
|
|
8
|
+
}))
|
|
9
|
+
|
|
10
|
+
const mockReportError = jest.fn()
|
|
11
|
+
let telemetryRuntime: { reportError: jest.Mock } | null = null
|
|
12
|
+
|
|
13
|
+
jest.mock('@open-mercato/shared/lib/telemetry/runtime', () => ({
|
|
14
|
+
getTelemetryRuntime: () => telemetryRuntime,
|
|
15
|
+
}))
|
|
16
|
+
|
|
17
|
+
const scheduleId = '11111111-1111-4111-8111-111111111111'
|
|
18
|
+
const nextRunAt = new Date(Date.now() + 24 * 60 * 60 * 1000)
|
|
19
|
+
|
|
20
|
+
const tenantId = '22222222-2222-4222-8222-222222222222'
|
|
21
|
+
const organizationId = '33333333-3333-4333-8333-333333333333'
|
|
22
|
+
|
|
23
|
+
const cronTiming: ScheduleTimingSnapshot = {
|
|
24
|
+
scheduleId,
|
|
25
|
+
tenantId,
|
|
26
|
+
organizationId,
|
|
27
|
+
scheduleType: 'cron',
|
|
28
|
+
scheduleValue: '0 6 * * *',
|
|
29
|
+
timezone: 'Europe/Warsaw',
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const intervalTiming: ScheduleTimingSnapshot = {
|
|
33
|
+
scheduleId,
|
|
34
|
+
tenantId,
|
|
35
|
+
organizationId,
|
|
36
|
+
scheduleType: 'interval',
|
|
37
|
+
scheduleValue: '15m',
|
|
38
|
+
timezone: 'UTC',
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function buildEm() {
|
|
42
|
+
const execute = jest.fn(async () => ({ affectedRows: 1 }))
|
|
43
|
+
const em = { getConnection: jest.fn(() => ({ execute })) } as unknown as EntityManager
|
|
44
|
+
return { em, execute, resolveEm: jest.fn(() => em) }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
describe('bullmqNextRunMatchesTiming', () => {
|
|
48
|
+
it('matches a cron scheduler on pattern and timezone', () => {
|
|
49
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, pattern: '0 6 * * *', timezone: 'Europe/Warsaw' }, cronTiming)).toBe(true)
|
|
50
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, pattern: '0 7 * * *', timezone: 'Europe/Warsaw' }, cronTiming)).toBe(false)
|
|
51
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, pattern: '0 6 * * *', timezone: 'UTC' }, cronTiming)).toBe(false)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it('treats a missing timezone on either side as UTC', () => {
|
|
55
|
+
expect(bullmqNextRunMatchesTiming(
|
|
56
|
+
{ nextRunAt, pattern: '0 6 * * *' },
|
|
57
|
+
{ ...cronTiming, timezone: 'UTC' },
|
|
58
|
+
)).toBe(true)
|
|
59
|
+
expect(bullmqNextRunMatchesTiming(
|
|
60
|
+
{ nextRunAt, pattern: '0 6 * * *', timezone: 'UTC' },
|
|
61
|
+
{ ...cronTiming, timezone: '' },
|
|
62
|
+
)).toBe(true)
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('does not match a cron schedule against an every-based scheduler', () => {
|
|
66
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 900000, timezone: 'Europe/Warsaw' }, cronTiming)).toBe(false)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('matches an interval scheduler on its period', () => {
|
|
70
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 900000, timezone: 'UTC' }, intervalTiming)).toBe(true)
|
|
71
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 3600000, timezone: 'UTC' }, intervalTiming)).toBe(false)
|
|
72
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, pattern: '*/15 * * * *', timezone: 'UTC' }, intervalTiming)).toBe(false)
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
it('matches a sub-minute interval against both the clamped and the pre-clamp period', () => {
|
|
76
|
+
const legacyTiming = { ...intervalTiming, scheduleValue: '30s' }
|
|
77
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 60000 }, legacyTiming)).toBe(true)
|
|
78
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 30000 }, legacyTiming)).toBe(true)
|
|
79
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 45000 }, legacyTiming)).toBe(false)
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('does not match an interval value that cannot be parsed', () => {
|
|
83
|
+
expect(bullmqNextRunMatchesTiming({ nextRunAt, every: 60000 }, { ...intervalTiming, scheduleValue: 'soon' })).toBe(false)
|
|
84
|
+
})
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
describe('syncScheduleNextRunAt', () => {
|
|
88
|
+
const originalStrategy = process.env.QUEUE_STRATEGY
|
|
89
|
+
|
|
90
|
+
beforeEach(() => {
|
|
91
|
+
jest.clearAllMocks()
|
|
92
|
+
process.env.QUEUE_STRATEGY = 'async'
|
|
93
|
+
telemetryRuntime = { reportError: mockReportError }
|
|
94
|
+
mockReportError.mockImplementation(() => undefined)
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
afterAll(() => {
|
|
98
|
+
if (originalStrategy === undefined) delete process.env.QUEUE_STRATEGY
|
|
99
|
+
else process.env.QUEUE_STRATEGY = originalStrategy
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it('writes the BullMQ next slot guarded by the loaded timing, the database clock and the stored value', async () => {
|
|
103
|
+
mockReadBullmqNextRun.mockResolvedValue({ nextRunAt, pattern: '0 6 * * *', timezone: 'Europe/Warsaw' })
|
|
104
|
+
const { execute, resolveEm } = buildEm()
|
|
105
|
+
|
|
106
|
+
await syncScheduleNextRunAt(resolveEm, cronTiming)
|
|
107
|
+
|
|
108
|
+
expect(mockReadBullmqNextRun).toHaveBeenCalledWith(scheduleId)
|
|
109
|
+
expect(execute).toHaveBeenCalledTimes(1)
|
|
110
|
+
const [sql, params, method] = execute.mock.calls[0] as unknown as [string, unknown[], string]
|
|
111
|
+
expect(sql.replace(/\s+/g, ' ')).toBe(
|
|
112
|
+
'update scheduled_jobs set next_run_at = ? where id = ?'
|
|
113
|
+
+ ' and tenant_id is not distinct from ? and organization_id is not distinct from ?'
|
|
114
|
+
+ ' and deleted_at is null and is_enabled = true'
|
|
115
|
+
+ ' and schedule_type = ? and schedule_value = ? and timezone = ? and ? > now()'
|
|
116
|
+
+ ' and next_run_at is distinct from ?',
|
|
117
|
+
)
|
|
118
|
+
expect(params).toEqual([
|
|
119
|
+
nextRunAt, scheduleId, tenantId, organizationId, 'cron', '0 6 * * *', 'Europe/Warsaw', nextRunAt, nextRunAt,
|
|
120
|
+
])
|
|
121
|
+
expect(method).toBe('run')
|
|
122
|
+
expect(mockReportError).not.toHaveBeenCalled()
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
it('binds a system-scoped schedule to null tenant and organization', async () => {
|
|
126
|
+
mockReadBullmqNextRun.mockResolvedValue({ nextRunAt, pattern: '0 6 * * *', timezone: 'Europe/Warsaw' })
|
|
127
|
+
const { execute, resolveEm } = buildEm()
|
|
128
|
+
|
|
129
|
+
await syncScheduleNextRunAt(resolveEm, { ...cronTiming, tenantId: null, organizationId: null })
|
|
130
|
+
|
|
131
|
+
const [, params] = execute.mock.calls[0] as unknown as [string, unknown[]]
|
|
132
|
+
expect(params.slice(1, 4)).toEqual([scheduleId, null, null])
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
it('does not set updated_at', async () => {
|
|
136
|
+
mockReadBullmqNextRun.mockResolvedValue({ nextRunAt, every: 900000, timezone: 'UTC' })
|
|
137
|
+
const { execute, resolveEm } = buildEm()
|
|
138
|
+
|
|
139
|
+
await syncScheduleNextRunAt(resolveEm, intervalTiming)
|
|
140
|
+
|
|
141
|
+
const [sql] = execute.mock.calls[0] as unknown as [string]
|
|
142
|
+
expect(sql).not.toMatch(/updated_at/)
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
it.each([undefined, 'local'])('does nothing when QUEUE_STRATEGY is %s', async (strategy) => {
|
|
146
|
+
if (strategy === undefined) delete process.env.QUEUE_STRATEGY
|
|
147
|
+
else process.env.QUEUE_STRATEGY = strategy
|
|
148
|
+
const { execute, resolveEm } = buildEm()
|
|
149
|
+
|
|
150
|
+
await syncScheduleNextRunAt(resolveEm, cronTiming)
|
|
151
|
+
|
|
152
|
+
expect(mockReadBullmqNextRun).not.toHaveBeenCalled()
|
|
153
|
+
expect(resolveEm).not.toHaveBeenCalled()
|
|
154
|
+
expect(execute).not.toHaveBeenCalled()
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
it('writes nothing when BullMQ has no next slot for the schedule', async () => {
|
|
158
|
+
mockReadBullmqNextRun.mockResolvedValue(null)
|
|
159
|
+
const { execute, resolveEm } = buildEm()
|
|
160
|
+
|
|
161
|
+
await syncScheduleNextRunAt(resolveEm, cronTiming)
|
|
162
|
+
|
|
163
|
+
expect(execute).not.toHaveBeenCalled()
|
|
164
|
+
expect(mockReportError).not.toHaveBeenCalled()
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
it('writes nothing when the BullMQ scheduler was registered from a different timing', async () => {
|
|
168
|
+
mockReadBullmqNextRun.mockResolvedValue({ nextRunAt, pattern: '0 0 1 1 *', timezone: 'Europe/Warsaw' })
|
|
169
|
+
const { execute, resolveEm } = buildEm()
|
|
170
|
+
|
|
171
|
+
await syncScheduleNextRunAt(resolveEm, cronTiming)
|
|
172
|
+
|
|
173
|
+
expect(execute).not.toHaveBeenCalled()
|
|
174
|
+
expect(mockReportError).not.toHaveBeenCalled()
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
it('swallows and reports a failed BullMQ read', async () => {
|
|
178
|
+
const failure = new Error('redis unavailable')
|
|
179
|
+
mockReadBullmqNextRun.mockRejectedValue(failure)
|
|
180
|
+
const { execute, resolveEm } = buildEm()
|
|
181
|
+
|
|
182
|
+
await expect(syncScheduleNextRunAt(resolveEm, cronTiming)).resolves.toBeUndefined()
|
|
183
|
+
|
|
184
|
+
expect(execute).not.toHaveBeenCalled()
|
|
185
|
+
expect(mockReportError).toHaveBeenCalledWith(failure, {
|
|
186
|
+
module: 'scheduler',
|
|
187
|
+
code: 'scheduler.next_run_sync_failed',
|
|
188
|
+
attributes: { scheduleId },
|
|
189
|
+
})
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
it('swallows and reports a failed database write', async () => {
|
|
193
|
+
mockReadBullmqNextRun.mockResolvedValue({ nextRunAt, pattern: '0 6 * * *', timezone: 'Europe/Warsaw' })
|
|
194
|
+
const { execute, resolveEm } = buildEm()
|
|
195
|
+
const failure = new Error('connection terminated')
|
|
196
|
+
execute.mockRejectedValue(failure)
|
|
197
|
+
|
|
198
|
+
await expect(syncScheduleNextRunAt(resolveEm, cronTiming)).resolves.toBeUndefined()
|
|
199
|
+
|
|
200
|
+
expect(mockReportError).toHaveBeenCalledWith(failure, expect.objectContaining({
|
|
201
|
+
code: 'scheduler.next_run_sync_failed',
|
|
202
|
+
}))
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
it('still resolves when the telemetry bridge itself throws', async () => {
|
|
206
|
+
mockReadBullmqNextRun.mockRejectedValue(new Error('redis unavailable'))
|
|
207
|
+
mockReportError.mockImplementation(() => {
|
|
208
|
+
throw new Error('telemetry bridge broken')
|
|
209
|
+
})
|
|
210
|
+
const { resolveEm } = buildEm()
|
|
211
|
+
|
|
212
|
+
await expect(syncScheduleNextRunAt(resolveEm, cronTiming)).resolves.toBeUndefined()
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
it('still resolves when no telemetry runtime is installed', async () => {
|
|
216
|
+
telemetryRuntime = null
|
|
217
|
+
mockReadBullmqNextRun.mockRejectedValue(new Error('redis unavailable'))
|
|
218
|
+
const { resolveEm } = buildEm()
|
|
219
|
+
|
|
220
|
+
await expect(syncScheduleNextRunAt(resolveEm, cronTiming)).resolves.toBeUndefined()
|
|
221
|
+
})
|
|
222
|
+
})
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
|
|
2
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
3
|
+
|
|
4
|
+
const logger = createLogger('scheduler').child({ component: 'next-run-sync' })
|
|
5
|
+
|
|
6
|
+
export const BULLMQ_NEXT_RUN_READ_TIMEOUT_MS = 2000
|
|
7
|
+
|
|
8
|
+
export type BullmqNextRun = {
|
|
9
|
+
nextRunAt: Date
|
|
10
|
+
pattern?: string
|
|
11
|
+
every?: number
|
|
12
|
+
timezone?: string
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
interface BullJobSchedulerState {
|
|
16
|
+
next?: number | null
|
|
17
|
+
pattern?: string | null
|
|
18
|
+
every?: number | null
|
|
19
|
+
tz?: string | null
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface BullNextRunQueue {
|
|
23
|
+
getJobScheduler?(id: string): Promise<BullJobSchedulerState | undefined>
|
|
24
|
+
on(event: 'error', listener: (error: Error) => void): unknown
|
|
25
|
+
close(): Promise<void>
|
|
26
|
+
disconnect(): Promise<void>
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function toBullmqNextRun(state: BullJobSchedulerState | undefined): BullmqNextRun | null {
|
|
30
|
+
if (!state || typeof state.next !== 'number' || !Number.isFinite(state.next)) return null
|
|
31
|
+
return {
|
|
32
|
+
nextRunAt: new Date(state.next),
|
|
33
|
+
...(typeof state.pattern === 'string' ? { pattern: state.pattern } : {}),
|
|
34
|
+
...(typeof state.every === 'number' ? { every: state.every } : {}),
|
|
35
|
+
...(typeof state.tz === 'string' ? { timezone: state.tz } : {}),
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function readAndClose(queue: BullNextRunQueue, scheduleId: string): Promise<BullmqNextRun | null> {
|
|
40
|
+
try {
|
|
41
|
+
if (typeof queue.getJobScheduler !== 'function') return null
|
|
42
|
+
return toBullmqNextRun(await queue.getJobScheduler(`schedule-${scheduleId}`))
|
|
43
|
+
} finally {
|
|
44
|
+
await queue.close().catch((error: unknown) => {
|
|
45
|
+
logger.debug('Failed to close the BullMQ queue after reading the job scheduler', { scheduleId, err: error })
|
|
46
|
+
})
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Reads the instant BullMQ will fire a schedule next, straight from its Job Scheduler.
|
|
52
|
+
*
|
|
53
|
+
* BullMQ moves a scheduler to its following slot before it hands the current
|
|
54
|
+
* slot's job to the processor, so from inside a running execution this is the
|
|
55
|
+
* upcoming fire, not the one being executed.
|
|
56
|
+
*
|
|
57
|
+
* The queue handle is opened and closed per call on purpose. Workers build a
|
|
58
|
+
* fresh DI container for every job and never dispose it, so the DI-registered
|
|
59
|
+
* `BullMQSchedulerService` — which caches its queue on the instance — would
|
|
60
|
+
* leave one Redis connection behind per execution.
|
|
61
|
+
*
|
|
62
|
+
* Open, read and close share one deadline: during a Redis outage commands queue
|
|
63
|
+
* up instead of failing, and a caller awaiting this from a `finally` must not
|
|
64
|
+
* hold its worker slot open. Past the deadline the handle is torn down both
|
|
65
|
+
* ways, because each covers a state the other waits on: `close()` drops a
|
|
66
|
+
* connection that never became ready, `disconnect()` drops one that is ready
|
|
67
|
+
* but not answering. The timeout error carries the last connection error
|
|
68
|
+
* BullMQ emitted as its `cause`; the listener also keeps BullMQ from printing
|
|
69
|
+
* those errors to the console itself. A close that fails after a completed read
|
|
70
|
+
* is only logged, so it can neither discard the value nor replace the read's
|
|
71
|
+
* own error. Returns `null` when the installed BullMQ has no
|
|
72
|
+
* `getJobScheduler`, when the scheduler does not exist, or when it has no next
|
|
73
|
+
* slot.
|
|
74
|
+
*/
|
|
75
|
+
export async function readBullmqNextRun(
|
|
76
|
+
scheduleId: string,
|
|
77
|
+
timeoutMs: number = BULLMQ_NEXT_RUN_READ_TIMEOUT_MS,
|
|
78
|
+
): Promise<BullmqNextRun | null> {
|
|
79
|
+
const { Queue } = await import('bullmq')
|
|
80
|
+
const queue = new Queue('scheduler-execution', {
|
|
81
|
+
connection: parseRedisUrl(getRedisUrlOrThrow('QUEUE')),
|
|
82
|
+
}) as unknown as BullNextRunQueue
|
|
83
|
+
let lastConnectionError: Error | undefined
|
|
84
|
+
queue.on('error', (error) => {
|
|
85
|
+
lastConnectionError = error
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
let timer: NodeJS.Timeout | undefined
|
|
89
|
+
let timedOut = false
|
|
90
|
+
const deadline = new Promise<never>((_resolve, reject) => {
|
|
91
|
+
timer = setTimeout(() => {
|
|
92
|
+
timedOut = true
|
|
93
|
+
reject(new Error(
|
|
94
|
+
`[internal] Timed out after ${timeoutMs}ms reading the BullMQ job scheduler`,
|
|
95
|
+
lastConnectionError ? { cause: lastConnectionError } : undefined,
|
|
96
|
+
))
|
|
97
|
+
}, timeoutMs)
|
|
98
|
+
})
|
|
99
|
+
const read = readAndClose(queue, scheduleId)
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
return await Promise.race([read, deadline])
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if (timedOut) {
|
|
105
|
+
queue.close().catch(() => undefined)
|
|
106
|
+
queue.disconnect().catch(() => undefined)
|
|
107
|
+
}
|
|
108
|
+
throw error
|
|
109
|
+
} finally {
|
|
110
|
+
if (timer) clearTimeout(timer)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { EntityManager } from '@mikro-orm/core'
|
|
2
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
3
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
4
|
+
import { readBullmqNextRun, type BullmqNextRun } from './bullmqNextRun.js'
|
|
5
|
+
import { parseInterval, resolveScheduleIntervalMs } from './intervalParser.js'
|
|
6
|
+
|
|
7
|
+
const logger = createLogger('scheduler').child({ component: 'next-run-sync' })
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The schedule's timing configuration as an execution loaded it, captured
|
|
11
|
+
* before the target runs.
|
|
12
|
+
*/
|
|
13
|
+
export type ScheduleTimingSnapshot = {
|
|
14
|
+
scheduleId: string
|
|
15
|
+
tenantId: string | null
|
|
16
|
+
organizationId: string | null
|
|
17
|
+
scheduleType: 'cron' | 'interval'
|
|
18
|
+
scheduleValue: string
|
|
19
|
+
timezone: string
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const UPDATE_NEXT_RUN_AT_SQL = `update scheduled_jobs
|
|
23
|
+
set next_run_at = ?
|
|
24
|
+
where id = ?
|
|
25
|
+
and tenant_id is not distinct from ?
|
|
26
|
+
and organization_id is not distinct from ?
|
|
27
|
+
and deleted_at is null
|
|
28
|
+
and is_enabled = true
|
|
29
|
+
and schedule_type = ?
|
|
30
|
+
and schedule_value = ?
|
|
31
|
+
and timezone = ?
|
|
32
|
+
and ? > now()
|
|
33
|
+
and next_run_at is distinct from ?`
|
|
34
|
+
|
|
35
|
+
function intervalCandidates(scheduleValue: string): number[] {
|
|
36
|
+
try {
|
|
37
|
+
return [resolveScheduleIntervalMs(scheduleValue), parseInterval(scheduleValue)]
|
|
38
|
+
} catch {
|
|
39
|
+
return []
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* True when the BullMQ scheduler was registered from the same timing
|
|
45
|
+
* configuration the execution loaded. A scheduler registered before the
|
|
46
|
+
* minimum-interval clamp keeps the unclamped `every`, so both forms match.
|
|
47
|
+
*/
|
|
48
|
+
export function bullmqNextRunMatchesTiming(next: BullmqNextRun, timing: ScheduleTimingSnapshot): boolean {
|
|
49
|
+
if (timing.scheduleType === 'cron') {
|
|
50
|
+
return next.pattern === timing.scheduleValue
|
|
51
|
+
&& (next.timezone ?? 'UTC') === (timing.timezone || 'UTC')
|
|
52
|
+
}
|
|
53
|
+
return typeof next.every === 'number' && intervalCandidates(timing.scheduleValue).includes(next.every)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Mirrors BullMQ's next fire time into `scheduled_jobs.next_run_at` after an
|
|
58
|
+
* execution under the async strategy, where BullMQ — not this row — decides
|
|
59
|
+
* when a schedule fires.
|
|
60
|
+
*
|
|
61
|
+
* The row is matched by id and by the tenant and organization the execution
|
|
62
|
+
* verified against its job payload, so a write can only land on that scope.
|
|
63
|
+
*
|
|
64
|
+
* The value is written only while it is still true: the BullMQ scheduler and
|
|
65
|
+
* the row must both still carry the timing configuration this execution
|
|
66
|
+
* loaded (an edit in flight writes its own `nextRunAt`), and the instant must
|
|
67
|
+
* still be ahead of the database clock (an execution holding an older slot,
|
|
68
|
+
* or a slot that is due but not picked up, writes nothing). When BullMQ has no
|
|
69
|
+
* usable value nothing is written: a locally computed time would show a run
|
|
70
|
+
* BullMQ is not going to perform.
|
|
71
|
+
*
|
|
72
|
+
* The statement is raw so that `updated_at` stays untouched — skipped and
|
|
73
|
+
* failed executions must not invalidate an editor's optimistic-lock version —
|
|
74
|
+
* and it runs on a pooled connection, outside the per-job unit of work, which
|
|
75
|
+
* may hold whatever a failed target left behind.
|
|
76
|
+
*
|
|
77
|
+
* Never throws: this is bookkeeping awaited from the worker's `finally`, so a
|
|
78
|
+
* failure here must neither fail a successful execution nor replace the
|
|
79
|
+
* target's own error.
|
|
80
|
+
*/
|
|
81
|
+
export async function syncScheduleNextRunAt(
|
|
82
|
+
resolveEm: () => EntityManager,
|
|
83
|
+
timing: ScheduleTimingSnapshot,
|
|
84
|
+
): Promise<void> {
|
|
85
|
+
if (process.env.QUEUE_STRATEGY !== 'async') return
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const next = await readBullmqNextRun(timing.scheduleId)
|
|
89
|
+
if (!next) return
|
|
90
|
+
|
|
91
|
+
if (!bullmqNextRunMatchesTiming(next, timing)) {
|
|
92
|
+
logger.debug('Skipping nextRunAt sync: BullMQ scheduler does not match the loaded schedule timing', {
|
|
93
|
+
scheduleId: timing.scheduleId,
|
|
94
|
+
scheduleType: timing.scheduleType,
|
|
95
|
+
scheduleValue: timing.scheduleValue,
|
|
96
|
+
timezone: timing.timezone,
|
|
97
|
+
bullmqPattern: next.pattern,
|
|
98
|
+
bullmqEvery: next.every,
|
|
99
|
+
bullmqTimezone: next.timezone,
|
|
100
|
+
})
|
|
101
|
+
return
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
await resolveEm().getConnection().execute(
|
|
105
|
+
UPDATE_NEXT_RUN_AT_SQL,
|
|
106
|
+
[
|
|
107
|
+
next.nextRunAt,
|
|
108
|
+
timing.scheduleId,
|
|
109
|
+
timing.tenantId,
|
|
110
|
+
timing.organizationId,
|
|
111
|
+
timing.scheduleType,
|
|
112
|
+
timing.scheduleValue,
|
|
113
|
+
timing.timezone,
|
|
114
|
+
next.nextRunAt,
|
|
115
|
+
next.nextRunAt,
|
|
116
|
+
],
|
|
117
|
+
'run',
|
|
118
|
+
)
|
|
119
|
+
} catch (error) {
|
|
120
|
+
logger.warn('Failed to sync nextRunAt from BullMQ', { scheduleId: timing.scheduleId, err: error })
|
|
121
|
+
try {
|
|
122
|
+
getTelemetryRuntime()?.reportError(error, {
|
|
123
|
+
module: 'scheduler',
|
|
124
|
+
code: 'scheduler.next_run_sync_failed',
|
|
125
|
+
attributes: { scheduleId: timing.scheduleId },
|
|
126
|
+
})
|
|
127
|
+
} catch (telemetryError) {
|
|
128
|
+
logger.warn('Failed to report a nextRunAt sync error to telemetry', { err: telemetryError })
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|