@open-mercato/scheduler 0.6.7-develop.6834.1.e76f4b8cbc → 0.6.7-develop.6851.1.c1680b00d8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/scheduler/api/jobs/[id]/executions/route.js +4 -2
- package/dist/modules/scheduler/api/jobs/[id]/executions/route.js.map +2 -2
- package/dist/modules/scheduler/api/queue-jobs/[jobId]/route.js +4 -2
- package/dist/modules/scheduler/api/queue-jobs/[jobId]/route.js.map +2 -2
- package/dist/modules/scheduler/i18n/ko.json +157 -0
- package/dist/modules/scheduler/services/bullmqSchedulerService.js +90 -41
- package/dist/modules/scheduler/services/bullmqSchedulerService.js.map +2 -2
- package/package.json +6 -6
- package/src/modules/scheduler/api/jobs/[id]/executions/route.ts +4 -2
- package/src/modules/scheduler/api/queue-jobs/[jobId]/route.ts +4 -2
- package/src/modules/scheduler/i18n/ko.json +157 -0
- package/src/modules/scheduler/services/__tests__/bullmqSchedulerService.test.ts +224 -163
- package/src/modules/scheduler/services/bullmqSchedulerService.ts +133 -47
|
@@ -3,11 +3,17 @@ import { ScheduledJob } from '../data/entities.js'
|
|
|
3
3
|
import { recalculateNextRun } from '../lib/nextRunCalculator'
|
|
4
4
|
import { parseCronExpression } from '../lib/cronParser'
|
|
5
5
|
import { resolveScheduleIntervalMs } from '../lib/intervalParser'
|
|
6
|
-
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
6
|
+
import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
|
|
7
7
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
8
8
|
|
|
9
9
|
const logger = createLogger('scheduler').child({ component: 'bullmq' })
|
|
10
10
|
|
|
11
|
+
interface BullJobScheduler {
|
|
12
|
+
key: string
|
|
13
|
+
name: string
|
|
14
|
+
id?: string | null
|
|
15
|
+
}
|
|
16
|
+
|
|
11
17
|
interface BullRepeatableJob {
|
|
12
18
|
key: string
|
|
13
19
|
name: string
|
|
@@ -21,20 +27,32 @@ interface BullRepeatOptions {
|
|
|
21
27
|
}
|
|
22
28
|
|
|
23
29
|
interface BullQueue {
|
|
24
|
-
add(name: string, data: unknown, opts?: unknown): Promise<unknown>
|
|
30
|
+
add(name: string, data: unknown, opts?: Record<string, unknown>): Promise<unknown>
|
|
25
31
|
getRepeatableJobs?(): Promise<BullRepeatableJob[]>
|
|
26
32
|
removeRepeatableByKey?(key: string): Promise<boolean>
|
|
33
|
+
upsertJobScheduler?(
|
|
34
|
+
id: string,
|
|
35
|
+
repeatOptions: BullRepeatOptions,
|
|
36
|
+
jobTemplate: { name: string; data: unknown; opts: Record<string, unknown> },
|
|
37
|
+
): Promise<unknown>
|
|
38
|
+
getJobSchedulers?(): Promise<BullJobScheduler[]>
|
|
39
|
+
removeJobScheduler?(id: string): Promise<boolean>
|
|
27
40
|
close(): Promise<void>
|
|
28
41
|
}
|
|
29
42
|
|
|
43
|
+
type BullJobSchedulerQueue = BullQueue & Required<Pick<
|
|
44
|
+
BullQueue,
|
|
45
|
+
'upsertJobScheduler' | 'getJobSchedulers' | 'removeJobScheduler'
|
|
46
|
+
>>
|
|
47
|
+
|
|
30
48
|
/**
|
|
31
|
-
* Production scheduler using BullMQ
|
|
49
|
+
* Production scheduler using BullMQ Job Schedulers.
|
|
32
50
|
*
|
|
33
51
|
* Requires Redis. Set QUEUE_STRATEGY=async to use this service.
|
|
34
52
|
*
|
|
35
|
-
* This service syncs database schedules with BullMQ's
|
|
53
|
+
* This service syncs database schedules with BullMQ's Job Scheduler mechanism.
|
|
36
54
|
* When a schedule is created/updated/deleted in the database, this service
|
|
37
|
-
* adds/updates/removes the corresponding BullMQ
|
|
55
|
+
* adds/updates/removes the corresponding BullMQ Job Scheduler.
|
|
38
56
|
*
|
|
39
57
|
* BullMQ handles:
|
|
40
58
|
* - Exact timing based on cron expressions or intervals
|
|
@@ -58,7 +76,9 @@ export class BullMQSchedulerService {
|
|
|
58
76
|
if (!this.queue) {
|
|
59
77
|
try {
|
|
60
78
|
const { Queue } = await import('bullmq')
|
|
61
|
-
this.queue = new Queue('scheduler-execution', {
|
|
79
|
+
this.queue = new Queue('scheduler-execution', {
|
|
80
|
+
connection: parseRedisUrl(getRedisUrlOrThrow('QUEUE')),
|
|
81
|
+
})
|
|
62
82
|
} catch {
|
|
63
83
|
throw new Error('BullMQ is required for async scheduler. Install it with: npm install bullmq')
|
|
64
84
|
}
|
|
@@ -70,22 +90,33 @@ export class BullMQSchedulerService {
|
|
|
70
90
|
return `schedule-${scheduleId}`
|
|
71
91
|
}
|
|
72
92
|
|
|
73
|
-
private
|
|
74
|
-
const candidate = job.id
|
|
75
|
-
|
|
76
|
-
: job.name?.startsWith('schedule-')
|
|
77
|
-
? job.name
|
|
78
|
-
: null
|
|
93
|
+
private getScheduleIdFromJobScheduler(job: BullJobScheduler): string | null {
|
|
94
|
+
const candidate = [job.key, job.id, job.name]
|
|
95
|
+
.find((value) => value?.startsWith('schedule-'))
|
|
79
96
|
|
|
80
97
|
return candidate ? candidate.slice('schedule-'.length) : null
|
|
81
98
|
}
|
|
82
99
|
|
|
100
|
+
private isJobSchedulerForSchedule(job: BullJobScheduler, scheduleId: string): boolean {
|
|
101
|
+
return this.getScheduleIdFromJobScheduler(job) === scheduleId
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
private isCanonicalJobScheduler(job: BullJobScheduler, scheduleId: string): boolean {
|
|
105
|
+
return job.key === this.getScheduleJobName(scheduleId)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private supportsJobSchedulers(queue: BullQueue): queue is BullJobSchedulerQueue {
|
|
109
|
+
return typeof queue.upsertJobScheduler === 'function'
|
|
110
|
+
&& typeof queue.getJobSchedulers === 'function'
|
|
111
|
+
&& typeof queue.removeJobScheduler === 'function'
|
|
112
|
+
}
|
|
113
|
+
|
|
83
114
|
private isRepeatableJobForSchedule(job: BullRepeatableJob, scheduleId: string): boolean {
|
|
84
115
|
const jobName = this.getScheduleJobName(scheduleId)
|
|
85
116
|
return job.id === jobName || job.name === jobName
|
|
86
117
|
}
|
|
87
118
|
|
|
88
|
-
private async
|
|
119
|
+
private async removeLegacyRepeatableJobs(queue: BullQueue, scheduleId: string): Promise<number> {
|
|
89
120
|
const repeatableJobs = await queue.getRepeatableJobs?.()
|
|
90
121
|
if (!repeatableJobs || typeof queue.removeRepeatableByKey !== 'function') {
|
|
91
122
|
return 0
|
|
@@ -97,12 +128,11 @@ export class BullMQSchedulerService {
|
|
|
97
128
|
await queue.removeRepeatableByKey(job.key)
|
|
98
129
|
removed += 1
|
|
99
130
|
}
|
|
100
|
-
|
|
101
131
|
return removed
|
|
102
132
|
}
|
|
103
133
|
|
|
104
134
|
/**
|
|
105
|
-
* Register a schedule with BullMQ
|
|
135
|
+
* Register a schedule with BullMQ Job Schedulers
|
|
106
136
|
* @param schedule - The schedule to register
|
|
107
137
|
* @param options - Optional configuration
|
|
108
138
|
* @param options.skipNextRunUpdate - If true, skip updating nextRunAt (used when called from hooks)
|
|
@@ -131,15 +161,14 @@ export class BullMQSchedulerService {
|
|
|
131
161
|
// Build BullMQ repeat options based on schedule type
|
|
132
162
|
const repeatOpts = this.buildRepeatOptions(schedule)
|
|
133
163
|
|
|
134
|
-
//
|
|
164
|
+
// Upsert the BullMQ Job Scheduler
|
|
135
165
|
// IMPORTANT: Wrap in QueuedJob format to match queue strategy expectations
|
|
136
166
|
// BullMQ will store this in job.data, and the async strategy worker expects
|
|
137
167
|
// job.data to be a QueuedJob with id, payload, and createdAt
|
|
138
168
|
const queue = await this.getQueue()
|
|
139
169
|
const jobName = this.getScheduleJobName(schedule.id)
|
|
140
|
-
await this.removeRepeatableJobsForSchedule(queue, schedule.id)
|
|
141
170
|
|
|
142
|
-
//
|
|
171
|
+
// Job Scheduler instances use a stable ID in the data
|
|
143
172
|
// that will be used for each repeat instance
|
|
144
173
|
// CRITICAL: Include scope information (tenantId, organizationId, scopeType)
|
|
145
174
|
// for proper multi-tenant isolation and auditing
|
|
@@ -154,7 +183,7 @@ export class BullMQSchedulerService {
|
|
|
154
183
|
createdAt: new Date().toISOString(),
|
|
155
184
|
}
|
|
156
185
|
|
|
157
|
-
logger.debug('
|
|
186
|
+
logger.debug('Upserting job scheduler', {
|
|
158
187
|
jobName,
|
|
159
188
|
scheduleId: schedule.id,
|
|
160
189
|
scopeType: schedule.scopeType,
|
|
@@ -163,22 +192,40 @@ export class BullMQSchedulerService {
|
|
|
163
192
|
repeatOpts,
|
|
164
193
|
})
|
|
165
194
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
195
|
+
const jobOptions = {
|
|
196
|
+
removeOnComplete: {
|
|
197
|
+
age: 86400 * 30,
|
|
198
|
+
count: 1000,
|
|
199
|
+
},
|
|
200
|
+
removeOnFail: {
|
|
201
|
+
age: 86400 * 90,
|
|
202
|
+
count: 5000,
|
|
203
|
+
},
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
if (this.supportsJobSchedulers(queue)) {
|
|
207
|
+
const existingJobSchedulers = await queue.getJobSchedulers()
|
|
208
|
+
await queue.upsertJobScheduler(
|
|
209
|
+
jobName,
|
|
210
|
+
repeatOpts,
|
|
211
|
+
{
|
|
212
|
+
name: jobName,
|
|
213
|
+
data: jobData,
|
|
214
|
+
opts: jobOptions,
|
|
179
215
|
},
|
|
216
|
+
)
|
|
217
|
+
for (const jobScheduler of existingJobSchedulers) {
|
|
218
|
+
if (
|
|
219
|
+
this.isJobSchedulerForSchedule(jobScheduler, schedule.id)
|
|
220
|
+
&& !this.isCanonicalJobScheduler(jobScheduler, schedule.id)
|
|
221
|
+
) {
|
|
222
|
+
await queue.removeJobScheduler(jobScheduler.key)
|
|
223
|
+
}
|
|
180
224
|
}
|
|
181
|
-
|
|
225
|
+
} else {
|
|
226
|
+
await this.removeLegacyRepeatableJobs(queue, schedule.id)
|
|
227
|
+
await queue.add(jobName, jobData, { repeat: repeatOpts, ...jobOptions })
|
|
228
|
+
}
|
|
182
229
|
|
|
183
230
|
logger.debug('Registered schedule', {
|
|
184
231
|
scheduleId: schedule.id,
|
|
@@ -194,13 +241,28 @@ export class BullMQSchedulerService {
|
|
|
194
241
|
}
|
|
195
242
|
|
|
196
243
|
/**
|
|
197
|
-
* Unregister a schedule from BullMQ
|
|
244
|
+
* Unregister a schedule from BullMQ Job Schedulers
|
|
198
245
|
*/
|
|
199
246
|
async unregister(scheduleId: string): Promise<void> {
|
|
200
247
|
try {
|
|
201
248
|
const queue = await this.getQueue()
|
|
249
|
+
if (this.supportsJobSchedulers(queue)) {
|
|
250
|
+
const jobSchedulers = await queue.getJobSchedulers()
|
|
251
|
+
const matchingJobSchedulers = jobSchedulers.filter((job) => (
|
|
252
|
+
this.isJobSchedulerForSchedule(job, scheduleId)
|
|
253
|
+
))
|
|
254
|
+
if (matchingJobSchedulers.length > 0) {
|
|
255
|
+
for (const jobScheduler of matchingJobSchedulers) {
|
|
256
|
+
await queue.removeJobScheduler(jobScheduler.key)
|
|
257
|
+
}
|
|
258
|
+
logger.debug('Unregistered schedule', { scheduleId })
|
|
259
|
+
} else {
|
|
260
|
+
logger.debug('No job scheduler found for schedule', { scheduleId })
|
|
261
|
+
}
|
|
262
|
+
return
|
|
263
|
+
}
|
|
202
264
|
|
|
203
|
-
const removed = await this.
|
|
265
|
+
const removed = await this.removeLegacyRepeatableJobs(queue, scheduleId)
|
|
204
266
|
if (removed > 0) {
|
|
205
267
|
logger.debug('Unregistered schedule', { scheduleId })
|
|
206
268
|
} else {
|
|
@@ -222,15 +284,33 @@ export class BullMQSchedulerService {
|
|
|
222
284
|
|
|
223
285
|
logger.debug('Starting full sync')
|
|
224
286
|
|
|
225
|
-
// Get all BullMQ repeatable jobs
|
|
226
|
-
const repeatableJobs = await queue.getRepeatableJobs?.() || []
|
|
227
287
|
const bullmqScheduleIds = new Set<string>()
|
|
228
288
|
const bullmqScheduleCounts = new Map<string, number>()
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
289
|
+
const canonicalJobSchedulerIds = new Set<string>()
|
|
290
|
+
if (this.supportsJobSchedulers(queue)) {
|
|
291
|
+
const jobSchedulers = await queue.getJobSchedulers()
|
|
292
|
+
for (const job of jobSchedulers) {
|
|
293
|
+
const scheduleId = this.getScheduleIdFromJobScheduler(job)
|
|
294
|
+
if (!scheduleId) continue
|
|
295
|
+
bullmqScheduleIds.add(scheduleId)
|
|
296
|
+
bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
|
|
297
|
+
if (this.isCanonicalJobScheduler(job, scheduleId)) {
|
|
298
|
+
canonicalJobSchedulerIds.add(scheduleId)
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
} else {
|
|
302
|
+
const repeatableJobs = await queue.getRepeatableJobs?.() ?? []
|
|
303
|
+
for (const job of repeatableJobs) {
|
|
304
|
+
const candidate = job.id?.startsWith('schedule-')
|
|
305
|
+
? job.id
|
|
306
|
+
: job.name?.startsWith('schedule-')
|
|
307
|
+
? job.name
|
|
308
|
+
: null
|
|
309
|
+
if (!candidate) continue
|
|
310
|
+
const scheduleId = candidate.slice('schedule-'.length)
|
|
311
|
+
bullmqScheduleIds.add(scheduleId)
|
|
312
|
+
bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
|
|
313
|
+
}
|
|
234
314
|
}
|
|
235
315
|
|
|
236
316
|
// Get enabled schedules from database in batches to avoid unbounded loads
|
|
@@ -254,8 +334,11 @@ export class BullMQSchedulerService {
|
|
|
254
334
|
if (!bullmqScheduleIds.has(schedule.id)) {
|
|
255
335
|
logger.debug('Registering missing schedule', { scheduleId: schedule.id, scheduleName: schedule.name })
|
|
256
336
|
await this.register(schedule)
|
|
257
|
-
} else if (
|
|
258
|
-
|
|
337
|
+
} else if (
|
|
338
|
+
(bullmqScheduleCounts.get(schedule.id) ?? 0) > 1
|
|
339
|
+
|| !canonicalJobSchedulerIds.has(schedule.id)
|
|
340
|
+
) {
|
|
341
|
+
logger.info('Migrating legacy or duplicate job scheduler', { scheduleId: schedule.id })
|
|
259
342
|
await this.register(schedule)
|
|
260
343
|
}
|
|
261
344
|
}
|
|
@@ -298,14 +381,17 @@ export class BullMQSchedulerService {
|
|
|
298
381
|
}
|
|
299
382
|
|
|
300
383
|
/**
|
|
301
|
-
* Get list of all
|
|
384
|
+
* Get list of all scheduled jobs from BullMQ
|
|
302
385
|
*/
|
|
303
386
|
async getRepeatableJobs(): Promise<unknown[]> {
|
|
304
387
|
try {
|
|
305
388
|
const queue = await this.getQueue()
|
|
306
|
-
|
|
389
|
+
if (this.supportsJobSchedulers(queue)) {
|
|
390
|
+
return await queue.getJobSchedulers()
|
|
391
|
+
}
|
|
392
|
+
return await queue.getRepeatableJobs?.() ?? []
|
|
307
393
|
} catch (error) {
|
|
308
|
-
logger.error('Failed to get
|
|
394
|
+
logger.error('Failed to get job schedulers', { err: error })
|
|
309
395
|
return []
|
|
310
396
|
}
|
|
311
397
|
}
|