@open-mercato/scheduler 0.6.7-develop.6834.1.e76f4b8cbc → 0.6.7-develop.6851.1.c1680b00d8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,11 +3,17 @@ import { ScheduledJob } from '../data/entities.js'
3
3
  import { recalculateNextRun } from '../lib/nextRunCalculator'
4
4
  import { parseCronExpression } from '../lib/cronParser'
5
5
  import { resolveScheduleIntervalMs } from '../lib/intervalParser'
6
- import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
6
+ import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
7
7
  import { createLogger } from '@open-mercato/shared/lib/logger'
8
8
 
9
9
  const logger = createLogger('scheduler').child({ component: 'bullmq' })
10
10
 
11
+ interface BullJobScheduler {
12
+ key: string
13
+ name: string
14
+ id?: string | null
15
+ }
16
+
11
17
  interface BullRepeatableJob {
12
18
  key: string
13
19
  name: string
@@ -21,20 +27,32 @@ interface BullRepeatOptions {
21
27
  }
22
28
 
23
29
  interface BullQueue {
24
- add(name: string, data: unknown, opts?: unknown): Promise<unknown>
30
+ add(name: string, data: unknown, opts?: Record<string, unknown>): Promise<unknown>
25
31
  getRepeatableJobs?(): Promise<BullRepeatableJob[]>
26
32
  removeRepeatableByKey?(key: string): Promise<boolean>
33
+ upsertJobScheduler?(
34
+ id: string,
35
+ repeatOptions: BullRepeatOptions,
36
+ jobTemplate: { name: string; data: unknown; opts: Record<string, unknown> },
37
+ ): Promise<unknown>
38
+ getJobSchedulers?(): Promise<BullJobScheduler[]>
39
+ removeJobScheduler?(id: string): Promise<boolean>
27
40
  close(): Promise<void>
28
41
  }
29
42
 
43
+ type BullJobSchedulerQueue = BullQueue & Required<Pick<
44
+ BullQueue,
45
+ 'upsertJobScheduler' | 'getJobSchedulers' | 'removeJobScheduler'
46
+ >>
47
+
30
48
  /**
31
- * Production scheduler using BullMQ repeatable jobs.
49
+ * Production scheduler using BullMQ Job Schedulers.
32
50
  *
33
51
  * Requires Redis. Set QUEUE_STRATEGY=async to use this service.
34
52
  *
35
- * This service syncs database schedules with BullMQ's repeat mechanism.
53
+ * This service syncs database schedules with BullMQ's Job Scheduler mechanism.
36
54
  * When a schedule is created/updated/deleted in the database, this service
37
- * adds/updates/removes the corresponding BullMQ repeatable job.
55
+ * adds/updates/removes the corresponding BullMQ Job Scheduler.
38
56
  *
39
57
  * BullMQ handles:
40
58
  * - Exact timing based on cron expressions or intervals
@@ -58,7 +76,9 @@ export class BullMQSchedulerService {
58
76
  if (!this.queue) {
59
77
  try {
60
78
  const { Queue } = await import('bullmq')
61
- this.queue = new Queue('scheduler-execution', { connection: { url: getRedisUrlOrThrow('QUEUE') } })
79
+ this.queue = new Queue('scheduler-execution', {
80
+ connection: parseRedisUrl(getRedisUrlOrThrow('QUEUE')),
81
+ })
62
82
  } catch {
63
83
  throw new Error('BullMQ is required for async scheduler. Install it with: npm install bullmq')
64
84
  }
@@ -70,22 +90,33 @@ export class BullMQSchedulerService {
70
90
  return `schedule-${scheduleId}`
71
91
  }
72
92
 
73
- private getScheduleIdFromRepeatableJob(job: BullRepeatableJob): string | null {
74
- const candidate = job.id?.startsWith('schedule-')
75
- ? job.id
76
- : job.name?.startsWith('schedule-')
77
- ? job.name
78
- : null
93
+ private getScheduleIdFromJobScheduler(job: BullJobScheduler): string | null {
94
+ const candidate = [job.key, job.id, job.name]
95
+ .find((value) => value?.startsWith('schedule-'))
79
96
 
80
97
  return candidate ? candidate.slice('schedule-'.length) : null
81
98
  }
82
99
 
100
+ private isJobSchedulerForSchedule(job: BullJobScheduler, scheduleId: string): boolean {
101
+ return this.getScheduleIdFromJobScheduler(job) === scheduleId
102
+ }
103
+
104
+ private isCanonicalJobScheduler(job: BullJobScheduler, scheduleId: string): boolean {
105
+ return job.key === this.getScheduleJobName(scheduleId)
106
+ }
107
+
108
+ private supportsJobSchedulers(queue: BullQueue): queue is BullJobSchedulerQueue {
109
+ return typeof queue.upsertJobScheduler === 'function'
110
+ && typeof queue.getJobSchedulers === 'function'
111
+ && typeof queue.removeJobScheduler === 'function'
112
+ }
113
+
83
114
  private isRepeatableJobForSchedule(job: BullRepeatableJob, scheduleId: string): boolean {
84
115
  const jobName = this.getScheduleJobName(scheduleId)
85
116
  return job.id === jobName || job.name === jobName
86
117
  }
87
118
 
88
- private async removeRepeatableJobsForSchedule(queue: BullQueue, scheduleId: string): Promise<number> {
119
+ private async removeLegacyRepeatableJobs(queue: BullQueue, scheduleId: string): Promise<number> {
89
120
  const repeatableJobs = await queue.getRepeatableJobs?.()
90
121
  if (!repeatableJobs || typeof queue.removeRepeatableByKey !== 'function') {
91
122
  return 0
@@ -97,12 +128,11 @@ export class BullMQSchedulerService {
97
128
  await queue.removeRepeatableByKey(job.key)
98
129
  removed += 1
99
130
  }
100
-
101
131
  return removed
102
132
  }
103
133
 
104
134
  /**
105
- * Register a schedule with BullMQ repeatable jobs
135
+ * Register a schedule with BullMQ Job Schedulers
106
136
  * @param schedule - The schedule to register
107
137
  * @param options - Optional configuration
108
138
  * @param options.skipNextRunUpdate - If true, skip updating nextRunAt (used when called from hooks)
@@ -131,15 +161,14 @@ export class BullMQSchedulerService {
131
161
  // Build BullMQ repeat options based on schedule type
132
162
  const repeatOpts = this.buildRepeatOptions(schedule)
133
163
 
134
- // Add repeatable job to BullMQ
164
+ // Upsert the BullMQ Job Scheduler
135
165
  // IMPORTANT: Wrap in QueuedJob format to match queue strategy expectations
136
166
  // BullMQ will store this in job.data, and the async strategy worker expects
137
167
  // job.data to be a QueuedJob with id, payload, and createdAt
138
168
  const queue = await this.getQueue()
139
169
  const jobName = this.getScheduleJobName(schedule.id)
140
- await this.removeRepeatableJobsForSchedule(queue, schedule.id)
141
170
 
142
- // For repeatable jobs, we need to provide a stable ID in the data
171
+ // Job Scheduler instances use a stable ID in the data
143
172
  // that will be used for each repeat instance
144
173
  // CRITICAL: Include scope information (tenantId, organizationId, scopeType)
145
174
  // for proper multi-tenant isolation and auditing
@@ -154,7 +183,7 @@ export class BullMQSchedulerService {
154
183
  createdAt: new Date().toISOString(),
155
184
  }
156
185
 
157
- logger.debug('Adding repeatable job', {
186
+ logger.debug('Upserting job scheduler', {
158
187
  jobName,
159
188
  scheduleId: schedule.id,
160
189
  scopeType: schedule.scopeType,
@@ -163,22 +192,40 @@ export class BullMQSchedulerService {
163
192
  repeatOpts,
164
193
  })
165
194
 
166
- await queue.add(
167
- jobName, // Job name - used as part of repeatable job key
168
- jobData, // Job data in QueuedJob format
169
- {
170
- repeat: repeatOpts,
171
- // Don't set jobId for repeatable jobs - BullMQ generates unique IDs for each instance
172
- removeOnComplete: {
173
- age: 86400 * 30, // Keep completed jobs for 30 days (execution history)
174
- count: 1000, // Keep last 1000 completed jobs
175
- },
176
- removeOnFail: {
177
- age: 86400 * 90, // Keep failed jobs for 90 days (debugging/audit)
178
- count: 5000, // Keep last 5000 failed jobs
195
+ const jobOptions = {
196
+ removeOnComplete: {
197
+ age: 86400 * 30,
198
+ count: 1000,
199
+ },
200
+ removeOnFail: {
201
+ age: 86400 * 90,
202
+ count: 5000,
203
+ },
204
+ }
205
+
206
+ if (this.supportsJobSchedulers(queue)) {
207
+ const existingJobSchedulers = await queue.getJobSchedulers()
208
+ await queue.upsertJobScheduler(
209
+ jobName,
210
+ repeatOpts,
211
+ {
212
+ name: jobName,
213
+ data: jobData,
214
+ opts: jobOptions,
179
215
  },
216
+ )
217
+ for (const jobScheduler of existingJobSchedulers) {
218
+ if (
219
+ this.isJobSchedulerForSchedule(jobScheduler, schedule.id)
220
+ && !this.isCanonicalJobScheduler(jobScheduler, schedule.id)
221
+ ) {
222
+ await queue.removeJobScheduler(jobScheduler.key)
223
+ }
180
224
  }
181
- )
225
+ } else {
226
+ await this.removeLegacyRepeatableJobs(queue, schedule.id)
227
+ await queue.add(jobName, jobData, { repeat: repeatOpts, ...jobOptions })
228
+ }
182
229
 
183
230
  logger.debug('Registered schedule', {
184
231
  scheduleId: schedule.id,
@@ -194,13 +241,28 @@ export class BullMQSchedulerService {
194
241
  }
195
242
 
196
243
  /**
197
- * Unregister a schedule from BullMQ repeatable jobs
244
+ * Unregister a schedule from BullMQ Job Schedulers
198
245
  */
199
246
  async unregister(scheduleId: string): Promise<void> {
200
247
  try {
201
248
  const queue = await this.getQueue()
249
+ if (this.supportsJobSchedulers(queue)) {
250
+ const jobSchedulers = await queue.getJobSchedulers()
251
+ const matchingJobSchedulers = jobSchedulers.filter((job) => (
252
+ this.isJobSchedulerForSchedule(job, scheduleId)
253
+ ))
254
+ if (matchingJobSchedulers.length > 0) {
255
+ for (const jobScheduler of matchingJobSchedulers) {
256
+ await queue.removeJobScheduler(jobScheduler.key)
257
+ }
258
+ logger.debug('Unregistered schedule', { scheduleId })
259
+ } else {
260
+ logger.debug('No job scheduler found for schedule', { scheduleId })
261
+ }
262
+ return
263
+ }
202
264
 
203
- const removed = await this.removeRepeatableJobsForSchedule(queue, scheduleId)
265
+ const removed = await this.removeLegacyRepeatableJobs(queue, scheduleId)
204
266
  if (removed > 0) {
205
267
  logger.debug('Unregistered schedule', { scheduleId })
206
268
  } else {
@@ -222,15 +284,33 @@ export class BullMQSchedulerService {
222
284
 
223
285
  logger.debug('Starting full sync')
224
286
 
225
- // Get all BullMQ repeatable jobs
226
- const repeatableJobs = await queue.getRepeatableJobs?.() || []
227
287
  const bullmqScheduleIds = new Set<string>()
228
288
  const bullmqScheduleCounts = new Map<string, number>()
229
- for (const job of repeatableJobs) {
230
- const scheduleId = this.getScheduleIdFromRepeatableJob(job)
231
- if (!scheduleId) continue
232
- bullmqScheduleIds.add(scheduleId)
233
- bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
289
+ const canonicalJobSchedulerIds = new Set<string>()
290
+ if (this.supportsJobSchedulers(queue)) {
291
+ const jobSchedulers = await queue.getJobSchedulers()
292
+ for (const job of jobSchedulers) {
293
+ const scheduleId = this.getScheduleIdFromJobScheduler(job)
294
+ if (!scheduleId) continue
295
+ bullmqScheduleIds.add(scheduleId)
296
+ bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
297
+ if (this.isCanonicalJobScheduler(job, scheduleId)) {
298
+ canonicalJobSchedulerIds.add(scheduleId)
299
+ }
300
+ }
301
+ } else {
302
+ const repeatableJobs = await queue.getRepeatableJobs?.() ?? []
303
+ for (const job of repeatableJobs) {
304
+ const candidate = job.id?.startsWith('schedule-')
305
+ ? job.id
306
+ : job.name?.startsWith('schedule-')
307
+ ? job.name
308
+ : null
309
+ if (!candidate) continue
310
+ const scheduleId = candidate.slice('schedule-'.length)
311
+ bullmqScheduleIds.add(scheduleId)
312
+ bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
313
+ }
234
314
  }
235
315
 
236
316
  // Get enabled schedules from database in batches to avoid unbounded loads
@@ -254,8 +334,11 @@ export class BullMQSchedulerService {
254
334
  if (!bullmqScheduleIds.has(schedule.id)) {
255
335
  logger.debug('Registering missing schedule', { scheduleId: schedule.id, scheduleName: schedule.name })
256
336
  await this.register(schedule)
257
- } else if ((bullmqScheduleCounts.get(schedule.id) ?? 0) > 1) {
258
- logger.info('Repairing duplicate repeatable jobs', { scheduleId: schedule.id })
337
+ } else if (
338
+ (bullmqScheduleCounts.get(schedule.id) ?? 0) > 1
339
+ || !canonicalJobSchedulerIds.has(schedule.id)
340
+ ) {
341
+ logger.info('Migrating legacy or duplicate job scheduler', { scheduleId: schedule.id })
259
342
  await this.register(schedule)
260
343
  }
261
344
  }
@@ -298,14 +381,17 @@ export class BullMQSchedulerService {
298
381
  }
299
382
 
300
383
  /**
301
- * Get list of all repeatable jobs from BullMQ
384
+ * Get list of all scheduled jobs from BullMQ
302
385
  */
303
386
  async getRepeatableJobs(): Promise<unknown[]> {
304
387
  try {
305
388
  const queue = await this.getQueue()
306
- return await queue.getRepeatableJobs?.() || []
389
+ if (this.supportsJobSchedulers(queue)) {
390
+ return await queue.getJobSchedulers()
391
+ }
392
+ return await queue.getRepeatableJobs?.() ?? []
307
393
  } catch (error) {
308
- logger.error('Failed to get repeatable jobs', { err: error })
394
+ logger.error('Failed to get job schedulers', { err: error })
309
395
  return []
310
396
  }
311
397
  }