@open-mercato/scheduler 0.6.6-develop.6353.1.efc82affea → 0.6.6-develop.6356.1.4d780a65a3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/scheduler/services/bullmqSchedulerService.js +46 -15
- package/dist/modules/scheduler/services/bullmqSchedulerService.js.map +2 -2
- package/package.json +5 -5
- package/src/modules/scheduler/services/__tests__/bullmqSchedulerService.test.ts +90 -0
- package/src/modules/scheduler/services/bullmqSchedulerService.ts +58 -21
|
@@ -22,6 +22,30 @@ class BullMQSchedulerService {
|
|
|
22
22
|
}
|
|
23
23
|
return this.queue;
|
|
24
24
|
}
|
|
25
|
+
getScheduleJobName(scheduleId) {
|
|
26
|
+
return `schedule-${scheduleId}`;
|
|
27
|
+
}
|
|
28
|
+
getScheduleIdFromRepeatableJob(job) {
|
|
29
|
+
const candidate = job.id?.startsWith("schedule-") ? job.id : job.name?.startsWith("schedule-") ? job.name : null;
|
|
30
|
+
return candidate ? candidate.slice("schedule-".length) : null;
|
|
31
|
+
}
|
|
32
|
+
isRepeatableJobForSchedule(job, scheduleId) {
|
|
33
|
+
const jobName = this.getScheduleJobName(scheduleId);
|
|
34
|
+
return job.id === jobName || job.name === jobName;
|
|
35
|
+
}
|
|
36
|
+
async removeRepeatableJobsForSchedule(queue, scheduleId) {
|
|
37
|
+
const repeatableJobs = await queue.getRepeatableJobs?.();
|
|
38
|
+
if (!repeatableJobs || typeof queue.removeRepeatableByKey !== "function") {
|
|
39
|
+
return 0;
|
|
40
|
+
}
|
|
41
|
+
let removed = 0;
|
|
42
|
+
for (const job of repeatableJobs) {
|
|
43
|
+
if (!this.isRepeatableJobForSchedule(job, scheduleId)) continue;
|
|
44
|
+
await queue.removeRepeatableByKey(job.key);
|
|
45
|
+
removed += 1;
|
|
46
|
+
}
|
|
47
|
+
return removed;
|
|
48
|
+
}
|
|
25
49
|
/**
|
|
26
50
|
* Register a schedule with BullMQ repeatable jobs
|
|
27
51
|
* @param schedule - The schedule to register
|
|
@@ -46,7 +70,8 @@ class BullMQSchedulerService {
|
|
|
46
70
|
}
|
|
47
71
|
const repeatOpts = this.buildRepeatOptions(schedule);
|
|
48
72
|
const queue = await this.getQueue();
|
|
49
|
-
const jobName =
|
|
73
|
+
const jobName = this.getScheduleJobName(schedule.id);
|
|
74
|
+
await this.removeRepeatableJobsForSchedule(queue, schedule.id);
|
|
50
75
|
const jobData = {
|
|
51
76
|
id: jobName,
|
|
52
77
|
payload: {
|
|
@@ -104,17 +129,12 @@ class BullMQSchedulerService {
|
|
|
104
129
|
async unregister(scheduleId) {
|
|
105
130
|
try {
|
|
106
131
|
const queue = await this.getQueue();
|
|
107
|
-
const
|
|
108
|
-
if (
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`);
|
|
113
|
-
return;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
132
|
+
const removed = await this.removeRepeatableJobsForSchedule(queue, scheduleId);
|
|
133
|
+
if (removed > 0) {
|
|
134
|
+
console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`);
|
|
135
|
+
} else {
|
|
136
|
+
console.debug(`[scheduler:bullmq] No repeatable job found for schedule: ${scheduleId}`);
|
|
116
137
|
}
|
|
117
|
-
console.debug(`[scheduler:bullmq] No repeatable job found for schedule: ${scheduleId}`);
|
|
118
138
|
} catch (error) {
|
|
119
139
|
console.error(`[scheduler:bullmq] Failed to unregister schedule: ${scheduleId}`, error);
|
|
120
140
|
throw error;
|
|
@@ -129,9 +149,14 @@ class BullMQSchedulerService {
|
|
|
129
149
|
const queue = await this.getQueue();
|
|
130
150
|
console.debug("[scheduler:bullmq] Starting full sync...");
|
|
131
151
|
const repeatableJobs = await queue.getRepeatableJobs?.() || [];
|
|
132
|
-
const bullmqScheduleIds = new Set(
|
|
133
|
-
|
|
134
|
-
)
|
|
152
|
+
const bullmqScheduleIds = /* @__PURE__ */ new Set();
|
|
153
|
+
const bullmqScheduleCounts = /* @__PURE__ */ new Map();
|
|
154
|
+
for (const job of repeatableJobs) {
|
|
155
|
+
const scheduleId = this.getScheduleIdFromRepeatableJob(job);
|
|
156
|
+
if (!scheduleId) continue;
|
|
157
|
+
bullmqScheduleIds.add(scheduleId);
|
|
158
|
+
bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1);
|
|
159
|
+
}
|
|
135
160
|
const BATCH_SIZE = 500;
|
|
136
161
|
const dbSchedules = [];
|
|
137
162
|
let offset = 0;
|
|
@@ -149,6 +174,9 @@ class BullMQSchedulerService {
|
|
|
149
174
|
if (!bullmqScheduleIds.has(schedule.id)) {
|
|
150
175
|
console.debug(`[scheduler:bullmq] Registering missing schedule: ${schedule.name}`);
|
|
151
176
|
await this.register(schedule);
|
|
177
|
+
} else if ((bullmqScheduleCounts.get(schedule.id) ?? 0) > 1) {
|
|
178
|
+
console.log(`[scheduler:bullmq] Repairing duplicate repeatable jobs for schedule: ${schedule.id}`);
|
|
179
|
+
await this.register(schedule);
|
|
152
180
|
}
|
|
153
181
|
}
|
|
154
182
|
for (const scheduleId of bullmqScheduleIds) {
|
|
@@ -167,7 +195,10 @@ class BullMQSchedulerService {
|
|
|
167
195
|
tz: schedule.timezone || "UTC"
|
|
168
196
|
};
|
|
169
197
|
if (schedule.scheduleType === "cron") {
|
|
170
|
-
parseCronExpression(schedule.scheduleValue, schedule.timezone || "UTC");
|
|
198
|
+
const parsedCron = parseCronExpression(schedule.scheduleValue, schedule.timezone || "UTC");
|
|
199
|
+
if (!parsedCron.isValid) {
|
|
200
|
+
throw new Error(parsedCron.error || "Invalid cron expression");
|
|
201
|
+
}
|
|
171
202
|
opts.pattern = schedule.scheduleValue;
|
|
172
203
|
} else if (schedule.scheduleType === "interval") {
|
|
173
204
|
const intervalMs = parseInterval(schedule.scheduleValue);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/scheduler/services/bullmqSchedulerService.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/core'\nimport { ScheduledJob } from '../data/entities.js'\nimport { recalculateNextRun } from '../lib/nextRunCalculator'\nimport { parseCronExpression } from '../lib/cronParser'\nimport { parseInterval } from '../lib/intervalParser'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\ninterface BullRepeatableJob {\n key: string\n name: string\n id?: string | null\n}\n\ninterface BullRepeatOptions {\n tz?: string\n pattern?: string\n every?: number\n}\n\ninterface BullQueue {\n add(name: string, data: unknown, opts?: unknown): Promise<unknown>\n getRepeatableJobs?(): Promise<BullRepeatableJob[]>\n removeRepeatableByKey?(key: string): Promise<boolean>\n close(): Promise<void>\n}\n\n/**\n * Production scheduler using BullMQ repeatable jobs.\n * \n * Requires Redis. Set QUEUE_STRATEGY=async to use this service.\n * \n * This service syncs database schedules with BullMQ's repeat mechanism.\n * When a schedule is created/updated/deleted in the database, this service\n * adds/updates/removes the corresponding BullMQ repeatable job.\n * \n * BullMQ handles:\n * - Exact timing based on cron expressions or intervals\n * - Distributed locking across multiple instances\n * - Automatic retries if worker fails\n * \n * The worker loads fresh schedule config from DB on each execution,\n * so updates to schedules take effect immediately.\n */\nexport class BullMQSchedulerService {\n private queue: BullQueue | null = null\n \n constructor(\n private em: () => EntityManager,\n ) {}\n\n /**\n * Lazy-load and return the BullMQ queue instance\n */\n private async getQueue(): Promise<BullQueue> {\n if (!this.queue) {\n try {\n const { Queue } = await import('bullmq')\n this.queue = new Queue('scheduler-execution', { connection: { url: getRedisUrlOrThrow('QUEUE') } })\n } catch {\n throw new Error('BullMQ is required for async scheduler. Install it with: npm install bullmq')\n }\n }\n return this.queue\n }\n\n /**\n * Register a schedule with BullMQ repeatable jobs\n * @param schedule - The schedule to register\n * @param options - Optional configuration\n * @param options.skipNextRunUpdate - If true, skip updating nextRunAt (used when called from hooks)\n */\n async register(schedule: ScheduledJob, options: { skipNextRunUpdate?: boolean } = {}): Promise<void> {\n if (!schedule.isEnabled) {\n console.debug(`[scheduler:bullmq] Skipping disabled schedule: ${schedule.id}`)\n return\n }\n\n try {\n // Calculate and update next run time (unless skipped)\n if (!options.skipNextRunUpdate) {\n const nextRun = recalculateNextRun(\n schedule.scheduleType,\n schedule.scheduleValue,\n schedule.timezone\n )\n \n if (nextRun) {\n schedule.nextRunAt = nextRun\n // Flush will be handled by the caller\n }\n }\n\n // Build BullMQ repeat options based on schedule type\n const repeatOpts = this.buildRepeatOptions(schedule)\n\n // Add repeatable job to BullMQ\n // IMPORTANT: Wrap in QueuedJob format to match queue strategy expectations\n // BullMQ will store this in job.data, and the async strategy worker expects\n // job.data to be a QueuedJob with id, payload, and createdAt\n const queue = await this.getQueue()\n const jobName = `schedule-${schedule.id}`\n \n // For repeatable jobs, we need to provide a stable ID in the data\n // that will be used for each repeat instance\n // CRITICAL: Include scope information (tenantId, organizationId, scopeType)\n // for proper multi-tenant isolation and auditing\n const jobData = {\n id: jobName,\n payload: { \n scheduleId: schedule.id,\n tenantId: schedule.tenantId,\n organizationId: schedule.organizationId,\n scopeType: schedule.scopeType,\n },\n createdAt: new Date().toISOString(),\n }\n \n console.debug(`[scheduler:bullmq] Adding repeatable job with data:`, {\n jobName,\n scheduleId: schedule.id,\n scopeType: schedule.scopeType,\n tenantId: schedule.tenantId,\n organizationId: schedule.organizationId,\n repeatOpts,\n jobData,\n })\n \n await queue.add(\n jobName, // Job name - used as part of repeatable job key\n jobData, // Job data in QueuedJob format\n {\n repeat: repeatOpts,\n // Don't set jobId for repeatable jobs - BullMQ generates unique IDs for each instance\n removeOnComplete: {\n age: 86400 * 30, // Keep completed jobs for 30 days (execution history)\n count: 1000, // Keep last 1000 completed jobs\n },\n removeOnFail: {\n age: 86400 * 90, // Keep failed jobs for 90 days (debugging/audit)\n count: 5000, // Keep last 5000 failed jobs\n },\n }\n )\n\n console.debug(`[scheduler:bullmq] Registered schedule: ${schedule.name} (${schedule.id})`, {\n type: schedule.scheduleType,\n pattern: schedule.scheduleValue,\n timezone: schedule.timezone,\n })\n } catch (error: unknown) {\n console.error(`[scheduler:bullmq] Failed to register schedule: ${schedule.id}`, error)\n throw error\n }\n }\n\n /**\n * Unregister a schedule from BullMQ repeatable jobs\n */\n async unregister(scheduleId: string): Promise<void> {\n try {\n const queue = await this.getQueue()\n \n // Remove repeatable job by key\n const repeatableJobs = await queue.getRepeatableJobs?.()\n \n if (repeatableJobs) {\n for (const job of repeatableJobs) {\n if (job.id === `schedule-${scheduleId}` || job.name === `schedule-${scheduleId}`) {\n await queue.removeRepeatableByKey?.(job.key)\n console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`)\n return\n }\n }\n }\n\n console.debug(`[scheduler:bullmq] No repeatable job found for schedule: ${scheduleId}`)\n } catch (error: unknown) {\n console.error(`[scheduler:bullmq] Failed to unregister schedule: ${scheduleId}`, error)\n throw error\n }\n }\n\n /**\n * Sync all enabled schedules with BullMQ\n * Useful for initialization or repair\n */\n async syncAll(): Promise<void> {\n const em = this.em().fork()\n const queue = await this.getQueue()\n \n console.debug('[scheduler:bullmq] Starting full sync...')\n\n // Get all BullMQ repeatable jobs\n const repeatableJobs = await queue.getRepeatableJobs?.() || []\n const bullmqScheduleIds = new Set<string>(\n repeatableJobs\n .filter((j) => j.id?.startsWith('schedule-') || j.name?.startsWith('schedule-'))\n .map((j) => String(j.id || j.name).replace('schedule-', ''))\n )\n\n // Get enabled schedules from database in batches to avoid unbounded loads\n const BATCH_SIZE = 500\n const dbSchedules: ScheduledJob[] = []\n let offset = 0\n let batch: ScheduledJob[]\n do {\n batch = await em.find(ScheduledJob, {\n isEnabled: true,\n deletedAt: null,\n }, { limit: BATCH_SIZE, offset })\n dbSchedules.push(...batch)\n offset += BATCH_SIZE\n } while (batch.length === BATCH_SIZE)\n\n const dbScheduleIds = new Set(dbSchedules.map(s => s.id))\n\n // Register schedules that exist in DB but not in BullMQ\n for (const schedule of dbSchedules) {\n if (!bullmqScheduleIds.has(schedule.id)) {\n console.debug(`[scheduler:bullmq] Registering missing schedule: ${schedule.name}`)\n await this.register(schedule)\n }\n }\n\n // Remove BullMQ jobs that don't exist in DB or are disabled\n for (const scheduleId of bullmqScheduleIds) {\n if (!dbScheduleIds.has(scheduleId)) {\n console.log(`[scheduler:bullmq] Removing orphaned schedule: ${scheduleId}`)\n await this.unregister(String(scheduleId))\n }\n }\n\n console.debug(`[scheduler:bullmq] Sync complete - ${dbSchedules.length} schedules active`)\n }\n\n /**\n * Build BullMQ repeat options from schedule configuration\n */\n private buildRepeatOptions(schedule: ScheduledJob): BullRepeatOptions {\n const opts: BullRepeatOptions = {\n tz: schedule.timezone || 'UTC',\n }\n\n if (schedule.scheduleType === 'cron') {\n // Validate cron expression\n parseCronExpression(schedule.scheduleValue, schedule.timezone || 'UTC')\n opts.pattern = schedule.scheduleValue\n } else if (schedule.scheduleType === 'interval') {\n // Parse interval (e.g., \"15m\", \"2h\", \"1d\")\n const intervalMs = parseInterval(schedule.scheduleValue)\n opts.every = intervalMs\n } else {\n throw new Error(`Unsupported schedule type: ${schedule.scheduleType}`)\n }\n\n return opts\n }\n\n /**\n * Get list of all repeatable jobs from BullMQ\n */\n async getRepeatableJobs(): Promise<unknown[]> {\n try {\n const queue = await this.getQueue()\n return await queue.getRepeatableJobs?.() || []\n } catch (error) {\n console.error('[scheduler:bullmq] Failed to get repeatable jobs:', error)\n return []\n }\n }\n\n /**\n * Close the cached BullMQ queue connection.\n * Must be called during graceful shutdown to prevent Redis connection leaks.\n */\n async destroy(): Promise<void> {\n if (this.queue) {\n try {\n await this.queue.close()\n this.queue = null\n console.debug('[scheduler:bullmq] Queue connection closed')\n } catch (error) {\n console.error('[scheduler:bullmq] Error closing queue:', error)\n }\n }\n }\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,oBAAoB;AAC7B,SAAS,0BAA0B;AACnC,SAAS,2BAA2B;AACpC,SAAS,qBAAqB;AAC9B,SAAS,0BAA0B;AAsC5B,MAAM,uBAAuB;AAAA,EAGlC,YACU,IACR;AADQ;AAHV,SAAQ,QAA0B;AAAA,EAI/B;AAAA;AAAA;AAAA;AAAA,EAKH,MAAc,WAA+B;AAC3C,QAAI,CAAC,KAAK,OAAO;AACf,UAAI;AACF,cAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,aAAK,QAAQ,IAAI,MAAM,uBAAuB,EAAE,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE,EAAE,CAAC;AAAA,MACpG,QAAQ;AACN,cAAM,IAAI,MAAM,6EAA6E;AAAA,MAC/F;AAAA,IACF;AACA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,SAAS,UAAwB,UAA2C,CAAC,GAAkB;AACnG,QAAI,CAAC,SAAS,WAAW;AACvB,cAAQ,MAAM,kDAAkD,SAAS,EAAE,EAAE;AAC7E;AAAA,IACF;AAEA,QAAI;AAEF,UAAI,CAAC,QAAQ,mBAAmB;AAC9B,cAAM,UAAU;AAAA,UACd,SAAS;AAAA,UACT,SAAS;AAAA,UACT,SAAS;AAAA,QACX;AAEA,YAAI,SAAS;AACX,mBAAS,YAAY;AAAA,QAEvB;AAAA,MACF;AAGA,YAAM,aAAa,KAAK,mBAAmB,QAAQ;AAMnD,YAAM,QAAQ,MAAM,KAAK,SAAS;AAClC,YAAM,UAAU,
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/core'\nimport { ScheduledJob } from '../data/entities.js'\nimport { recalculateNextRun } from '../lib/nextRunCalculator'\nimport { parseCronExpression } from '../lib/cronParser'\nimport { parseInterval } from '../lib/intervalParser'\nimport { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'\n\ninterface BullRepeatableJob {\n key: string\n name: string\n id?: string | null\n}\n\ninterface BullRepeatOptions {\n tz?: string\n pattern?: string\n every?: number\n}\n\ninterface BullQueue {\n add(name: string, data: unknown, opts?: unknown): Promise<unknown>\n getRepeatableJobs?(): Promise<BullRepeatableJob[]>\n removeRepeatableByKey?(key: string): Promise<boolean>\n close(): Promise<void>\n}\n\n/**\n * Production scheduler using BullMQ repeatable jobs.\n * \n * Requires Redis. Set QUEUE_STRATEGY=async to use this service.\n * \n * This service syncs database schedules with BullMQ's repeat mechanism.\n * When a schedule is created/updated/deleted in the database, this service\n * adds/updates/removes the corresponding BullMQ repeatable job.\n * \n * BullMQ handles:\n * - Exact timing based on cron expressions or intervals\n * - Distributed locking across multiple instances\n * - Automatic retries if worker fails\n * \n * The worker loads fresh schedule config from DB on each execution,\n * so updates to schedules take effect immediately.\n */\nexport class BullMQSchedulerService {\n private queue: BullQueue | null = null\n \n constructor(\n private em: () => EntityManager,\n ) {}\n\n /**\n * Lazy-load and return the BullMQ queue instance\n */\n private async getQueue(): Promise<BullQueue> {\n if (!this.queue) {\n try {\n const { Queue } = await import('bullmq')\n this.queue = new Queue('scheduler-execution', { connection: { url: getRedisUrlOrThrow('QUEUE') } })\n } catch {\n throw new Error('BullMQ is required for async scheduler. Install it with: npm install bullmq')\n }\n }\n return this.queue\n }\n\n private getScheduleJobName(scheduleId: string): string {\n return `schedule-${scheduleId}`\n }\n\n private getScheduleIdFromRepeatableJob(job: BullRepeatableJob): string | null {\n const candidate = job.id?.startsWith('schedule-')\n ? job.id\n : job.name?.startsWith('schedule-')\n ? job.name\n : null\n\n return candidate ? candidate.slice('schedule-'.length) : null\n }\n\n private isRepeatableJobForSchedule(job: BullRepeatableJob, scheduleId: string): boolean {\n const jobName = this.getScheduleJobName(scheduleId)\n return job.id === jobName || job.name === jobName\n }\n\n private async removeRepeatableJobsForSchedule(queue: BullQueue, scheduleId: string): Promise<number> {\n const repeatableJobs = await queue.getRepeatableJobs?.()\n if (!repeatableJobs || typeof queue.removeRepeatableByKey !== 'function') {\n return 0\n }\n\n let removed = 0\n for (const job of repeatableJobs) {\n if (!this.isRepeatableJobForSchedule(job, scheduleId)) continue\n await queue.removeRepeatableByKey(job.key)\n removed += 1\n }\n\n return removed\n }\n\n /**\n * Register a schedule with BullMQ repeatable jobs\n * @param schedule - The schedule to register\n * @param options - Optional configuration\n * @param options.skipNextRunUpdate - If true, skip updating nextRunAt (used when called from hooks)\n */\n async register(schedule: ScheduledJob, options: { skipNextRunUpdate?: boolean } = {}): Promise<void> {\n if (!schedule.isEnabled) {\n console.debug(`[scheduler:bullmq] Skipping disabled schedule: ${schedule.id}`)\n return\n }\n\n try {\n // Calculate and update next run time (unless skipped)\n if (!options.skipNextRunUpdate) {\n const nextRun = recalculateNextRun(\n schedule.scheduleType,\n schedule.scheduleValue,\n schedule.timezone\n )\n \n if (nextRun) {\n schedule.nextRunAt = nextRun\n // Flush will be handled by the caller\n }\n }\n\n // Build BullMQ repeat options based on schedule type\n const repeatOpts = this.buildRepeatOptions(schedule)\n\n // Add repeatable job to BullMQ\n // IMPORTANT: Wrap in QueuedJob format to match queue strategy expectations\n // BullMQ will store this in job.data, and the async strategy worker expects\n // job.data to be a QueuedJob with id, payload, and createdAt\n const queue = await this.getQueue()\n const jobName = this.getScheduleJobName(schedule.id)\n await this.removeRepeatableJobsForSchedule(queue, schedule.id)\n \n // For repeatable jobs, we need to provide a stable ID in the data\n // that will be used for each repeat instance\n // CRITICAL: Include scope information (tenantId, organizationId, scopeType)\n // for proper multi-tenant isolation and auditing\n const jobData = {\n id: jobName,\n payload: { \n scheduleId: schedule.id,\n tenantId: schedule.tenantId,\n organizationId: schedule.organizationId,\n scopeType: schedule.scopeType,\n },\n createdAt: new Date().toISOString(),\n }\n \n console.debug(`[scheduler:bullmq] Adding repeatable job with data:`, {\n jobName,\n scheduleId: schedule.id,\n scopeType: schedule.scopeType,\n tenantId: schedule.tenantId,\n organizationId: schedule.organizationId,\n repeatOpts,\n jobData,\n })\n \n await queue.add(\n jobName, // Job name - used as part of repeatable job key\n jobData, // Job data in QueuedJob format\n {\n repeat: repeatOpts,\n // Don't set jobId for repeatable jobs - BullMQ generates unique IDs for each instance\n removeOnComplete: {\n age: 86400 * 30, // Keep completed jobs for 30 days (execution history)\n count: 1000, // Keep last 1000 completed jobs\n },\n removeOnFail: {\n age: 86400 * 90, // Keep failed jobs for 90 days (debugging/audit)\n count: 5000, // Keep last 5000 failed jobs\n },\n }\n )\n\n console.debug(`[scheduler:bullmq] Registered schedule: ${schedule.name} (${schedule.id})`, {\n type: schedule.scheduleType,\n pattern: schedule.scheduleValue,\n timezone: schedule.timezone,\n })\n } catch (error: unknown) {\n console.error(`[scheduler:bullmq] Failed to register schedule: ${schedule.id}`, error)\n throw error\n }\n }\n\n /**\n * Unregister a schedule from BullMQ repeatable jobs\n */\n async unregister(scheduleId: string): Promise<void> {\n try {\n const queue = await this.getQueue()\n\n const removed = await this.removeRepeatableJobsForSchedule(queue, scheduleId)\n if (removed > 0) {\n console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`)\n } else {\n console.debug(`[scheduler:bullmq] No repeatable job found for schedule: ${scheduleId}`)\n }\n } catch (error: unknown) {\n console.error(`[scheduler:bullmq] Failed to unregister schedule: ${scheduleId}`, error)\n throw error\n }\n }\n\n /**\n * Sync all enabled schedules with BullMQ\n * Useful for initialization or repair\n */\n async syncAll(): Promise<void> {\n const em = this.em().fork()\n const queue = await this.getQueue()\n \n console.debug('[scheduler:bullmq] Starting full sync...')\n\n // Get all BullMQ repeatable jobs\n const repeatableJobs = await queue.getRepeatableJobs?.() || []\n const bullmqScheduleIds = new Set<string>()\n const bullmqScheduleCounts = new Map<string, number>()\n for (const job of repeatableJobs) {\n const scheduleId = this.getScheduleIdFromRepeatableJob(job)\n if (!scheduleId) continue\n bullmqScheduleIds.add(scheduleId)\n bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)\n }\n\n // Get enabled schedules from database in batches to avoid unbounded loads\n const BATCH_SIZE = 500\n const dbSchedules: ScheduledJob[] = []\n let offset = 0\n let batch: ScheduledJob[]\n do {\n batch = await em.find(ScheduledJob, {\n isEnabled: true,\n deletedAt: null,\n }, { limit: BATCH_SIZE, offset })\n dbSchedules.push(...batch)\n offset += BATCH_SIZE\n } while (batch.length === BATCH_SIZE)\n\n const dbScheduleIds = new Set(dbSchedules.map(s => s.id))\n\n // Register schedules that exist in DB but not in BullMQ\n for (const schedule of dbSchedules) {\n if (!bullmqScheduleIds.has(schedule.id)) {\n console.debug(`[scheduler:bullmq] Registering missing schedule: ${schedule.name}`)\n await this.register(schedule)\n } else if ((bullmqScheduleCounts.get(schedule.id) ?? 0) > 1) {\n console.log(`[scheduler:bullmq] Repairing duplicate repeatable jobs for schedule: ${schedule.id}`)\n await this.register(schedule)\n }\n }\n\n // Remove BullMQ jobs that don't exist in DB or are disabled\n for (const scheduleId of bullmqScheduleIds) {\n if (!dbScheduleIds.has(scheduleId)) {\n console.log(`[scheduler:bullmq] Removing orphaned schedule: ${scheduleId}`)\n await this.unregister(String(scheduleId))\n }\n }\n\n console.debug(`[scheduler:bullmq] Sync complete - ${dbSchedules.length} schedules active`)\n }\n\n /**\n * Build BullMQ repeat options from schedule configuration\n */\n private buildRepeatOptions(schedule: ScheduledJob): BullRepeatOptions {\n const opts: BullRepeatOptions = {\n tz: schedule.timezone || 'UTC',\n }\n\n if (schedule.scheduleType === 'cron') {\n // Validate cron expression\n const parsedCron = parseCronExpression(schedule.scheduleValue, schedule.timezone || 'UTC')\n if (!parsedCron.isValid) {\n throw new Error(parsedCron.error || 'Invalid cron expression')\n }\n opts.pattern = schedule.scheduleValue\n } else if (schedule.scheduleType === 'interval') {\n // Parse interval (e.g., \"15m\", \"2h\", \"1d\")\n const intervalMs = parseInterval(schedule.scheduleValue)\n opts.every = intervalMs\n } else {\n throw new Error(`Unsupported schedule type: ${schedule.scheduleType}`)\n }\n\n return opts\n }\n\n /**\n * Get list of all repeatable jobs from BullMQ\n */\n async getRepeatableJobs(): Promise<unknown[]> {\n try {\n const queue = await this.getQueue()\n return await queue.getRepeatableJobs?.() || []\n } catch (error) {\n console.error('[scheduler:bullmq] Failed to get repeatable jobs:', error)\n return []\n }\n }\n\n /**\n * Close the cached BullMQ queue connection.\n * Must be called during graceful shutdown to prevent Redis connection leaks.\n */\n async destroy(): Promise<void> {\n if (this.queue) {\n try {\n await this.queue.close()\n this.queue = null\n console.debug('[scheduler:bullmq] Queue connection closed')\n } catch (error) {\n console.error('[scheduler:bullmq] Error closing queue:', error)\n }\n }\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oBAAoB;AAC7B,SAAS,0BAA0B;AACnC,SAAS,2BAA2B;AACpC,SAAS,qBAAqB;AAC9B,SAAS,0BAA0B;AAsC5B,MAAM,uBAAuB;AAAA,EAGlC,YACU,IACR;AADQ;AAHV,SAAQ,QAA0B;AAAA,EAI/B;AAAA;AAAA;AAAA;AAAA,EAKH,MAAc,WAA+B;AAC3C,QAAI,CAAC,KAAK,OAAO;AACf,UAAI;AACF,cAAM,EAAE,MAAM,IAAI,MAAM,OAAO,QAAQ;AACvC,aAAK,QAAQ,IAAI,MAAM,uBAAuB,EAAE,YAAY,EAAE,KAAK,mBAAmB,OAAO,EAAE,EAAE,CAAC;AAAA,MACpG,QAAQ;AACN,cAAM,IAAI,MAAM,6EAA6E;AAAA,MAC/F;AAAA,IACF;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEQ,mBAAmB,YAA4B;AACrD,WAAO,YAAY,UAAU;AAAA,EAC/B;AAAA,EAEQ,+BAA+B,KAAuC;AAC5E,UAAM,YAAY,IAAI,IAAI,WAAW,WAAW,IAC5C,IAAI,KACJ,IAAI,MAAM,WAAW,WAAW,IAC9B,IAAI,OACJ;AAEN,WAAO,YAAY,UAAU,MAAM,YAAY,MAAM,IAAI;AAAA,EAC3D;AAAA,EAEQ,2BAA2B,KAAwB,YAA6B;AACtF,UAAM,UAAU,KAAK,mBAAmB,UAAU;AAClD,WAAO,IAAI,OAAO,WAAW,IAAI,SAAS;AAAA,EAC5C;AAAA,EAEA,MAAc,gCAAgC,OAAkB,YAAqC;AACnG,UAAM,iBAAiB,MAAM,MAAM,oBAAoB;AACvD,QAAI,CAAC,kBAAkB,OAAO,MAAM,0BAA0B,YAAY;AACxE,aAAO;AAAA,IACT;AAEA,QAAI,UAAU;AACd,eAAW,OAAO,gBAAgB;AAChC,UAAI,CAAC,KAAK,2BAA2B,KAAK,UAAU,EAAG;AACvD,YAAM,MAAM,sBAAsB,IAAI,GAAG;AACzC,iBAAW;AAAA,IACb;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,SAAS,UAAwB,UAA2C,CAAC,GAAkB;AACnG,QAAI,CAAC,SAAS,WAAW;AACvB,cAAQ,MAAM,kDAAkD,SAAS,EAAE,EAAE;AAC7E;AAAA,IACF;AAEA,QAAI;AAEF,UAAI,CAAC,QAAQ,mBAAmB;AAC9B,cAAM,UAAU;AAAA,UACd,SAAS;AAAA,UACT,SAAS;AAAA,UACT,SAAS;AAAA,QACX;AAEA,YAAI,SAAS;AACX,mBAAS,YAAY;AAAA,QAEvB;AAAA,MACF;AAGA,YAAM,aAAa,KAAK,mBAAmB,QAAQ;AAMnD,YAAM,QAAQ,MAAM,KAAK,SAAS;AAClC,YAAM,UAAU,KAAK,mBAAmB,SAAS,EAAE;AACnD,YAAM,KAAK,gCAAgC,OAAO,SAAS,EAAE;AAM7D,YAAM,UAAU;AAAA,QACd,IAAI;AAAA,QACJ,SAAS;AAAA,UACP,YAAY,SAAS;AAAA,UACrB,UAAU,SAAS;AAAA,UACnB,gBAAgB,SAAS;AAAA,UACzB,WAAW,SAAS;AAAA,QACtB;AAAA,QACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MACpC;AAEA,cAAQ,MAAM,uDAAuD;AAAA,QACnE;AAAA,QACA,YAAY,SAAS;AAAA,QACrB,WAAW,SAAS;AAAA,QACpB,UAAU,SAAS;AAAA,QACnB,gBAAgB,SAAS;AAAA,QACzB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM,MAAM;AAAA,QACV;AAAA;AAAA,QACA;AAAA;AAAA,QACA;AAAA,UACE,QAAQ;AAAA;AAAA,UAER,kBAAkB;AAAA,YAChB,KAAK,QAAQ;AAAA;AAAA,YACb,OAAO;AAAA;AAAA,UACT;AAAA,UACA,cAAc;AAAA,YACZ,KAAK,QAAQ;AAAA;AAAA,YACb,OAAO;AAAA;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAEA,cAAQ,MAAM,2CAA2C,SAAS,IAAI,KAAK,SAAS,EAAE,KAAK;AAAA,QACzF,MAAM,SAAS;AAAA,QACf,SAAS,SAAS;AAAA,QAClB,UAAU,SAAS;AAAA,MACrB,CAAC;AAAA,IACH,SAAS,OAAgB;AACvB,cAAQ,MAAM,mDAAmD,SAAS,EAAE,IAAI,KAAK;AACrF,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,WAAW,YAAmC;AAClD,QAAI;AACF,YAAM,QAAQ,MAAM,KAAK,SAAS;AAElC,YAAM,UAAU,MAAM,KAAK,gCAAgC,OAAO,UAAU;AAC5E,UAAI,UAAU,GAAG;AACf,gBAAQ,MAAM,6CAA6C,UAAU,EAAE;AAAA,MACzE,OAAO;AACL,gBAAQ,MAAM,4DAA4D,UAAU,EAAE;AAAA,MACxF;AAAA,IACF,SAAS,OAAgB;AACvB,cAAQ,MAAM,qDAAqD,UAAU,IAAI,KAAK;AACtF,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAyB;AAC7B,UAAM,KAAK,KAAK,GAAG,EAAE,KAAK;AAC1B,UAAM,QAAQ,MAAM,KAAK,SAAS;AAElC,YAAQ,MAAM,0CAA0C;AAGxD,UAAM,iBAAiB,MAAM,MAAM,oBAAoB,KAAK,CAAC;AAC7D,UAAM,oBAAoB,oBAAI,IAAY;AAC1C,UAAM,uBAAuB,oBAAI,IAAoB;AACrD,eAAW,OAAO,gBAAgB;AAChC,YAAM,aAAa,KAAK,+BAA+B,GAAG;AAC1D,UAAI,CAAC,WAAY;AACjB,wBAAkB,IAAI,UAAU;AAChC,2BAAqB,IAAI,aAAa,qBAAqB,IAAI,UAAU,KAAK,KAAK,CAAC;AAAA,IACtF;AAGA,UAAM,aAAa;AACnB,UAAM,cAA8B,CAAC;AACrC,QAAI,SAAS;AACb,QAAI;AACJ,OAAG;AACD,cAAQ,MAAM,GAAG,KAAK,cAAc;AAAA,QAClC,WAAW;AAAA,QACX,WAAW;AAAA,MACb,GAAG,EAAE,OAAO,YAAY,OAAO,CAAC;AAChC,kBAAY,KAAK,GAAG,KAAK;AACzB,gBAAU;AAAA,IACZ,SAAS,MAAM,WAAW;AAE1B,UAAM,gBAAgB,IAAI,IAAI,YAAY,IAAI,OAAK,EAAE,EAAE,CAAC;AAGxD,eAAW,YAAY,aAAa;AAClC,UAAI,CAAC,kBAAkB,IAAI,SAAS,EAAE,GAAG;AACvC,gBAAQ,MAAM,oDAAoD,SAAS,IAAI,EAAE;AACjF,cAAM,KAAK,SAAS,QAAQ;AAAA,MAC9B,YAAY,qBAAqB,IAAI,SAAS,EAAE,KAAK,KAAK,GAAG;AAC3D,gBAAQ,IAAI,wEAAwE,SAAS,EAAE,EAAE;AACjG,cAAM,KAAK,SAAS,QAAQ;AAAA,MAC9B;AAAA,IACF;AAGA,eAAW,cAAc,mBAAmB;AAC1C,UAAI,CAAC,cAAc,IAAI,UAAU,GAAG;AAClC,gBAAQ,IAAI,kDAAkD,UAAU,EAAE;AAC1E,cAAM,KAAK,WAAW,OAAO,UAAU,CAAC;AAAA,MAC1C;AAAA,IACF;AAEA,YAAQ,MAAM,sCAAsC,YAAY,MAAM,mBAAmB;AAAA,EAC3F;AAAA;AAAA;AAAA;AAAA,EAKQ,mBAAmB,UAA2C;AACpE,UAAM,OAA0B;AAAA,MAC9B,IAAI,SAAS,YAAY;AAAA,IAC3B;AAEA,QAAI,SAAS,iBAAiB,QAAQ;AAEpC,YAAM,aAAa,oBAAoB,SAAS,eAAe,SAAS,YAAY,KAAK;AACzF,UAAI,CAAC,WAAW,SAAS;AACvB,cAAM,IAAI,MAAM,WAAW,SAAS,yBAAyB;AAAA,MAC/D;AACA,WAAK,UAAU,SAAS;AAAA,IAC1B,WAAW,SAAS,iBAAiB,YAAY;AAE/C,YAAM,aAAa,cAAc,SAAS,aAAa;AACvD,WAAK,QAAQ;AAAA,IACf,OAAO;AACL,YAAM,IAAI,MAAM,8BAA8B,SAAS,YAAY,EAAE;AAAA,IACvE;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,oBAAwC;AAC5C,QAAI;AACF,YAAM,QAAQ,MAAM,KAAK,SAAS;AAClC,aAAO,MAAM,MAAM,oBAAoB,KAAK,CAAC;AAAA,IAC/C,SAAS,OAAO;AACd,cAAQ,MAAM,qDAAqD,KAAK;AACxE,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAyB;AAC7B,QAAI,KAAK,OAAO;AACd,UAAI;AACF,cAAM,KAAK,MAAM,MAAM;AACvB,aAAK,QAAQ;AACb,gBAAQ,MAAM,4CAA4C;AAAA,MAC5D,SAAS,OAAO;AACd,gBAAQ,MAAM,2CAA2C,KAAK;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/scheduler",
|
|
3
|
-
"version": "0.6.6-develop.
|
|
3
|
+
"version": "0.6.6-develop.6356.1.4d780a65a3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Database-managed scheduled jobs with admin UI",
|
|
6
6
|
"type": "module",
|
|
@@ -59,13 +59,13 @@
|
|
|
59
59
|
"./*/*/*/*/*/*.json": "./src/*/*/*/*/*/*.json"
|
|
60
60
|
},
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@open-mercato/events": "0.6.6-develop.
|
|
63
|
-
"@open-mercato/queue": "0.6.6-develop.
|
|
62
|
+
"@open-mercato/events": "0.6.6-develop.6356.1.4d780a65a3",
|
|
63
|
+
"@open-mercato/queue": "0.6.6-develop.6356.1.4d780a65a3",
|
|
64
64
|
"cron-parser": "^5.6.0"
|
|
65
65
|
},
|
|
66
66
|
"peerDependencies": {
|
|
67
67
|
"@mikro-orm/core": "^7.0.14",
|
|
68
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
68
|
+
"@open-mercato/shared": "0.6.6-develop.6356.1.4d780a65a3",
|
|
69
69
|
"bullmq": "^5.0.0",
|
|
70
70
|
"ioredis": "^5.0.0",
|
|
71
71
|
"zod": ">=3.23.0"
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
}
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
82
|
+
"@open-mercato/shared": "0.6.6-develop.6356.1.4d780a65a3",
|
|
83
83
|
"@types/jest": "^30.0.0",
|
|
84
84
|
"@types/node": "^26.0.0",
|
|
85
85
|
"jest": "^30.4.2",
|
|
@@ -27,6 +27,9 @@ describe('BullMQSchedulerService', () => {
|
|
|
27
27
|
|
|
28
28
|
beforeEach(() => {
|
|
29
29
|
jest.clearAllMocks()
|
|
30
|
+
mockQueue.add.mockResolvedValue({})
|
|
31
|
+
mockQueue.getRepeatableJobs.mockResolvedValue([])
|
|
32
|
+
mockQueue.removeRepeatableByKey.mockResolvedValue(true)
|
|
30
33
|
|
|
31
34
|
// Create mock forked EM
|
|
32
35
|
mockForkedEm = {
|
|
@@ -170,6 +173,44 @@ describe('BullMQSchedulerService', () => {
|
|
|
170
173
|
)
|
|
171
174
|
})
|
|
172
175
|
|
|
176
|
+
it('should remove stale repeatable jobs before registering an updated schedule', async () => {
|
|
177
|
+
const schedule = {
|
|
178
|
+
id: 'test-1',
|
|
179
|
+
name: 'Test',
|
|
180
|
+
isEnabled: true,
|
|
181
|
+
scheduleType: 'cron',
|
|
182
|
+
scheduleValue: '*/15 * * * *',
|
|
183
|
+
timezone: 'UTC',
|
|
184
|
+
scopeType: 'system',
|
|
185
|
+
} as ScheduledJob
|
|
186
|
+
|
|
187
|
+
mockQueue.getRepeatableJobs.mockResolvedValue([
|
|
188
|
+
{ id: 'schedule-test-1', name: 'schedule-test-1', key: 'old-cron-key' },
|
|
189
|
+
{ id: 'schedule-test-1', name: 'schedule-test-1', key: 'older-cron-key' },
|
|
190
|
+
{ id: 'schedule-other', name: 'schedule-other', key: 'other-key' },
|
|
191
|
+
])
|
|
192
|
+
mockQueue.removeRepeatableByKey.mockResolvedValue(true)
|
|
193
|
+
mockQueue.add.mockResolvedValue({})
|
|
194
|
+
|
|
195
|
+
await service.register(schedule)
|
|
196
|
+
|
|
197
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledTimes(2)
|
|
198
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(1, 'old-cron-key')
|
|
199
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(2, 'older-cron-key')
|
|
200
|
+
expect(mockQueue.add).toHaveBeenCalledWith(
|
|
201
|
+
'schedule-test-1',
|
|
202
|
+
expect.any(Object),
|
|
203
|
+
expect.objectContaining({
|
|
204
|
+
repeat: expect.objectContaining({
|
|
205
|
+
pattern: '*/15 * * * *',
|
|
206
|
+
}),
|
|
207
|
+
}),
|
|
208
|
+
)
|
|
209
|
+
expect(mockQueue.removeRepeatableByKey.mock.invocationCallOrder[1]).toBeLessThan(
|
|
210
|
+
mockQueue.add.mock.invocationCallOrder[0],
|
|
211
|
+
)
|
|
212
|
+
})
|
|
213
|
+
|
|
173
214
|
it('should update nextRunAt when skipNextRunUpdate is false', async () => {
|
|
174
215
|
const schedule = {
|
|
175
216
|
id: 'test-1',
|
|
@@ -287,6 +328,23 @@ describe('BullMQSchedulerService', () => {
|
|
|
287
328
|
consoleDebugSpy.mockRestore()
|
|
288
329
|
})
|
|
289
330
|
|
|
331
|
+
it('should remove all repeatable jobs for the same schedule id', async () => {
|
|
332
|
+
mockQueue.getRepeatableJobs.mockResolvedValue([
|
|
333
|
+
{ id: 'schedule-test-1', name: 'schedule-test-1', key: 'key-1' },
|
|
334
|
+
{ id: 'schedule-test-1', name: 'schedule-test-1', key: 'key-2' },
|
|
335
|
+
{ name: 'schedule-test-1', key: 'key-3' },
|
|
336
|
+
{ id: 'schedule-other', name: 'schedule-other', key: 'other-key' },
|
|
337
|
+
])
|
|
338
|
+
mockQueue.removeRepeatableByKey.mockResolvedValue(true)
|
|
339
|
+
|
|
340
|
+
await service.unregister('test-1')
|
|
341
|
+
|
|
342
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledTimes(3)
|
|
343
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(1, 'key-1')
|
|
344
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(2, 'key-2')
|
|
345
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(3, 'key-3')
|
|
346
|
+
})
|
|
347
|
+
|
|
290
348
|
it('should handle schedule not found', async () => {
|
|
291
349
|
mockQueue.getRepeatableJobs.mockResolvedValue([
|
|
292
350
|
{ id: 'schedule-other', name: 'schedule-other', key: 'key-1' },
|
|
@@ -389,6 +447,38 @@ describe('BullMQSchedulerService', () => {
|
|
|
389
447
|
consoleLogSpy.mockRestore()
|
|
390
448
|
})
|
|
391
449
|
|
|
450
|
+
it('should repair duplicate repeatable jobs for existing schedules', async () => {
|
|
451
|
+
const dbSchedules = [
|
|
452
|
+
{ id: 'schedule-1', name: 'Schedule 1', isEnabled: true, scheduleType: 'cron', scheduleValue: '0 0 * * *', timezone: 'UTC', scopeType: 'system' },
|
|
453
|
+
] as ScheduledJob[]
|
|
454
|
+
|
|
455
|
+
mockForkedEm.find.mockResolvedValue(dbSchedules)
|
|
456
|
+
mockQueue.getRepeatableJobs.mockResolvedValue([
|
|
457
|
+
{ id: 'schedule-schedule-1', name: 'schedule-schedule-1', key: 'old-key-1' },
|
|
458
|
+
{ id: 'schedule-schedule-1', name: 'schedule-schedule-1', key: 'old-key-2' },
|
|
459
|
+
])
|
|
460
|
+
mockQueue.removeRepeatableByKey.mockResolvedValue(true)
|
|
461
|
+
mockQueue.add.mockResolvedValue({})
|
|
462
|
+
|
|
463
|
+
const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation()
|
|
464
|
+
|
|
465
|
+
await service.syncAll()
|
|
466
|
+
|
|
467
|
+
expect(consoleLogSpy).toHaveBeenCalledWith(
|
|
468
|
+
'[scheduler:bullmq] Repairing duplicate repeatable jobs for schedule: schedule-1'
|
|
469
|
+
)
|
|
470
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledTimes(2)
|
|
471
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(1, 'old-key-1')
|
|
472
|
+
expect(mockQueue.removeRepeatableByKey).toHaveBeenNthCalledWith(2, 'old-key-2')
|
|
473
|
+
expect(mockQueue.add).toHaveBeenCalledWith(
|
|
474
|
+
'schedule-schedule-1',
|
|
475
|
+
expect.any(Object),
|
|
476
|
+
expect.any(Object),
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
consoleLogSpy.mockRestore()
|
|
480
|
+
})
|
|
481
|
+
|
|
392
482
|
it('should log sync completion', async () => {
|
|
393
483
|
mockForkedEm.find.mockResolvedValue([])
|
|
394
484
|
mockQueue.getRepeatableJobs.mockResolvedValue([])
|
|
@@ -63,6 +63,41 @@ export class BullMQSchedulerService {
|
|
|
63
63
|
return this.queue
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
private getScheduleJobName(scheduleId: string): string {
|
|
67
|
+
return `schedule-${scheduleId}`
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
private getScheduleIdFromRepeatableJob(job: BullRepeatableJob): string | null {
|
|
71
|
+
const candidate = job.id?.startsWith('schedule-')
|
|
72
|
+
? job.id
|
|
73
|
+
: job.name?.startsWith('schedule-')
|
|
74
|
+
? job.name
|
|
75
|
+
: null
|
|
76
|
+
|
|
77
|
+
return candidate ? candidate.slice('schedule-'.length) : null
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
private isRepeatableJobForSchedule(job: BullRepeatableJob, scheduleId: string): boolean {
|
|
81
|
+
const jobName = this.getScheduleJobName(scheduleId)
|
|
82
|
+
return job.id === jobName || job.name === jobName
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private async removeRepeatableJobsForSchedule(queue: BullQueue, scheduleId: string): Promise<number> {
|
|
86
|
+
const repeatableJobs = await queue.getRepeatableJobs?.()
|
|
87
|
+
if (!repeatableJobs || typeof queue.removeRepeatableByKey !== 'function') {
|
|
88
|
+
return 0
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
let removed = 0
|
|
92
|
+
for (const job of repeatableJobs) {
|
|
93
|
+
if (!this.isRepeatableJobForSchedule(job, scheduleId)) continue
|
|
94
|
+
await queue.removeRepeatableByKey(job.key)
|
|
95
|
+
removed += 1
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return removed
|
|
99
|
+
}
|
|
100
|
+
|
|
66
101
|
/**
|
|
67
102
|
* Register a schedule with BullMQ repeatable jobs
|
|
68
103
|
* @param schedule - The schedule to register
|
|
@@ -98,7 +133,8 @@ export class BullMQSchedulerService {
|
|
|
98
133
|
// BullMQ will store this in job.data, and the async strategy worker expects
|
|
99
134
|
// job.data to be a QueuedJob with id, payload, and createdAt
|
|
100
135
|
const queue = await this.getQueue()
|
|
101
|
-
const jobName =
|
|
136
|
+
const jobName = this.getScheduleJobName(schedule.id)
|
|
137
|
+
await this.removeRepeatableJobsForSchedule(queue, schedule.id)
|
|
102
138
|
|
|
103
139
|
// For repeatable jobs, we need to provide a stable ID in the data
|
|
104
140
|
// that will be used for each repeat instance
|
|
@@ -159,21 +195,13 @@ export class BullMQSchedulerService {
|
|
|
159
195
|
async unregister(scheduleId: string): Promise<void> {
|
|
160
196
|
try {
|
|
161
197
|
const queue = await this.getQueue()
|
|
162
|
-
|
|
163
|
-
// Remove repeatable job by key
|
|
164
|
-
const repeatableJobs = await queue.getRepeatableJobs?.()
|
|
165
|
-
|
|
166
|
-
if (repeatableJobs) {
|
|
167
|
-
for (const job of repeatableJobs) {
|
|
168
|
-
if (job.id === `schedule-${scheduleId}` || job.name === `schedule-${scheduleId}`) {
|
|
169
|
-
await queue.removeRepeatableByKey?.(job.key)
|
|
170
|
-
console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`)
|
|
171
|
-
return
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
198
|
|
|
176
|
-
|
|
199
|
+
const removed = await this.removeRepeatableJobsForSchedule(queue, scheduleId)
|
|
200
|
+
if (removed > 0) {
|
|
201
|
+
console.debug(`[scheduler:bullmq] Unregistered schedule: ${scheduleId}`)
|
|
202
|
+
} else {
|
|
203
|
+
console.debug(`[scheduler:bullmq] No repeatable job found for schedule: ${scheduleId}`)
|
|
204
|
+
}
|
|
177
205
|
} catch (error: unknown) {
|
|
178
206
|
console.error(`[scheduler:bullmq] Failed to unregister schedule: ${scheduleId}`, error)
|
|
179
207
|
throw error
|
|
@@ -192,11 +220,14 @@ export class BullMQSchedulerService {
|
|
|
192
220
|
|
|
193
221
|
// Get all BullMQ repeatable jobs
|
|
194
222
|
const repeatableJobs = await queue.getRepeatableJobs?.() || []
|
|
195
|
-
const bullmqScheduleIds = new Set<string>(
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
223
|
+
const bullmqScheduleIds = new Set<string>()
|
|
224
|
+
const bullmqScheduleCounts = new Map<string, number>()
|
|
225
|
+
for (const job of repeatableJobs) {
|
|
226
|
+
const scheduleId = this.getScheduleIdFromRepeatableJob(job)
|
|
227
|
+
if (!scheduleId) continue
|
|
228
|
+
bullmqScheduleIds.add(scheduleId)
|
|
229
|
+
bullmqScheduleCounts.set(scheduleId, (bullmqScheduleCounts.get(scheduleId) ?? 0) + 1)
|
|
230
|
+
}
|
|
200
231
|
|
|
201
232
|
// Get enabled schedules from database in batches to avoid unbounded loads
|
|
202
233
|
const BATCH_SIZE = 500
|
|
@@ -219,6 +250,9 @@ export class BullMQSchedulerService {
|
|
|
219
250
|
if (!bullmqScheduleIds.has(schedule.id)) {
|
|
220
251
|
console.debug(`[scheduler:bullmq] Registering missing schedule: ${schedule.name}`)
|
|
221
252
|
await this.register(schedule)
|
|
253
|
+
} else if ((bullmqScheduleCounts.get(schedule.id) ?? 0) > 1) {
|
|
254
|
+
console.log(`[scheduler:bullmq] Repairing duplicate repeatable jobs for schedule: ${schedule.id}`)
|
|
255
|
+
await this.register(schedule)
|
|
222
256
|
}
|
|
223
257
|
}
|
|
224
258
|
|
|
@@ -243,7 +277,10 @@ export class BullMQSchedulerService {
|
|
|
243
277
|
|
|
244
278
|
if (schedule.scheduleType === 'cron') {
|
|
245
279
|
// Validate cron expression
|
|
246
|
-
parseCronExpression(schedule.scheduleValue, schedule.timezone || 'UTC')
|
|
280
|
+
const parsedCron = parseCronExpression(schedule.scheduleValue, schedule.timezone || 'UTC')
|
|
281
|
+
if (!parsedCron.isValid) {
|
|
282
|
+
throw new Error(parsedCron.error || 'Invalid cron expression')
|
|
283
|
+
}
|
|
247
284
|
opts.pattern = schedule.scheduleValue
|
|
248
285
|
} else if (schedule.scheduleType === 'interval') {
|
|
249
286
|
// Parse interval (e.g., "15m", "2h", "1d")
|