@monque/core 1.5.2 → 1.7.0

package/src/scheduler/services/job-manager.ts

@@ -39,47 +39,48 @@ export class JobManager {
 
 		const _id = new ObjectId(jobId);
 
-		// Fetch job first to allow emitting the full job object in the event
-		const jobDoc = await this.ctx.collection.findOne({ _id });
-		if (!jobDoc) return null;
+		try {
+			const now = new Date();
+			const result = await this.ctx.collection.findOneAndUpdate(
+				{ _id, status: JobStatus.PENDING },
+				{
+					$set: {
+						status: JobStatus.CANCELLED,
+						updatedAt: now,
+					},
+				},
+				{ returnDocument: 'after' },
+			);
 
-		if (jobDoc['status'] === JobStatus.CANCELLED) {
-			return this.ctx.documentToPersistedJob(jobDoc);
-		}
+			if (!result) {
+				const jobDoc = await this.ctx.collection.findOne({ _id });
+				if (!jobDoc) return null;
 
-		if (jobDoc['status'] !== JobStatus.PENDING) {
-			throw new JobStateError(
-				`Cannot cancel job in status '${jobDoc['status']}'`,
-				jobId,
-				jobDoc['status'],
-				'cancel',
-			);
-		}
+				if (jobDoc['status'] === JobStatus.CANCELLED) {
+					return this.ctx.documentToPersistedJob(jobDoc);
+				}
 
-		const result = await this.ctx.collection.findOneAndUpdate(
-			{ _id, status: JobStatus.PENDING },
-			{
-				$set: {
-					status: JobStatus.CANCELLED,
-					updatedAt: new Date(),
-				},
-			},
-			{ returnDocument: 'after' },
-		);
-
-		if (!result) {
-			// Race condition: job changed state between check and update
-			throw new JobStateError(
-				'Job status changed during cancellation attempt',
-				jobId,
-				'unknown',
-				'cancel',
+				throw new JobStateError(
+					`Cannot cancel job in status '${jobDoc['status']}'`,
+					jobId,
+					jobDoc['status'],
+					'cancel',
+				);
+			}
+
+			const job = this.ctx.documentToPersistedJob(result);
+			this.ctx.emit('job:cancelled', { job });
+			return job;
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during cancelJob';
+			throw new ConnectionError(
+				`Failed to cancel job: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
 			);
 		}
-
-		const job = this.ctx.documentToPersistedJob(result);
-		this.ctx.emit('job:cancelled', { job });
-		return job;
 	}
 
 	/**
@@ -104,51 +105,69 @@ export class JobManager {
 		if (!ObjectId.isValid(jobId)) return null;
 
 		const _id = new ObjectId(jobId);
-		const currentJob = await this.ctx.collection.findOne({ _id });
-
-		if (!currentJob) return null;
 
-		if (currentJob['status'] !== JobStatus.FAILED && currentJob['status'] !== JobStatus.CANCELLED) {
-			throw new JobStateError(
-				`Cannot retry job in status '${currentJob['status']}'`,
-				jobId,
-				currentJob['status'],
-				'retry',
+		try {
+			const now = new Date();
+			const result = await this.ctx.collection.findOneAndUpdate(
+				{
+					_id,
+					status: { $in: [JobStatus.FAILED, JobStatus.CANCELLED] },
+				},
+				{
+					$set: {
+						status: JobStatus.PENDING,
+						failCount: 0,
+						nextRunAt: now,
+						updatedAt: now,
+					},
+					$unset: {
+						failReason: '',
+						lockedAt: '',
+						claimedBy: '',
+						lastHeartbeat: '',
+					},
+				},
+				{ returnDocument: 'before' },
 			);
-		}
 
-		const previousStatus = currentJob['status'] as 'failed' | 'cancelled';
+			if (!result) {
+				const currentJob = await this.ctx.collection.findOne({ _id });
+				if (!currentJob) return null;
 
-		const result = await this.ctx.collection.findOneAndUpdate(
-			{
-				_id,
-				status: { $in: [JobStatus.FAILED, JobStatus.CANCELLED] },
-			},
-			{
-				$set: {
-					status: JobStatus.PENDING,
-					failCount: 0,
-					nextRunAt: new Date(),
-					updatedAt: new Date(),
-				},
-				$unset: {
-					failReason: '',
-					lockedAt: '',
-					claimedBy: '',
-					lastHeartbeat: '',
-					heartbeatInterval: '',
-				},
-			},
-			{ returnDocument: 'after' },
-		);
+				throw new JobStateError(
+					`Cannot retry job in status '${currentJob['status']}'`,
+					jobId,
+					currentJob['status'],
+					'retry',
+				);
+			}
 
-		if (!result) {
-			throw new JobStateError('Job status changed during retry attempt', jobId, 'unknown', 'retry');
+			const previousStatus = result['status'] as 'failed' | 'cancelled';
+
+			const updatedDoc = { ...result };
+			updatedDoc['status'] = JobStatus.PENDING;
+			updatedDoc['failCount'] = 0;
+			updatedDoc['nextRunAt'] = now;
+			updatedDoc['updatedAt'] = now;
+			delete updatedDoc['failReason'];
+			delete updatedDoc['lockedAt'];
+			delete updatedDoc['claimedBy'];
+			delete updatedDoc['lastHeartbeat'];
+
+			const job = this.ctx.documentToPersistedJob(updatedDoc);
+			this.ctx.notifyPendingJob(job.name, job.nextRunAt);
+			this.ctx.emit('job:retried', { job, previousStatus });
+			return job;
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during retryJob';
+			throw new ConnectionError(
+				`Failed to retry job: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
 		}
-
-		const job = this.ctx.documentToPersistedJob(result);
-		this.ctx.emit('job:retried', { job, previousStatus });
-		return job;
 	}
 
 	/**
@@ -171,40 +190,45 @@ export class JobManager {
 		if (!ObjectId.isValid(jobId)) return null;
 
 		const _id = new ObjectId(jobId);
-		const currentJobDoc = await this.ctx.collection.findOne({ _id });
-
-		if (!currentJobDoc) return null;
 
-		if (currentJobDoc['status'] !== JobStatus.PENDING) {
-			throw new JobStateError(
-				`Cannot reschedule job in status '${currentJobDoc['status']}'`,
-				jobId,
-				currentJobDoc['status'],
-				'reschedule',
+		try {
+			const now = new Date();
+			const result = await this.ctx.collection.findOneAndUpdate(
+				{ _id, status: JobStatus.PENDING },
+				{
+					$set: {
+						nextRunAt: runAt,
+						updatedAt: now,
+					},
+				},
+				{ returnDocument: 'after' },
 			);
-		}
 
-		const result = await this.ctx.collection.findOneAndUpdate(
-			{ _id, status: JobStatus.PENDING },
-			{
-				$set: {
-					nextRunAt: runAt,
-					updatedAt: new Date(),
-				},
-			},
-			{ returnDocument: 'after' },
-		);
-
-		if (!result) {
-			throw new JobStateError(
-				'Job status changed during reschedule attempt',
-				jobId,
-				'unknown',
-				'reschedule',
+			if (!result) {
+				const currentJobDoc = await this.ctx.collection.findOne({ _id });
+				if (!currentJobDoc) return null;
+
+				throw new JobStateError(
+					`Cannot reschedule job in status '${currentJobDoc['status']}'`,
+					jobId,
+					currentJobDoc['status'],
+					'reschedule',
+				);
+			}
+
+			const job = this.ctx.documentToPersistedJob(result);
+			this.ctx.notifyPendingJob(job.name, job.nextRunAt);
+			return job;
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during rescheduleJob';
+			throw new ConnectionError(
+				`Failed to reschedule job: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
 			);
 		}
-
-		return this.ctx.documentToPersistedJob(result);
 	}
 
 	/**
@@ -229,14 +253,25 @@ export class JobManager {
 
 		const _id = new ObjectId(jobId);
 
-		const result = await this.ctx.collection.deleteOne({ _id });
+		try {
+			const result = await this.ctx.collection.deleteOne({ _id });
 
-		if (result.deletedCount > 0) {
-			this.ctx.emit('job:deleted', { jobId });
-			return true;
-		}
+			if (result.deletedCount > 0) {
+				this.ctx.emit('job:deleted', { jobId });
+				return true;
+			}
 
-		return false;
+			return false;
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during deleteJob';
+			throw new ConnectionError(
+				`Failed to delete job: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
 	}
 
 	// ─────────────────────────────────────────────────────────────────────────────
@@ -276,10 +311,11 @@ export class JobManager {
 		query['status'] = JobStatus.PENDING;
 
 		try {
+			const now = new Date();
 			const result = await this.ctx.collection.updateMany(query, {
 				$set: {
 					status: JobStatus.CANCELLED,
-					updatedAt: new Date(),
+					updatedAt: now,
 				},
 			});
 
@@ -343,19 +379,20 @@ export class JobManager {
 		const spreadWindowMs = 30_000; // 30s max spread for staggered retry
 
 		try {
+			const now = new Date();
 			const result = await this.ctx.collection.updateMany(query, [
 				{
 					$set: {
 						status: JobStatus.PENDING,
 						failCount: 0,
 						nextRunAt: {
-							$add: [new Date(), { $multiply: [{ $rand: {} }, spreadWindowMs] }],
+							$add: [now, { $multiply: [{ $rand: {} }, spreadWindowMs] }],
 						},
-						updatedAt: new Date(),
+						updatedAt: now,
 					},
 				},
 				{
-					$unset: ['failReason', 'lockedAt', 'claimedBy', 'lastHeartbeat', 'heartbeatInterval'],
+					$unset: ['failReason', 'lockedAt', 'claimedBy', 'lastHeartbeat'],
 				},
 			]);
 

package/src/scheduler/services/job-processor.ts

@@ -16,20 +16,19 @@ export class JobProcessor {
 	/** Guard flag to prevent concurrent poll() execution */
 	private _isPolling = false;
 
-	constructor(private readonly ctx: SchedulerContext) {}
+	/** Flag to request a re-poll after the current poll finishes */
+	private _repollRequested = false;
 
 	/**
-	 * Get the total number of active jobs across all workers.
+	 * O(1) counter tracking the total number of active jobs across all workers.
 	 *
-	 * Used for instance-level throttling when `instanceConcurrency` is configured.
+	 * Incremented when a job is added to `worker.activeJobs` in `_doPoll`,
+	 * decremented in the `processJob` finally block. Replaces the previous
+	 * O(workers) loop in `getTotalActiveJobs()` for instance-level throttling.
 	 */
-	private getTotalActiveJobs(): number {
-		let total = 0;
-		for (const worker of this.ctx.workers.values()) {
-			total += worker.activeJobs.size;
-		}
-		return total;
-	}
+	private _totalActiveJobs = 0;
+
+	constructor(private readonly ctx: SchedulerContext) {}
 
 	/**
 	 * Get the number of available slots considering the global instanceConcurrency limit.
@@ -44,8 +43,7 @@ export class JobProcessor {
 			return workerAvailableSlots;
 		}
 
-		const totalActive = this.getTotalActiveJobs();
-		const globalAvailable = instanceConcurrency - totalActive;
+		const globalAvailable = instanceConcurrency - this._totalActiveJobs;
 
 		return Math.min(workerAvailableSlots, globalAvailable);
 	}
@@ -57,16 +55,34 @@ export class JobProcessor {
 	 * attempts to acquire jobs up to the worker's available concurrency slots.
 	 * Aborts early if the scheduler is stopping (`isRunning` is false) or if
 	 * the instance-level `instanceConcurrency` limit is reached.
+	 *
+	 * If a poll is requested while one is already running, it is queued and
+	 * executed as a full poll after the current one finishes. This prevents
+	 * change-stream-triggered polls from being silently dropped.
+	 *
+	 * @param targetNames - Optional set of worker names to poll. When provided, only the
+	 * specified workers are checked. Used by change stream handler for targeted polling.
 	 */
-	async poll(): Promise<void> {
-		if (!this.ctx.isRunning() || this._isPolling) {
+	async poll(targetNames?: ReadonlySet<string>): Promise<void> {
+		if (!this.ctx.isRunning()) {
+			return;
+		}
+
+		if (this._isPolling) {
+			// Queue a re-poll so work discovered during this poll isn't missed
+			this._repollRequested = true;
 			return;
 		}
 
 		this._isPolling = true;
 
 		try {
-			await this._doPoll();
+			do {
+				this._repollRequested = false;
+				await this._doPoll(targetNames);
+				// Re-polls are always full polls to catch all pending work
+				targetNames = undefined;
+			} while (this._repollRequested && this.ctx.isRunning());
 		} finally {
 			this._isPolling = false;
 		}
@@ -75,15 +91,20 @@ export class JobProcessor {
 	/**
 	 * Internal poll implementation.
 	 */
-	private async _doPoll(): Promise<void> {
+	private async _doPoll(targetNames?: ReadonlySet<string>): Promise<void> {
 		// Early exit if global instanceConcurrency is reached
 		const { instanceConcurrency } = this.ctx.options;
 
-		if (instanceConcurrency !== undefined && this.getTotalActiveJobs() >= instanceConcurrency) {
+		if (instanceConcurrency !== undefined && this._totalActiveJobs >= instanceConcurrency) {
 			return;
 		}
 
 		for (const [name, worker] of this.ctx.workers) {
+			// Skip workers not in the target set (if provided)
+			if (targetNames && !targetNames.has(name)) {
+				continue;
+			}
+
 			// Check if worker has capacity
 			const workerAvailableSlots = worker.concurrency - worker.activeJobs.size;
 
@@ -99,31 +120,57 @@ export class JobProcessor {
 				return;
 			}
 
-			// Try to acquire jobs up to available slots
+			// Try to acquire jobs up to available slots in parallel
+			if (!this.ctx.isRunning()) {
+				return;
+			}
+
+			const acquisitionPromises: Promise<void>[] = [];
 			for (let i = 0; i < availableSlots; i++) {
-				if (!this.ctx.isRunning()) {
-					return;
-				}
-
-				// Re-check global limit before each acquisition
-				if (instanceConcurrency !== undefined && this.getTotalActiveJobs() >= instanceConcurrency) {
-					return;
-				}
-
-				const job = await this.acquireJob(name);
-
-				if (job) {
-					// Add to activeJobs immediately to correctly track concurrency
-					worker.activeJobs.set(job._id.toString(), job);
-
-					this.processJob(job, worker).catch((error: unknown) => {
-						this.ctx.emit('job:error', { error: toError(error), job });
-					});
-				} else {
-					// No more jobs available for this worker
-					break;
-				}
+				acquisitionPromises.push(
+					this.acquireJob(name)
+						.then(async (job) => {
+							if (!job) {
+								return;
+							}
+
+							if (this.ctx.isRunning()) {
+								// Add to activeJobs immediately to correctly track concurrency
+								worker.activeJobs.set(job._id.toString(), job);
+								this._totalActiveJobs++;
+
+								this.processJob(job, worker).catch((error: unknown) => {
+									this.ctx.emit('job:error', { error: toError(error), job });
+								});
+							} else {
+								// Revert claim if shut down while acquiring
+								try {
+									await this.ctx.collection.updateOne(
+										{ _id: job._id, status: JobStatus.PROCESSING, claimedBy: this.ctx.instanceId },
+										{
+											$set: {
+												status: JobStatus.PENDING,
+												updatedAt: new Date(),
+											},
+											$unset: {
+												lockedAt: '',
+												claimedBy: '',
+												lastHeartbeat: '',
+											},
+										},
+									);
+								} catch (error) {
+									this.ctx.emit('job:error', { error: toError(error) });
+								}
+							}
+						})
+						.catch((error: unknown) => {
+							this.ctx.emit('job:error', { error: toError(error) });
+						}),
+				);
 			}
+
+			await Promise.allSettled(acquisitionPromises);
 		}
 	}
 
@@ -171,10 +218,6 @@ export class JobProcessor {
 			},
 		);
 
-		if (!this.ctx.isRunning()) {
-			return null;
-		}
-
 		if (!result) {
 			return null;
 		}
@@ -222,6 +265,8 @@ export class JobProcessor {
 			}
 		} finally {
 			worker.activeJobs.delete(jobId);
+			this._totalActiveJobs--;
+			this.ctx.notifyJobFinished();
 		}
 	}
 
@@ -245,6 +290,8 @@ export class JobProcessor {
 			return null;
 		}
 
+		const now = new Date();
+
 		if (job.repeatInterval) {
 			// Recurring job - schedule next run
 			const nextRunAt = getNextCronDate(job.repeatInterval);
@@ -255,20 +302,25 @@ export class JobProcessor {
 						status: JobStatus.PENDING,
 						nextRunAt,
 						failCount: 0,
-						updatedAt: new Date(),
+						updatedAt: now,
 					},
 					$unset: {
 						lockedAt: '',
 						claimedBy: '',
 						lastHeartbeat: '',
-						heartbeatInterval: '',
 						failReason: '',
 					},
 				},
 				{ returnDocument: 'after' },
 			);
 
-			return result ? this.ctx.documentToPersistedJob(result) : null;
+			if (!result) {
+				return null;
+			}
+
+			const persistedJob = this.ctx.documentToPersistedJob(result);
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+			return persistedJob;
 		}
 
 		// One-time job - mark as completed
@@ -277,20 +329,24 @@ export class JobProcessor {
 			{
 				$set: {
 					status: JobStatus.COMPLETED,
-					updatedAt: new Date(),
+					updatedAt: now,
 				},
 				$unset: {
 					lockedAt: '',
 					claimedBy: '',
 					lastHeartbeat: '',
-					heartbeatInterval: '',
 					failReason: '',
 				},
 			},
 			{ returnDocument: 'after' },
 		);
 
-		return result ? this.ctx.documentToPersistedJob(result) : null;
+		if (!result) {
+			return null;
+		}
+
+		const persistedJob = this.ctx.documentToPersistedJob(result);
+		return persistedJob;
 	}
 
 	/**
@@ -316,6 +372,7 @@ export class JobProcessor {
 			return null;
 		}
 
+		const now = new Date();
 		const newFailCount = job.failCount + 1;
 
 		if (newFailCount >= this.ctx.options.maxRetries) {
@@ -327,13 +384,12 @@ export class JobProcessor {
 						status: JobStatus.FAILED,
 						failCount: newFailCount,
 						failReason: error.message,
-						updatedAt: new Date(),
+						updatedAt: now,
 					},
 					$unset: {
 						lockedAt: '',
 						claimedBy: '',
 						lastHeartbeat: '',
-						heartbeatInterval: '',
 					},
 				},
 				{ returnDocument: 'after' },
@@ -357,13 +413,12 @@ export class JobProcessor {
 					failCount: newFailCount,
 					failReason: error.message,
 					nextRunAt,
-					updatedAt: new Date(),
+					updatedAt: now,
 				},
 				$unset: {
 					lockedAt: '',
 					claimedBy: '',
 					lastHeartbeat: '',
-					heartbeatInterval: '',
 				},
 			},
 			{ returnDocument: 'after' },