@monque/core 1.5.2 → 1.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@monque/core",
-	"version": "1.5.2",
+	"version": "1.6.0",
 	"description": "MongoDB-backed job scheduler with atomic locking, exponential backoff, and cron scheduling",
 	"author": "Maurice de Bruyn <debruyn.maurice@gmail.com>",
 	"repository": {
@@ -16,7 +16,7 @@
 	"sideEffects": false,
 	"type": "module",
 	"engines": {
-		"node": ">=22.0.0"
+		"node": ">=22.12.0"
 	},
 	"publishConfig": {
 		"access": "public"
@@ -79,10 +79,10 @@
 		"@testcontainers/mongodb": "^11.12.0",
 		"@total-typescript/ts-reset": "^0.6.1",
 		"@types/node": "^22.19.15",
-		"@vitest/coverage-v8": "^4.0.18",
+		"@vitest/coverage-v8": "^4.1.0",
 		"fishery": "^2.4.0",
 		"mongodb": "^7.1.0",
-		"tsdown": "^0.21.1",
-		"vitest": "^4.0.18"
+		"tsdown": "^0.21.2",
+		"vitest": "^4.1.0"
 	}
 }

package/src/scheduler/monque.ts

@@ -39,6 +39,7 @@ import type { MonqueOptions } from './types.js';
 const DEFAULTS = {
 	collectionName: 'monque_jobs',
 	pollInterval: 1000,
+	safetyPollInterval: 30_000,
 	maxRetries: 10,
 	baseRetryInterval: 1000,
 	shutdownTimeout: 30000,
@@ -135,6 +136,7 @@ export class Monque extends EventEmitter {
 		this.options = {
 			collectionName: options.collectionName ?? DEFAULTS.collectionName,
 			pollInterval: options.pollInterval ?? DEFAULTS.pollInterval,
+			safetyPollInterval: options.safetyPollInterval ?? DEFAULTS.safetyPollInterval,
 			maxRetries: options.maxRetries ?? DEFAULTS.maxRetries,
 			baseRetryInterval: options.baseRetryInterval ?? DEFAULTS.baseRetryInterval,
 			shutdownTimeout: options.shutdownTimeout ?? DEFAULTS.shutdownTimeout,
@@ -186,7 +188,9 @@ export class Monque extends EventEmitter {
 			this._manager = new JobManager(ctx);
 			this._query = new JobQueryService(ctx);
 			this._processor = new JobProcessor(ctx);
-			this._changeStreamHandler = new ChangeStreamHandler(ctx, () => this.processor.poll());
+			this._changeStreamHandler = new ChangeStreamHandler(ctx, (targetNames) =>
+				this.handleChangeStreamPoll(targetNames),
+			);
 			this._lifecycleManager = new LifecycleManager(ctx);
 
 			this.isInitialized = true;
@@ -255,6 +259,18 @@ export class Monque extends EventEmitter {
 		return this._lifecycleManager;
 	}
 
+	/**
+	 * Handle a change-stream-triggered poll and reset the safety poll timer.
+	 *
+	 * Used as the `onPoll` callback for {@link ChangeStreamHandler}. Runs a
+	 * targeted poll for the given worker names, then resets the adaptive safety
+	 * poll timer so it doesn't fire redundantly.
+	 */
+	private async handleChangeStreamPoll(targetNames?: ReadonlySet<string>): Promise<void> {
+		await this.processor.poll(targetNames);
+		this.lifecycleManager.resetPollTimer();
+	}
+
 	/**
 	 * Build the shared context for internal services.
 	 */
@@ -271,6 +287,13 @@ export class Monque extends EventEmitter {
 			isRunning: () => this.isRunning,
 			emit: <K extends keyof MonqueEventMap>(event: K, payload: MonqueEventMap[K]) =>
 				this.emit(event, payload),
+			notifyPendingJob: (name: string, nextRunAt: Date) => {
+				if (!this.isRunning || !this._changeStreamHandler) {
+					return;
+				}
+
+				this._changeStreamHandler.notifyPendingJob(name, nextRunAt);
+			},
 			documentToPersistedJob: <T>(doc: WithId<Document>) => documentToPersistedJob<T>(doc),
 		};
 	}
@@ -1032,6 +1055,7 @@ export class Monque extends EventEmitter {
 		this.lifecycleManager.startTimers({
 			poll: () => this.processor.poll(),
 			updateHeartbeats: () => this.processor.updateHeartbeats(),
+			isChangeStreamActive: () => this.changeStreamHandler.isActive(),
 		});
 	}
 

package/src/scheduler/services/change-stream-handler.ts

@@ -5,12 +5,22 @@ import { toError } from '@/shared';
 
 import type { SchedulerContext } from './types.js';
 
+/** Minimum poll interval floor to prevent tight loops (ms) */
+const MIN_POLL_INTERVAL = 100;
+
+/** Grace period after nextRunAt before scheduling a wakeup poll (ms) */
+const POLL_GRACE_PERIOD = 200;
+
 /**
  * Internal service for MongoDB Change Stream lifecycle.
  *
  * Provides real-time job notifications when available, with automatic
  * reconnection and graceful fallback to polling-only mode.
  *
+ * Leverages the full document from change stream events to:
+ * - Trigger **targeted polls** for specific workers (using the job `name`)
+ * - Schedule **precise wakeup timers** for future-dated jobs (using `nextRunAt`)
+ *
  * @internal Not part of public API.
  */
 export class ChangeStreamHandler {
@@ -32,9 +42,18 @@ export class ChangeStreamHandler {
 	/** Whether the scheduler is currently using change streams */
 	private usingChangeStreams = false;
 
+	/** Job names collected during the current debounce window for targeted polling */
+	private pendingTargetNames: Set<string> = new Set();
+
+	/** Wakeup timer for the earliest known future job */
+	private wakeupTimer: ReturnType<typeof setTimeout> | null = null;
+
+	/** Time of the currently scheduled wakeup */
+	private wakeupTime: Date | null = null;
+
 	constructor(
 		private readonly ctx: SchedulerContext,
-		private readonly onPoll: () => Promise<void>,
+		private readonly onPoll: (targetNames?: ReadonlySet<string>) => Promise<void>,
 	) {}
 
 	/**
@@ -67,7 +86,10 @@ export class ChangeStreamHandler {
 							{ operationType: 'insert' },
 							{
 								operationType: 'update',
-								'updateDescription.updatedFields.status': { $exists: true },
+								$or: [
+									{ 'updateDescription.updatedFields.status': { $exists: true } },
+									{ 'updateDescription.updatedFields.nextRunAt': { $exists: true } },
+								],
 							},
 						],
 					},
@@ -102,11 +124,20 @@ export class ChangeStreamHandler {
 	}
 
 	/**
-	 * Handle a change stream event by triggering a debounced poll.
+	 * Handle a change stream event using the full document for intelligent routing.
+	 *
+	 * For **immediate jobs** (`nextRunAt <= now`): collects the job name and triggers
+	 * a debounced targeted poll for only the relevant workers.
+	 *
+	 * For **future jobs** (`nextRunAt > now`): schedules a precise wakeup timer so
+	 * the job is picked up near its scheduled time without blind polling.
 	 *
-	 * Events are debounced to prevent "claim storms" when multiple changes arrive
-	 * in rapid succession (e.g., bulk job inserts). A 100ms debounce window
-	 * collects multiple events and triggers a single poll.
+	 * For **completed/failed jobs** (slot freed): triggers a targeted re-poll for that
+	 * worker so the next pending job is picked up immediately, maintaining continuous
+	 * throughput without waiting for the safety poll interval.
+	 *
+	 * Falls back to a full poll (no target names) if the document is missing
+	 * required fields.
 	 *
 	 * @param change - The change stream event document
 	 */
@@ -121,25 +152,121 @@ export class ChangeStreamHandler {
 
 		// Get fullDocument if available (for insert or with updateLookup option)
 		const fullDocument = 'fullDocument' in change ? change.fullDocument : undefined;
-		const isPendingStatus = fullDocument?.['status'] === JobStatus.PENDING;
+		const currentStatus = fullDocument?.['status'] as string | undefined;
+		const isPendingStatus = currentStatus === JobStatus.PENDING;
+
+		// A completed/failed status change means a concurrency slot was freed.
+		// Trigger a re-poll so the next pending job is picked up immediately,
+		// rather than waiting for the safety poll interval.
+		const isSlotFreed =
+			isUpdate && (currentStatus === JobStatus.COMPLETED || currentStatus === JobStatus.FAILED);
 
 		// For inserts: always trigger since new pending jobs need processing
-		// For updates: trigger if status changed to pending (retry/release scenario)
-		const shouldTrigger = isInsert || (isUpdate && isPendingStatus);
+		// For updates to pending: trigger (retry/release/recurring reschedule)
+		// For updates to completed/failed: trigger (concurrency slot freed)
+		const shouldTrigger = isInsert || (isUpdate && isPendingStatus) || isSlotFreed;
 
-		if (shouldTrigger) {
-			// Debounce poll triggers to avoid claim storms
-			if (this.debounceTimer) {
-				clearTimeout(this.debounceTimer);
+		if (!shouldTrigger) {
+			return;
+		}
+
+		// Slot-freed events: targeted poll for that worker to pick up waiting jobs
+		if (isSlotFreed) {
+			const jobName = fullDocument?.['name'] as string | undefined;
+			if (jobName) {
+				this.pendingTargetNames.add(jobName);
 			}
+			this.debouncedPoll();
+			return;
+		}
+
+		// Extract job metadata from the full document for smart routing
+		const jobName = fullDocument?.['name'] as string | undefined;
+		const nextRunAt = fullDocument?.['nextRunAt'] as Date | undefined;
+
+		if (jobName && nextRunAt) {
+			this.notifyPendingJob(jobName, nextRunAt);
+			return;
+		}
+
+		// Immediate job or missing metadata — collect for targeted/full poll
+		if (jobName) {
+			this.pendingTargetNames.add(jobName);
+		}
+		this.debouncedPoll();
+	}
+
+	/**
+	 * Notify the handler about a pending job created or updated by this process.
+	 *
+	 * Reuses the same routing logic as change stream events so local writes don't
+	 * depend on the MongoDB change stream cursor already being fully ready.
+	 *
+	 * @param jobName - Worker name for targeted polling
+	 * @param nextRunAt - When the job becomes eligible for processing
+	 */
+	notifyPendingJob(jobName: string, nextRunAt: Date): void {
+		if (!this.ctx.isRunning()) {
+			return;
+		}
 
-			this.debounceTimer = setTimeout(() => {
-				this.debounceTimer = null;
-				this.onPoll().catch((error: unknown) => {
-					this.ctx.emit('job:error', { error: toError(error) });
-				});
-			}, 100);
+		if (nextRunAt.getTime() > Date.now()) {
+			this.scheduleWakeup(nextRunAt);
+			return;
 		}
+
+		this.pendingTargetNames.add(jobName);
+		this.debouncedPoll();
+	}
+
+	/**
+	 * Schedule a debounced poll with collected target names.
+	 *
+	 * Collects job names from multiple change stream events during the debounce
+	 * window, then triggers a single targeted poll for only those workers.
+	 */
+	private debouncedPoll(): void {
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer);
+		}
+
+		this.debounceTimer = setTimeout(() => {
+			this.debounceTimer = null;
+			const names = this.pendingTargetNames.size > 0 ? new Set(this.pendingTargetNames) : undefined;
+			this.pendingTargetNames.clear();
+			this.onPoll(names).catch((error: unknown) => {
+				this.ctx.emit('job:error', { error: toError(error) });
+			});
+		}, 100);
+	}
+
+	/**
+	 * Schedule a wakeup timer for a future-dated job.
+	 *
+	 * Maintains a single timer set to the earliest known future job's `nextRunAt`.
+	 * When the timer fires, triggers a full poll to pick up all due jobs.
+	 *
+	 * @param nextRunAt - When the future job should become ready
+	 */
+	private scheduleWakeup(nextRunAt: Date): void {
+		// Only update if this job is earlier than the current wakeup
+		if (this.wakeupTime && nextRunAt >= this.wakeupTime) {
+			return;
+		}
+
+		this.clearWakeupTimer();
+		this.wakeupTime = nextRunAt;
+
+		const delay = Math.max(nextRunAt.getTime() - Date.now() + POLL_GRACE_PERIOD, MIN_POLL_INTERVAL);
+
+		this.wakeupTimer = setTimeout(() => {
+			this.wakeupTime = null;
+			this.wakeupTimer = null;
+			// Full poll — there may be multiple jobs due at this time
+			this.onPoll().catch((error: unknown) => {
+				this.ctx.emit('job:error', { error: toError(error) });
+			});
+		}, delay);
 	}
 
 	/**
@@ -158,12 +285,15 @@ export class ChangeStreamHandler {
 
 		this.reconnectAttempts++;
 
-		if (this.reconnectAttempts > this.maxReconnectAttempts) {
-			// Fall back to polling-only mode
-			this.usingChangeStreams = false;
+		// Immediately reset active state: clears stale debounce/wakeup timers,
+		// closes the broken cursor, and sets isActive() to false so the lifecycle
+		// manager switches to fast polling during backoff.
+		this.resetActiveState();
+		this.closeChangeStream();
 
+		if (this.reconnectAttempts > this.maxReconnectAttempts) {
+			// Permanent fallback to polling-only mode
 			this.clearReconnectTimer();
-			this.closeChangeStream();
 
 			this.ctx.emit('changestream:fallback', {
 				reason: `Exhausted ${this.maxReconnectAttempts} reconnection attempts: ${error.message}`,
@@ -211,6 +341,32 @@ export class ChangeStreamHandler {
 		this.reconnectTimer = null;
 	}
 
+	/**
+	 * Reset all active change stream state: clear debounce timer, wakeup timer,
+	 * pending target names, and mark as inactive.
+	 *
+	 * Does NOT close the cursor (callers handle sync vs async close) or clear
+	 * the reconnect timer/attempts (callers manage reconnection lifecycle).
+	 */
+	private resetActiveState(): void {
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer);
+			this.debounceTimer = null;
+		}
+
+		this.pendingTargetNames.clear();
+		this.clearWakeupTimer();
+		this.usingChangeStreams = false;
+	}
+
+	private clearWakeupTimer(): void {
+		if (this.wakeupTimer) {
+			clearTimeout(this.wakeupTimer);
+			this.wakeupTimer = null;
+		}
+		this.wakeupTime = null;
+	}
+
 	private closeChangeStream(): void {
 		if (!this.changeStream) {
 			return;
@@ -224,13 +380,10 @@ export class ChangeStreamHandler {
 	 * Close the change stream cursor and emit closed event.
 	 */
 	async close(): Promise<void> {
-		// Clear debounce timer
-		if (this.debounceTimer) {
-			clearTimeout(this.debounceTimer);
-			this.debounceTimer = null;
-		}
+		const wasActive = this.usingChangeStreams;
 
-		// Clear reconnection timer
+		// Clear all active state (debounce, wakeup, pending names, flag)
+		this.resetActiveState();
 		this.clearReconnectTimer();
 
 		if (this.changeStream) {
@@ -241,12 +394,11 @@ export class ChangeStreamHandler {
 			}
 			this.changeStream = null;
 
-			if (this.usingChangeStreams) {
+			if (wasActive) {
 				this.ctx.emit('changestream:closed', undefined);
 			}
 		}
 
-		this.usingChangeStreams = false;
 		this.reconnectAttempts = 0;
 	}
 

package/src/scheduler/services/job-manager.ts

@@ -147,6 +147,7 @@ export class JobManager {
 		}
 
 		const job = this.ctx.documentToPersistedJob(result);
+		this.ctx.notifyPendingJob(job.name, job.nextRunAt);
 		this.ctx.emit('job:retried', { job, previousStatus });
 		return job;
 	}
@@ -204,7 +205,9 @@ export class JobManager {
 			);
 		}
 
-		return this.ctx.documentToPersistedJob(result);
+		const job = this.ctx.documentToPersistedJob(result);
+		this.ctx.notifyPendingJob(job.name, job.nextRunAt);
+		return job;
 	}
 
 	/**

package/src/scheduler/services/job-processor.ts

@@ -16,6 +16,9 @@ export class JobProcessor {
 	/** Guard flag to prevent concurrent poll() execution */
 	private _isPolling = false;
 
+	/** Flag to request a re-poll after the current poll finishes */
+	private _repollRequested = false;
+
 	constructor(private readonly ctx: SchedulerContext) {}
 
 	/**
@@ -57,16 +60,34 @@ export class JobProcessor {
 	 * attempts to acquire jobs up to the worker's available concurrency slots.
 	 * Aborts early if the scheduler is stopping (`isRunning` is false) or if
 	 * the instance-level `instanceConcurrency` limit is reached.
+	 *
+	 * If a poll is requested while one is already running, it is queued and
+	 * executed as a full poll after the current one finishes. This prevents
+	 * change-stream-triggered polls from being silently dropped.
+	 *
+	 * @param targetNames - Optional set of worker names to poll. When provided, only the
+	 * specified workers are checked. Used by change stream handler for targeted polling.
 	 */
-	async poll(): Promise<void> {
-		if (!this.ctx.isRunning() || this._isPolling) {
+	async poll(targetNames?: ReadonlySet<string>): Promise<void> {
+		if (!this.ctx.isRunning()) {
+			return;
+		}
+
+		if (this._isPolling) {
+			// Queue a re-poll so work discovered during this poll isn't missed
+			this._repollRequested = true;
 			return;
 		}
 
 		this._isPolling = true;
 
 		try {
-			await this._doPoll();
+			do {
+				this._repollRequested = false;
+				await this._doPoll(targetNames);
+				// Re-polls are always full polls to catch all pending work
+				targetNames = undefined;
+			} while (this._repollRequested && this.ctx.isRunning());
 		} finally {
 			this._isPolling = false;
 		}
@@ -75,7 +96,7 @@ export class JobProcessor {
 	/**
 	 * Internal poll implementation.
 	 */
-	private async _doPoll(): Promise<void> {
+	private async _doPoll(targetNames?: ReadonlySet<string>): Promise<void> {
 		// Early exit if global instanceConcurrency is reached
 		const { instanceConcurrency } = this.ctx.options;
 
@@ -84,6 +105,11 @@ export class JobProcessor {
 		}
 
 		for (const [name, worker] of this.ctx.workers) {
+			// Skip workers not in the target set (if provided)
+			if (targetNames && !targetNames.has(name)) {
+				continue;
+			}
+
 			// Check if worker has capacity
 			const workerAvailableSlots = worker.concurrency - worker.activeJobs.size;
 
@@ -268,7 +294,13 @@ export class JobProcessor {
 				{ returnDocument: 'after' },
 			);
 
-			return result ? this.ctx.documentToPersistedJob(result) : null;
+			if (!result) {
+				return null;
+			}
+
+			const persistedJob = this.ctx.documentToPersistedJob(result);
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+			return persistedJob;
 		}
 
 		// One-time job - mark as completed
@@ -290,7 +322,12 @@ export class JobProcessor {
 			{ returnDocument: 'after' },
 		);
 
-		return result ? this.ctx.documentToPersistedJob(result) : null;
+		if (!result) {
+			return null;
+		}
+
+		const persistedJob = this.ctx.documentToPersistedJob(result);
+		return persistedJob;
 	}
 
 	/**

package/src/scheduler/services/job-scheduler.ts

@@ -141,12 +141,19 @@ export class JobScheduler {
 					throw new ConnectionError('Failed to enqueue job: findOneAndUpdate returned no document');
 				}
 
-				return this.ctx.documentToPersistedJob<T>(result);
+				const persistedJob = this.ctx.documentToPersistedJob<T>(result);
+				if (persistedJob.status === JobStatus.PENDING) {
+					this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+				}
+
+				return persistedJob;
 			}
 
 			const result = await this.ctx.collection.insertOne(job as Document);
+			const persistedJob = { ...job, _id: result.insertedId } as PersistedJob<T>;
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
 
-			return { ...job, _id: result.insertedId } as PersistedJob<T>;
+			return persistedJob;
 		} catch (error) {
 			if (error instanceof ConnectionError) {
 				throw error;
@@ -287,12 +294,19 @@ export class JobScheduler {
 					);
 				}
 
-				return this.ctx.documentToPersistedJob<T>(result);
+				const persistedJob = this.ctx.documentToPersistedJob<T>(result);
+				if (persistedJob.status === JobStatus.PENDING) {
+					this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+				}
+
+				return persistedJob;
 			}
 
 			const result = await this.ctx.collection.insertOne(job as Document);
+			const persistedJob = { ...job, _id: result.insertedId } as PersistedJob<T>;
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
 
-			return { ...job, _id: result.insertedId } as PersistedJob<T>;
+			return persistedJob;
 		} catch (error) {
 			if (error instanceof MonqueError) {
 				throw error;