@monque/core 1.5.2 → 1.6.0

package/dist/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @monque/core
 
+## 1.6.0
+
+### Minor Changes
+
+- [#232](https://github.com/ueberBrot/monque/pull/232) [`c3d2c83`](https://github.com/ueberBrot/monque/commit/c3d2c83b89d3fc8e77ec1958695d05b68f357d8d) Thanks [@ueberBrot](https://github.com/ueberBrot)! - Add adaptive poll scheduling and targeted change stream processing
+
+  - **Adaptive polling**: When change streams are active, safety polling runs at `safetyPollInterval` (default 30s) instead of the fast `pollInterval`. Falls back to `pollInterval` when change streams are unavailable.
+  - **Targeted polling**: Change stream events now leverage the full document to poll only the specific worker(s) for the affected job type, skipping unrelated workers.
+  - **Wakeup timers**: Future-dated jobs (`nextRunAt > now`) get a precise wakeup timer instead of waiting for the next poll cycle.
+  - **Local pending-job notifications**: Jobs created or moved back to `pending` by the local scheduler now trigger the same targeted polling and wakeup-timer path immediately, avoiding startup races before the change stream cursor is fully ready.
+  - **Slot-freed re-polling**: When a job completes or permanently fails, a targeted re-poll immediately picks up the next waiting job for that worker.
+  - **Re-poll queuing**: Poll requests arriving while a poll is running are queued and executed after, preventing silently dropped change-stream-triggered polls.
+  - New configuration option: `safetyPollInterval` (default: 30000ms).
+
 ## 1.5.2
 
 ### Patch Changes

package/dist/index.cjs

@@ -714,12 +714,20 @@ function decodeCursor(cursor) {
 }
 //#endregion
 //#region src/scheduler/services/change-stream-handler.ts
+/** Minimum poll interval floor to prevent tight loops (ms) */
+const MIN_POLL_INTERVAL = 100;
+/** Grace period after nextRunAt before scheduling a wakeup poll (ms) */
+const POLL_GRACE_PERIOD = 200;
 /**
 * Internal service for MongoDB Change Stream lifecycle.
 *
 * Provides real-time job notifications when available, with automatic
 * reconnection and graceful fallback to polling-only mode.
 *
+* Leverages the full document from change stream events to:
+* - Trigger **targeted polls** for specific workers (using the job `name`)
+* - Schedule **precise wakeup timers** for future-dated jobs (using `nextRunAt`)
+*
 * @internal Not part of public API.
 */
 var ChangeStreamHandler = class {
@@ -735,6 +743,12 @@ var ChangeStreamHandler = class {
 	reconnectTimer = null;
 	/** Whether the scheduler is currently using change streams */
 	usingChangeStreams = false;
+	/** Job names collected during the current debounce window for targeted polling */
+	pendingTargetNames = /* @__PURE__ */ new Set();
+	/** Wakeup timer for the earliest known future job */
+	wakeupTimer = null;
+	/** Time of the currently scheduled wakeup */
+	wakeupTime = null;
 	constructor(ctx, onPoll) {
 		this.ctx = ctx;
 		this.onPoll = onPoll;
@@ -759,7 +773,7 @@ var ChangeStreamHandler = class {
 		try {
 			this.changeStream = this.ctx.collection.watch([{ $match: { $or: [{ operationType: "insert" }, {
 				operationType: "update",
-				"updateDescription.updatedFields.status": { $exists: true }
+				$or: [{ "updateDescription.updatedFields.status": { $exists: true } }, { "updateDescription.updatedFields.nextRunAt": { $exists: true } }]
 			}] } }], { fullDocument: "updateLookup" });
 			this.changeStream.on("change", (change) => {
 				this.handleEvent(change);
@@ -778,11 +792,20 @@ var ChangeStreamHandler = class {
 		}
 	}
 	/**
-	* Handle a change stream event by triggering a debounced poll.
+	* Handle a change stream event using the full document for intelligent routing.
+	*
+	* For **immediate jobs** (`nextRunAt <= now`): collects the job name and triggers
+	* a debounced targeted poll for only the relevant workers.
+	*
+	* For **future jobs** (`nextRunAt > now`): schedules a precise wakeup timer so
+	* the job is picked up near its scheduled time without blind polling.
 	*
-	* Events are debounced to prevent "claim storms" when multiple changes arrive
-	* in rapid succession (e.g., bulk job inserts). A 100ms debounce window
-	* collects multiple events and triggers a single poll.
+	* For **completed/failed jobs** (slot freed): triggers a targeted re-poll for that
+	* worker so the next pending job is picked up immediately, maintaining continuous
+	* throughput without waiting for the safety poll interval.
+	*
+	* Falls back to a full poll (no target names) if the document is missing
+	* required fields.
 	*
 	* @param change - The change stream event document
 	*/
@@ -790,16 +813,81 @@ var ChangeStreamHandler = class {
 		if (!this.ctx.isRunning()) return;
 		const isInsert = change.operationType === "insert";
 		const isUpdate = change.operationType === "update";
-		const isPendingStatus = ("fullDocument" in change ? change.fullDocument : void 0)?.["status"] === JobStatus.PENDING;
-		if (isInsert || isUpdate && isPendingStatus) {
-			if (this.debounceTimer) clearTimeout(this.debounceTimer);
-			this.debounceTimer = setTimeout(() => {
-				this.debounceTimer = null;
-				this.onPoll().catch((error) => {
-					this.ctx.emit("job:error", { error: toError(error) });
-				});
-			}, 100);
+		const fullDocument = "fullDocument" in change ? change.fullDocument : void 0;
+		const currentStatus = fullDocument?.["status"];
+		const isPendingStatus = currentStatus === JobStatus.PENDING;
+		const isSlotFreed = isUpdate && (currentStatus === JobStatus.COMPLETED || currentStatus === JobStatus.FAILED);
+		if (!(isInsert || isUpdate && isPendingStatus || isSlotFreed)) return;
+		if (isSlotFreed) {
+			const jobName = fullDocument?.["name"];
+			if (jobName) this.pendingTargetNames.add(jobName);
+			this.debouncedPoll();
+			return;
+		}
+		const jobName = fullDocument?.["name"];
+		const nextRunAt = fullDocument?.["nextRunAt"];
+		if (jobName && nextRunAt) {
+			this.notifyPendingJob(jobName, nextRunAt);
+			return;
 		}
+		if (jobName) this.pendingTargetNames.add(jobName);
+		this.debouncedPoll();
+	}
+	/**
+	* Notify the handler about a pending job created or updated by this process.
+	*
+	* Reuses the same routing logic as change stream events so local writes don't
+	* depend on the MongoDB change stream cursor already being fully ready.
+	*
+	* @param jobName - Worker name for targeted polling
+	* @param nextRunAt - When the job becomes eligible for processing
+	*/
+	notifyPendingJob(jobName, nextRunAt) {
+		if (!this.ctx.isRunning()) return;
+		if (nextRunAt.getTime() > Date.now()) {
+			this.scheduleWakeup(nextRunAt);
+			return;
+		}
+		this.pendingTargetNames.add(jobName);
+		this.debouncedPoll();
+	}
+	/**
+	* Schedule a debounced poll with collected target names.
+	*
+	* Collects job names from multiple change stream events during the debounce
+	* window, then triggers a single targeted poll for only those workers.
+	*/
+	debouncedPoll() {
+		if (this.debounceTimer) clearTimeout(this.debounceTimer);
+		this.debounceTimer = setTimeout(() => {
+			this.debounceTimer = null;
+			const names = this.pendingTargetNames.size > 0 ? new Set(this.pendingTargetNames) : void 0;
+			this.pendingTargetNames.clear();
+			this.onPoll(names).catch((error) => {
+				this.ctx.emit("job:error", { error: toError(error) });
+			});
+		}, 100);
+	}
+	/**
+	* Schedule a wakeup timer for a future-dated job.
+	*
+	* Maintains a single timer set to the earliest known future job's `nextRunAt`.
+	* When the timer fires, triggers a full poll to pick up all due jobs.
+	*
+	* @param nextRunAt - When the future job should become ready
+	*/
+	scheduleWakeup(nextRunAt) {
+		if (this.wakeupTime && nextRunAt >= this.wakeupTime) return;
+		this.clearWakeupTimer();
+		this.wakeupTime = nextRunAt;
+		const delay = Math.max(nextRunAt.getTime() - Date.now() + POLL_GRACE_PERIOD, MIN_POLL_INTERVAL);
+		this.wakeupTimer = setTimeout(() => {
+			this.wakeupTime = null;
+			this.wakeupTimer = null;
+			this.onPoll().catch((error) => {
+				this.ctx.emit("job:error", { error: toError(error) });
+			});
+		}, delay);
 	}
 	/**
 	* Handle change stream errors with exponential backoff reconnection.
@@ -813,10 +901,10 @@ var ChangeStreamHandler = class {
 	handleError(error) {
 		if (!this.ctx.isRunning()) return;
 		this.reconnectAttempts++;
+		this.resetActiveState();
+		this.closeChangeStream();
 		if (this.reconnectAttempts > this.maxReconnectAttempts) {
-			this.usingChangeStreams = false;
 			this.clearReconnectTimer();
-			this.closeChangeStream();
 			this.ctx.emit("changestream:fallback", { reason: `Exhausted ${this.maxReconnectAttempts} reconnection attempts: ${error.message}` });
 			return;
 		}
@@ -839,6 +927,29 @@ var ChangeStreamHandler = class {
 		clearTimeout(this.reconnectTimer);
 		this.reconnectTimer = null;
 	}
+	/**
+	* Reset all active change stream state: clear debounce timer, wakeup timer,
+	* pending target names, and mark as inactive.
+	*
+	* Does NOT close the cursor (callers handle sync vs async close) or clear
+	* the reconnect timer/attempts (callers manage reconnection lifecycle).
+	*/
+	resetActiveState() {
+		if (this.debounceTimer) {
+			clearTimeout(this.debounceTimer);
+			this.debounceTimer = null;
+		}
+		this.pendingTargetNames.clear();
+		this.clearWakeupTimer();
+		this.usingChangeStreams = false;
+	}
+	clearWakeupTimer() {
+		if (this.wakeupTimer) {
+			clearTimeout(this.wakeupTimer);
+			this.wakeupTimer = null;
+		}
+		this.wakeupTime = null;
+	}
 	closeChangeStream() {
 		if (!this.changeStream) return;
 		this.changeStream.close().catch(() => {});
@@ -848,19 +959,16 @@ var ChangeStreamHandler = class {
 	* Close the change stream cursor and emit closed event.
 	*/
 	async close() {
-		if (this.debounceTimer) {
-			clearTimeout(this.debounceTimer);
-			this.debounceTimer = null;
-		}
+		const wasActive = this.usingChangeStreams;
+		this.resetActiveState();
 		this.clearReconnectTimer();
 		if (this.changeStream) {
 			try {
 				await this.changeStream.close();
 			} catch {}
 			this.changeStream = null;
-			if (this.usingChangeStreams) this.ctx.emit("changestream:closed", void 0);
+			if (wasActive) this.ctx.emit("changestream:closed", void 0);
 		}
-		this.usingChangeStreams = false;
 		this.reconnectAttempts = 0;
 	}
 	/**
@@ -965,6 +1073,7 @@ var JobManager = class {
 		}, { returnDocument: "after" });
 		if (!result) throw new JobStateError("Job status changed during retry attempt", jobId, "unknown", "retry");
 		const job = this.ctx.documentToPersistedJob(result);
+		this.ctx.notifyPendingJob(job.name, job.nextRunAt);
 		this.ctx.emit("job:retried", {
 			job,
 			previousStatus
@@ -1001,7 +1110,9 @@ var JobManager = class {
 			updatedAt: /* @__PURE__ */ new Date()
 		} }, { returnDocument: "after" });
 		if (!result) throw new JobStateError("Job status changed during reschedule attempt", jobId, "unknown", "reschedule");
-		return this.ctx.documentToPersistedJob(result);
+		const job = this.ctx.documentToPersistedJob(result);
+		this.ctx.notifyPendingJob(job.name, job.nextRunAt);
+		return job;
 	}
 	/**
 	* Permanently delete a job.
@@ -1175,6 +1286,8 @@ var JobManager = class {
 var JobProcessor = class {
 	/** Guard flag to prevent concurrent poll() execution */
 	_isPolling = false;
+	/** Flag to request a re-poll after the current poll finishes */
+	_repollRequested = false;
 	constructor(ctx) {
 		this.ctx = ctx;
 	}
@@ -1207,12 +1320,27 @@ var JobProcessor = class {
 	* attempts to acquire jobs up to the worker's available concurrency slots.
 	* Aborts early if the scheduler is stopping (`isRunning` is false) or if
 	* the instance-level `instanceConcurrency` limit is reached.
+	*
+	* If a poll is requested while one is already running, it is queued and
+	* executed as a full poll after the current one finishes. This prevents
+	* change-stream-triggered polls from being silently dropped.
+	*
+	* @param targetNames - Optional set of worker names to poll. When provided, only the
+	* specified workers are checked. Used by change stream handler for targeted polling.
 	*/
-	async poll() {
-		if (!this.ctx.isRunning() || this._isPolling) return;
+	async poll(targetNames) {
+		if (!this.ctx.isRunning()) return;
+		if (this._isPolling) {
+			this._repollRequested = true;
+			return;
+		}
 		this._isPolling = true;
 		try {
-			await this._doPoll();
+			do {
+				this._repollRequested = false;
+				await this._doPoll(targetNames);
+				targetNames = void 0;
+			} while (this._repollRequested && this.ctx.isRunning());
 		} finally {
 			this._isPolling = false;
 		}
@@ -1220,10 +1348,11 @@ var JobProcessor = class {
 	/**
 	* Internal poll implementation.
 	*/
-	async _doPoll() {
+	async _doPoll(targetNames) {
 		const { instanceConcurrency } = this.ctx.options;
 		if (instanceConcurrency !== void 0 && this.getTotalActiveJobs() >= instanceConcurrency) return;
 		for (const [name, worker] of this.ctx.workers) {
+			if (targetNames && !targetNames.has(name)) continue;
 			const workerAvailableSlots = worker.concurrency - worker.activeJobs.size;
 			if (workerAvailableSlots <= 0) continue;
 			const availableSlots = this.getGloballyAvailableSlots(workerAvailableSlots);
@@ -1360,7 +1489,10 @@ var JobProcessor = class {
 					failReason: ""
 				}
 			}, { returnDocument: "after" });
-			return result ? this.ctx.documentToPersistedJob(result) : null;
+			if (!result) return null;
+			const persistedJob = this.ctx.documentToPersistedJob(result);
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+			return persistedJob;
 		}
 		const result = await this.ctx.collection.findOneAndUpdate({
 			_id: job._id,
@@ -1379,7 +1511,8 @@ var JobProcessor = class {
 				failReason: ""
 			}
 		}, { returnDocument: "after" });
-		return result ? this.ctx.documentToPersistedJob(result) : null;
+		if (!result) return null;
+		return this.ctx.documentToPersistedJob(result);
 	}
 	/**
 	* Handle job failure with exponential backoff retry logic using an atomic status transition.
@@ -1872,13 +2005,17 @@ var JobScheduler = class {
 					returnDocument: "after"
 				});
 				if (!result) throw new ConnectionError("Failed to enqueue job: findOneAndUpdate returned no document");
-				return this.ctx.documentToPersistedJob(result);
+				const persistedJob = this.ctx.documentToPersistedJob(result);
+				if (persistedJob.status === JobStatus.PENDING) this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+				return persistedJob;
 			}
 			const result = await this.ctx.collection.insertOne(job);
-			return {
+			const persistedJob = {
 				...job,
 				_id: result.insertedId
 			};
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+			return persistedJob;
 		} catch (error) {
 			if (error instanceof ConnectionError) throw error;
 			throw new ConnectionError(`Failed to enqueue job: ${error instanceof Error ? error.message : "Unknown error during enqueue"}`, error instanceof Error ? { cause: error } : void 0);
@@ -1986,13 +2123,17 @@ var JobScheduler = class {
 					returnDocument: "after"
 				});
 				if (!result) throw new ConnectionError("Failed to schedule job: findOneAndUpdate returned no document");
-				return this.ctx.documentToPersistedJob(result);
+				const persistedJob = this.ctx.documentToPersistedJob(result);
+				if (persistedJob.status === JobStatus.PENDING) this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+				return persistedJob;
 			}
 			const result = await this.ctx.collection.insertOne(job);
-			return {
+			const persistedJob = {
 				...job,
 				_id: result.insertedId
 			};
+			this.ctx.notifyPendingJob(persistedJob.name, persistedJob.nextRunAt);
+			return persistedJob;
 		} catch (error) {
 			if (error instanceof MonqueError) throw error;
 			throw new ConnectionError(`Failed to schedule job: ${error instanceof Error ? error.message : "Unknown error during schedule"}`, error instanceof Error ? { cause: error } : void 0);
@@ -2008,14 +2149,19 @@ const DEFAULT_RETENTION_INTERVAL = 36e5;
 /**
 * Manages scheduler lifecycle timers and job cleanup.
 *
-* Owns poll interval, heartbeat interval, cleanup interval, and the
+* Owns poll scheduling, heartbeat interval, cleanup interval, and the
 * cleanupJobs logic. Extracted from Monque to keep the facade thin.
 *
+* Uses adaptive poll scheduling: when change streams are active, polls at
+* `safetyPollInterval` (safety net only). When change streams are inactive,
+* polls at `pollInterval` (primary discovery mechanism).
+*
 * @internal Not part of public API.
 */
 var LifecycleManager = class {
 	ctx;
-	pollIntervalId = null;
+	callbacks = null;
+	pollTimeoutId = null;
 	heartbeatIntervalId = null;
 	cleanupIntervalId = null;
 	constructor(ctx) {
@@ -2024,17 +2170,13 @@ var LifecycleManager = class {
 	/**
 	* Start all lifecycle timers.
 	*
-	* Sets up poll interval, heartbeat interval, and (if configured)
+	* Sets up adaptive poll scheduling, heartbeat interval, and (if configured)
 	* cleanup interval. Runs an initial poll immediately.
 	*
 	* @param callbacks - Functions to invoke on each timer tick
 	*/
 	startTimers(callbacks) {
-		this.pollIntervalId = setInterval(() => {
-			callbacks.poll().catch((error) => {
-				this.ctx.emit("job:error", { error: toError(error) });
-			});
-		}, this.ctx.options.pollInterval);
+		this.callbacks = callbacks;
 		this.heartbeatIntervalId = setInterval(() => {
 			callbacks.updateHeartbeats().catch((error) => {
 				this.ctx.emit("job:error", { error: toError(error) });
@@ -2051,23 +2193,22 @@ var LifecycleManager = class {
 				});
 			}, interval);
 		}
-		callbacks.poll().catch((error) => {
-			this.ctx.emit("job:error", { error: toError(error) });
-		});
+		this.executePollAndScheduleNext();
 	}
 	/**
 	* Stop all lifecycle timers.
 	*
-	* Clears poll, heartbeat, and cleanup intervals.
+	* Clears poll timeout, heartbeat interval, and cleanup interval.
 	*/
 	stopTimers() {
+		this.callbacks = null;
 		if (this.cleanupIntervalId) {
 			clearInterval(this.cleanupIntervalId);
 			this.cleanupIntervalId = null;
 		}
-		if (this.pollIntervalId) {
-			clearInterval(this.pollIntervalId);
-			this.pollIntervalId = null;
+		if (this.pollTimeoutId) {
+			clearTimeout(this.pollTimeoutId);
+			this.pollTimeoutId = null;
 		}
 		if (this.heartbeatIntervalId) {
 			clearInterval(this.heartbeatIntervalId);
@@ -2075,6 +2216,43 @@ var LifecycleManager = class {
 		}
 	}
 	/**
+	* Reset the poll timer to reschedule the next poll.
+	*
+	* Called after change-stream-triggered polls to ensure the safety poll timer
+	* is recalculated (not fired redundantly from an old schedule).
+	*/
+	resetPollTimer() {
+		this.scheduleNextPoll();
+	}
+	/**
+	* Execute a poll and schedule the next one adaptively.
+	*/
+	executePollAndScheduleNext() {
+		if (!this.callbacks) return;
+		this.callbacks.poll().catch((error) => {
+			this.ctx.emit("job:error", { error: toError(error) });
+		}).then(() => {
+			this.scheduleNextPoll();
+		});
+	}
+	/**
+	* Schedule the next poll using adaptive timing.
+	*
+	* When change streams are active, uses `safetyPollInterval` (longer, safety net only).
+	* When change streams are inactive, uses `pollInterval` (shorter, primary discovery).
+	*/
+	scheduleNextPoll() {
+		if (this.pollTimeoutId) {
+			clearTimeout(this.pollTimeoutId);
+			this.pollTimeoutId = null;
+		}
+		if (!this.ctx.isRunning() || !this.callbacks) return;
+		const delay = this.callbacks.isChangeStreamActive() ? this.ctx.options.safetyPollInterval : this.ctx.options.pollInterval;
+		this.pollTimeoutId = setTimeout(() => {
+			this.executePollAndScheduleNext();
+		}, delay);
+	}
+	/**
 	* Clean up old completed and failed jobs based on retention policy.
 	*
 	* - Removes completed jobs older than `jobRetention.completed`
@@ -2114,6 +2292,7 @@ var LifecycleManager = class {
 const DEFAULTS = {
 	collectionName: "monque_jobs",
 	pollInterval: 1e3,
+	safetyPollInterval: 3e4,
 	maxRetries: 10,
 	baseRetryInterval: 1e3,
 	shutdownTimeout: 3e4,
@@ -2206,6 +2385,7 @@ var Monque = class extends node_events.EventEmitter {
 		this.options = {
 			collectionName: options.collectionName ?? DEFAULTS.collectionName,
 			pollInterval: options.pollInterval ?? DEFAULTS.pollInterval,
+			safetyPollInterval: options.safetyPollInterval ?? DEFAULTS.safetyPollInterval,
 			maxRetries: options.maxRetries ?? DEFAULTS.maxRetries,
 			baseRetryInterval: options.baseRetryInterval ?? DEFAULTS.baseRetryInterval,
 			shutdownTimeout: options.shutdownTimeout ?? DEFAULTS.shutdownTimeout,
@@ -2240,7 +2420,7 @@ var Monque = class extends node_events.EventEmitter {
 			this._manager = new JobManager(ctx);
 			this._query = new JobQueryService(ctx);
 			this._processor = new JobProcessor(ctx);
-			this._changeStreamHandler = new ChangeStreamHandler(ctx, () => this.processor.poll());
+			this._changeStreamHandler = new ChangeStreamHandler(ctx, (targetNames) => this.handleChangeStreamPoll(targetNames));
 			this._lifecycleManager = new LifecycleManager(ctx);
 			this.isInitialized = true;
 		} catch (error) {
@@ -2278,6 +2458,17 @@ var Monque = class extends node_events.EventEmitter {
 		return this._lifecycleManager;
 	}
 	/**
+	* Handle a change-stream-triggered poll and reset the safety poll timer.
+	*
+	* Used as the `onPoll` callback for {@link ChangeStreamHandler}. Runs a
+	* targeted poll for the given worker names, then resets the adaptive safety
+	* poll timer so it doesn't fire redundantly.
+	*/
+	async handleChangeStreamPoll(targetNames) {
+		await this.processor.poll(targetNames);
+		this.lifecycleManager.resetPollTimer();
+	}
+	/**
 	* Build the shared context for internal services.
 	*/
 	buildContext() {
@@ -2289,6 +2480,10 @@ var Monque = class extends node_events.EventEmitter {
 			workers: this.workers,
 			isRunning: () => this.isRunning,
 			emit: (event, payload) => this.emit(event, payload),
+			notifyPendingJob: (name, nextRunAt) => {
+				if (!this.isRunning || !this._changeStreamHandler) return;
+				this._changeStreamHandler.notifyPendingJob(name, nextRunAt);
+			},
 			documentToPersistedJob: (doc) => documentToPersistedJob(doc)
 		};
 	}
@@ -2989,7 +3184,8 @@ var Monque = class extends node_events.EventEmitter {
 		this.changeStreamHandler.setup();
 		this.lifecycleManager.startTimers({
 			poll: () => this.processor.poll(),
-			updateHeartbeats: () => this.processor.updateHeartbeats()
+			updateHeartbeats: () => this.processor.updateHeartbeats(),
+			isChangeStreamActive: () => this.changeStreamHandler.isActive()
 		});
 	}
 	/**