@monque/core 1.3.0 → 1.5.0

package/src/scheduler/services/change-stream-handler.ts

@@ -1,6 +1,7 @@
 import type { ChangeStream, ChangeStreamDocument, Document } from 'mongodb';
 
 import { JobStatus } from '@/jobs';
+import { toError } from '@/shared';
 
 import type { SchedulerContext } from './types.js';
 
@@ -133,7 +134,7 @@ export class ChangeStreamHandler {
 			this.debounceTimer = setTimeout(() => {
 				this.debounceTimer = null;
 				this.onPoll().catch((error: unknown) => {
-					this.ctx.emit('job:error', { error: error as Error });
+					this.ctx.emit('job:error', { error: toError(error) });
 				});
 			}, 100);
 		}

package/src/scheduler/services/index.ts

@@ -4,5 +4,6 @@ export { JobManager } from './job-manager.js';
 export { JobProcessor } from './job-processor.js';
 export { JobQueryService } from './job-query.js';
 export { JobScheduler } from './job-scheduler.js';
+export { LifecycleManager } from './lifecycle-manager.js';
 // Types
 export type { ResolvedMonqueOptions, SchedulerContext } from './types.js';

package/src/scheduler/services/job-manager.ts

@@ -1,14 +1,8 @@
-import { ObjectId, type WithId } from 'mongodb';
-
-import {
-	type BulkOperationResult,
-	type Job,
-	type JobSelector,
-	JobStatus,
-	type PersistedJob,
-} from '@/jobs';
+import { ObjectId } from 'mongodb';
+
+import { type BulkOperationResult, type JobSelector, JobStatus, type PersistedJob } from '@/jobs';
 import { buildSelectorQuery } from '@/scheduler';
-import { JobStateError } from '@/shared';
+import { ConnectionError, JobStateError, MonqueError } from '@/shared';
 
 import type { SchedulerContext } from './types.js';
 
@@ -49,17 +43,15 @@ export class JobManager {
 		const jobDoc = await this.ctx.collection.findOne({ _id });
 		if (!jobDoc) return null;
 
-		const currentJob = jobDoc as unknown as WithId<Job>;
-
-		if (currentJob.status === JobStatus.CANCELLED) {
-			return this.ctx.documentToPersistedJob(currentJob);
+		if (jobDoc['status'] === JobStatus.CANCELLED) {
+			return this.ctx.documentToPersistedJob(jobDoc);
 		}
 
-		if (currentJob.status !== JobStatus.PENDING) {
+		if (jobDoc['status'] !== JobStatus.PENDING) {
 			throw new JobStateError(
-				`Cannot cancel job in status '${currentJob.status}'`,
+				`Cannot cancel job in status '${jobDoc['status']}'`,
 				jobId,
-				currentJob.status,
+				jobDoc['status'],
 				'cancel',
 			);
 		}
@@ -183,13 +175,11 @@ export class JobManager {
 
 		if (!currentJobDoc) return null;
 
-		const currentJob = currentJobDoc as unknown as WithId<Job>;
-
-		if (currentJob.status !== JobStatus.PENDING) {
+		if (currentJobDoc['status'] !== JobStatus.PENDING) {
 			throw new JobStateError(
-				`Cannot reschedule job in status '${currentJob.status}'`,
+				`Cannot reschedule job in status '${currentJobDoc['status']}'`,
 				jobId,
-				currentJob.status,
+				currentJobDoc['status'],
 				'reschedule',
 			);
 		}
@@ -254,14 +244,15 @@ export class JobManager {
 	// ─────────────────────────────────────────────────────────────────────────────
 
 	/**
-	 * Cancel multiple jobs matching the given filter.
+	 * Cancel multiple jobs matching the given filter via a single updateMany call.
 	 *
-	 * Only cancels jobs in 'pending' status. Jobs in other states are collected
-	 * as errors in the result. Emits a 'jobs:cancelled' event with the IDs of
+	 * Only cancels jobs in 'pending' status — the status guard is applied regardless
+	 * of what the filter specifies. Jobs in other states are silently skipped (not
+	 * matched by the query). Emits a 'jobs:cancelled' event with the count of
 	 * successfully cancelled jobs.
 	 *
 	 * @param filter - Selector for which jobs to cancel (name, status, date range)
-	 * @returns Result with count of cancelled jobs and any errors encountered
+	 * @returns Result with count of cancelled jobs (errors array always empty for bulk ops)
 	 *
 	 * @example Cancel all pending jobs for a queue
 	 * ```typescript
@@ -273,76 +264,54 @@ export class JobManager {
 	 * ```
 	 */
 	async cancelJobs(filter: JobSelector): Promise<BulkOperationResult> {
-		const baseQuery = buildSelectorQuery(filter);
-		const errors: Array<{ jobId: string; error: string }> = [];
-		const cancelledIds: string[] = [];
-
-		// Find all matching jobs and stream them to avoid memory pressure
-		const cursor = this.ctx.collection.find(baseQuery);
-
-		for await (const doc of cursor) {
-			const job = doc as unknown as WithId<Job>;
-			const jobId = job._id.toString();
-
-			if (job.status !== JobStatus.PENDING && job.status !== JobStatus.CANCELLED) {
-				errors.push({
-					jobId,
-					error: `Cannot cancel job in status '${job.status}'`,
-				});
-				continue;
-			}
+		const query = buildSelectorQuery(filter);
 
-			// Skip already cancelled jobs (idempotent)
-			if (job.status === JobStatus.CANCELLED) {
-				cancelledIds.push(jobId);
-				continue;
+		// Enforce allowed status, but respect explicit status filters
+		if (filter.status !== undefined) {
+			const requested = Array.isArray(filter.status) ? filter.status : [filter.status];
+			if (!requested.includes(JobStatus.PENDING)) {
+				return { count: 0, errors: [] };
 			}
+		}
+		query['status'] = JobStatus.PENDING;
 
-			// Atomically update to cancelled
-			const result = await this.ctx.collection.findOneAndUpdate(
-				{ _id: job._id, status: JobStatus.PENDING },
-				{
-					$set: {
-						status: JobStatus.CANCELLED,
-						updatedAt: new Date(),
-					},
+		try {
+			const result = await this.ctx.collection.updateMany(query, {
+				$set: {
+					status: JobStatus.CANCELLED,
+					updatedAt: new Date(),
 				},
-				{ returnDocument: 'after' },
-			);
+			});
+
+			const count = result.modifiedCount;
 
-			if (result) {
-				cancelledIds.push(jobId);
-			} else {
-				// Race condition: status changed
-				errors.push({
-					jobId,
-					error: 'Job status changed during cancellation',
-				});
+			if (count > 0) {
+				this.ctx.emit('jobs:cancelled', { count });
 			}
-		}
 
-		if (cancelledIds.length > 0) {
-			this.ctx.emit('jobs:cancelled', {
-				jobIds: cancelledIds,
-				count: cancelledIds.length,
-			});
+			return { count, errors: [] };
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during cancelJobs';
+			throw new ConnectionError(
+				`Failed to cancel jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
 		}
-
-		return {
-			count: cancelledIds.length,
-			errors,
-		};
 	}
 
 	/**
-	 * Retry multiple jobs matching the given filter.
+	 * Retry multiple jobs matching the given filter via a single pipeline-style updateMany call.
 	 *
-	 * Only retries jobs in 'failed' or 'cancelled' status. Jobs in other states
-	 * are collected as errors in the result. Emits a 'jobs:retried' event with
-	 * the IDs of successfully retried jobs.
+	 * Only retries jobs in 'failed' or 'cancelled' status — the status guard is applied
+	 * regardless of what the filter specifies. Jobs in other states are silently skipped.
+	 * Uses `$rand` for per-document staggered `nextRunAt` to avoid thundering herd on retry.
+	 * Emits a 'jobs:retried' event with the count of successfully retried jobs.
 	 *
 	 * @param filter - Selector for which jobs to retry (name, status, date range)
-	 * @returns Result with count of retried jobs and any errors encountered
+	 * @returns Result with count of retried jobs (errors array always empty for bulk ops)
 	 *
 	 * @example Retry all failed jobs
 	 * ```typescript
@@ -353,68 +322,60 @@ export class JobManager {
 	 * ```
 	 */
 	async retryJobs(filter: JobSelector): Promise<BulkOperationResult> {
-		const baseQuery = buildSelectorQuery(filter);
-		const errors: Array<{ jobId: string; error: string }> = [];
-		const retriedIds: string[] = [];
-
-		const cursor = this.ctx.collection.find(baseQuery);
-
-		for await (const doc of cursor) {
-			const job = doc as unknown as WithId<Job>;
-			const jobId = job._id.toString();
-
-			if (job.status !== JobStatus.FAILED && job.status !== JobStatus.CANCELLED) {
-				errors.push({
-					jobId,
-					error: `Cannot retry job in status '${job.status}'`,
-				});
-				continue;
+		const query = buildSelectorQuery(filter);
+
+		// Enforce allowed statuses, but respect explicit status filters
+		const retryable = [JobStatus.FAILED, JobStatus.CANCELLED] as const;
+		if (filter.status !== undefined) {
+			const requested = Array.isArray(filter.status) ? filter.status : [filter.status];
+			const allowed = requested.filter(
+				(status): status is (typeof retryable)[number] =>
+					status === JobStatus.FAILED || status === JobStatus.CANCELLED,
+			);
+			if (allowed.length === 0) {
+				return { count: 0, errors: [] };
 			}
+			query['status'] = allowed.length === 1 ? allowed[0] : { $in: allowed };
+		} else {
+			query['status'] = { $in: retryable };
+		}
 
-			const result = await this.ctx.collection.findOneAndUpdate(
-				{
-					_id: job._id,
-					status: { $in: [JobStatus.FAILED, JobStatus.CANCELLED] },
-				},
+		const spreadWindowMs = 30_000; // 30s max spread for staggered retry
+
+		try {
+			const result = await this.ctx.collection.updateMany(query, [
 				{
 					$set: {
 						status: JobStatus.PENDING,
 						failCount: 0,
-						nextRunAt: new Date(),
+						nextRunAt: {
+							$add: [new Date(), { $multiply: [{ $rand: {} }, spreadWindowMs] }],
+						},
 						updatedAt: new Date(),
 					},
-					$unset: {
-						failReason: '',
-						lockedAt: '',
-						claimedBy: '',
-						lastHeartbeat: '',
-						heartbeatInterval: '',
-					},
 				},
-				{ returnDocument: 'after' },
-			);
+				{
+					$unset: ['failReason', 'lockedAt', 'claimedBy', 'lastHeartbeat', 'heartbeatInterval'],
+				},
+			]);
+
+			const count = result.modifiedCount;
 
-			if (result) {
-				retriedIds.push(jobId);
-			} else {
-				errors.push({
-					jobId,
-					error: 'Job status changed during retry attempt',
-				});
+			if (count > 0) {
+				this.ctx.emit('jobs:retried', { count });
 			}
-		}
 
-		if (retriedIds.length > 0) {
-			this.ctx.emit('jobs:retried', {
-				jobIds: retriedIds,
-				count: retriedIds.length,
-			});
+			return { count, errors: [] };
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during retryJobs';
+			throw new ConnectionError(
+				`Failed to retry jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
 		}
-
-		return {
-			count: retriedIds.length,
-			errors,
-		};
 	}
 
 	/**
@@ -440,16 +401,27 @@ export class JobManager {
 	async deleteJobs(filter: JobSelector): Promise<BulkOperationResult> {
 		const query = buildSelectorQuery(filter);
 
-		// Use deleteMany for efficiency
-		const result = await this.ctx.collection.deleteMany(query);
+		try {
+			// Use deleteMany for efficiency
+			const result = await this.ctx.collection.deleteMany(query);
 
-		if (result.deletedCount > 0) {
-			this.ctx.emit('jobs:deleted', { count: result.deletedCount });
-		}
+			if (result.deletedCount > 0) {
+				this.ctx.emit('jobs:deleted', { count: result.deletedCount });
+			}
 
-		return {
-			count: result.deletedCount,
-			errors: [],
-		};
+			return {
+				count: result.deletedCount,
+				errors: [],
+			};
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during deleteJobs';
+			throw new ConnectionError(
+				`Failed to delete jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
 	}
 }

package/src/scheduler/services/job-processor.ts

@@ -1,5 +1,5 @@
 import { isPersistedJob, type Job, JobStatus, type PersistedJob } from '@/jobs';
-import { calculateBackoff, getNextCronDate } from '@/shared';
+import { calculateBackoff, getNextCronDate, toError } from '@/shared';
 import type { WorkerRegistration } from '@/workers';
 
 import type { SchedulerContext } from './types.js';
@@ -117,7 +117,7 @@ export class JobProcessor {
 					worker.activeJobs.set(job._id.toString(), job);
 
 					this.processJob(job, worker).catch((error: unknown) => {
-						this.ctx.emit('job:error', { error: error as Error, job });
+						this.ctx.emit('job:error', { error: toError(error), job });
 					});
 				} else {
 					// No more jobs available for this worker
@@ -189,6 +189,10 @@ export class JobProcessor {
 	 * both success and failure cases. On success, calls `completeJob()`. On failure,
 	 * calls `failJob()` which implements exponential backoff retry logic.
 	 *
+	 * Events are only emitted when the underlying atomic status transition succeeds,
+	 * ensuring event consumers receive reliable, consistent data backed by the actual
+	 * database state.
+	 *
 	 * @param job - The job to process
 	 * @param worker - The worker registration containing the handler and active job tracking
 	 */
@@ -202,39 +206,50 @@ export class JobProcessor {
 
 			// Job completed successfully
 			const duration = Date.now() - startTime;
-			await this.completeJob(job);
-			this.ctx.emit('job:complete', { job, duration });
+			const updatedJob = await this.completeJob(job);
+
+			if (updatedJob) {
+				this.ctx.emit('job:complete', { job: updatedJob, duration });
+			}
 		} catch (error) {
 			// Job failed
 			const err = error instanceof Error ? error : new Error(String(error));
-			await this.failJob(job, err);
+			const updatedJob = await this.failJob(job, err);
 
-			const willRetry = job.failCount + 1 < this.ctx.options.maxRetries;
-			this.ctx.emit('job:fail', { job, error: err, willRetry });
+			if (updatedJob) {
+				const willRetry = updatedJob.status === JobStatus.PENDING;
+				this.ctx.emit('job:fail', { job: updatedJob, error: err, willRetry });
+			}
 		} finally {
 			worker.activeJobs.delete(jobId);
 		}
 	}
 
 	/**
-	 * Mark a job as completed successfully.
+	 * Mark a job as completed successfully using an atomic status transition.
+	 *
+	 * Uses `findOneAndUpdate` with `status: processing` and `claimedBy: instanceId`
+	 * preconditions to ensure the transition only occurs if the job is still owned by this
+	 * scheduler instance. Returns `null` if the job was concurrently modified (e.g., reclaimed
+	 * by another instance after stale recovery).
 	 *
 	 * For recurring jobs (with `repeatInterval`), schedules the next run based on the cron
 	 * expression and resets `failCount` to 0. For one-time jobs, sets status to `completed`.
 	 * Clears `lockedAt` and `failReason` fields in both cases.
 	 *
 	 * @param job - The job that completed successfully
+	 * @returns The updated job document, or `null` if the transition could not be applied
 	 */
-	async completeJob(job: Job): Promise<void> {
+	async completeJob(job: Job): Promise<PersistedJob | null> {
 		if (!isPersistedJob(job)) {
-			return;
+			return null;
 		}
 
 		if (job.repeatInterval) {
 			// Recurring job - schedule next run
 			const nextRunAt = getNextCronDate(job.repeatInterval);
-			await this.ctx.collection.updateOne(
-				{ _id: job._id },
+			const result = await this.ctx.collection.findOneAndUpdate(
+				{ _id: job._id, status: JobStatus.PROCESSING, claimedBy: this.ctx.instanceId },
 				{
 					$set: {
 						status: JobStatus.PENDING,
@@ -250,52 +265,63 @@ export class JobProcessor {
 						failReason: '',
 					},
 				},
+				{ returnDocument: 'after' },
 			);
-		} else {
-			// One-time job - mark as completed
-			await this.ctx.collection.updateOne(
-				{ _id: job._id },
-				{
-					$set: {
-						status: JobStatus.COMPLETED,
-						updatedAt: new Date(),
-					},
-					$unset: {
-						lockedAt: '',
-						claimedBy: '',
-						lastHeartbeat: '',
-						heartbeatInterval: '',
-						failReason: '',
-					},
-				},
-			);
-			job.status = JobStatus.COMPLETED;
+
+			return result ? this.ctx.documentToPersistedJob(result) : null;
 		}
+
+		// One-time job - mark as completed
+		const result = await this.ctx.collection.findOneAndUpdate(
+			{ _id: job._id, status: JobStatus.PROCESSING, claimedBy: this.ctx.instanceId },
+			{
+				$set: {
+					status: JobStatus.COMPLETED,
+					updatedAt: new Date(),
+				},
+				$unset: {
+					lockedAt: '',
+					claimedBy: '',
+					lastHeartbeat: '',
+					heartbeatInterval: '',
+					failReason: '',
+				},
+			},
+			{ returnDocument: 'after' },
+		);
+
+		return result ? this.ctx.documentToPersistedJob(result) : null;
 	}
 
 	/**
-	 * Handle job failure with exponential backoff retry logic.
+	 * Handle job failure with exponential backoff retry logic using an atomic status transition.
+	 *
+	 * Uses `findOneAndUpdate` with `status: processing` and `claimedBy: instanceId`
+	 * preconditions to ensure the transition only occurs if the job is still owned by this
+	 * scheduler instance. Returns `null` if the job was concurrently modified (e.g., reclaimed
+	 * by another instance after stale recovery).
 	 *
 	 * Increments `failCount` and calculates next retry time using exponential backoff:
-	 * `nextRunAt = 2^failCount × baseRetryInterval` (capped by optional `maxBackoffDelay`).
+	 * `nextRunAt = 2^failCount * baseRetryInterval` (capped by optional `maxBackoffDelay`).
 	 *
 	 * If `failCount >= maxRetries`, marks job as permanently `failed`. Otherwise, resets
 	 * to `pending` status for retry. Stores error message in `failReason` field.
 	 *
 	 * @param job - The job that failed
 	 * @param error - The error that caused the failure
+	 * @returns The updated job document, or `null` if the transition could not be applied
 	 */
-	async failJob(job: Job, error: Error): Promise<void> {
+	async failJob(job: Job, error: Error): Promise<PersistedJob | null> {
 		if (!isPersistedJob(job)) {
-			return;
+			return null;
 		}
 
 		const newFailCount = job.failCount + 1;
 
 		if (newFailCount >= this.ctx.options.maxRetries) {
 			// Permanent failure
-			await this.ctx.collection.updateOne(
-				{ _id: job._id },
+			const result = await this.ctx.collection.findOneAndUpdate(
+				{ _id: job._id, status: JobStatus.PROCESSING, claimedBy: this.ctx.instanceId },
 				{
 					$set: {
 						status: JobStatus.FAILED,
@@ -310,34 +336,40 @@ export class JobProcessor {
 						heartbeatInterval: '',
 					},
 				},
-			);
-		} else {
-			// Schedule retry with exponential backoff
-			const nextRunAt = calculateBackoff(
-				newFailCount,
-				this.ctx.options.baseRetryInterval,
-				this.ctx.options.maxBackoffDelay,
+				{ returnDocument: 'after' },
 			);
 
-			await this.ctx.collection.updateOne(
-				{ _id: job._id },
-				{
-					$set: {
-						status: JobStatus.PENDING,
-						failCount: newFailCount,
-						failReason: error.message,
-						nextRunAt,
-						updatedAt: new Date(),
-					},
-					$unset: {
-						lockedAt: '',
-						claimedBy: '',
-						lastHeartbeat: '',
-						heartbeatInterval: '',
-					},
-				},
-			);
+			return result ? this.ctx.documentToPersistedJob(result) : null;
 		}
+
+		// Schedule retry with exponential backoff
+		const nextRunAt = calculateBackoff(
+			newFailCount,
+			this.ctx.options.baseRetryInterval,
+			this.ctx.options.maxBackoffDelay,
+		);
+
+		const result = await this.ctx.collection.findOneAndUpdate(
+			{ _id: job._id, status: JobStatus.PROCESSING, claimedBy: this.ctx.instanceId },
+			{
+				$set: {
+					status: JobStatus.PENDING,
+					failCount: newFailCount,
+					failReason: error.message,
+					nextRunAt,
+					updatedAt: new Date(),
+				},
+				$unset: {
+					lockedAt: '',
+					claimedBy: '',
+					lastHeartbeat: '',
+					heartbeatInterval: '',
+				},
+			},
+			{ returnDocument: 'after' },
+		);
+
+		return result ? this.ctx.documentToPersistedJob(result) : null;
 	}
 
 	/**