@monque/core 1.4.0 → 1.5.0

package/src/scheduler/services/job-manager.ts

@@ -2,7 +2,7 @@ import { ObjectId } from 'mongodb';
 
 import { type BulkOperationResult, type JobSelector, JobStatus, type PersistedJob } from '@/jobs';
 import { buildSelectorQuery } from '@/scheduler';
-import { JobStateError } from '@/shared';
+import { ConnectionError, JobStateError, MonqueError } from '@/shared';
 
 import type { SchedulerContext } from './types.js';
 
@@ -244,14 +244,15 @@ export class JobManager {
 	// ─────────────────────────────────────────────────────────────────────────────
 
 	/**
-	 * Cancel multiple jobs matching the given filter.
+	 * Cancel multiple jobs matching the given filter via a single updateMany call.
 	 *
-	 * Only cancels jobs in 'pending' status. Jobs in other states are collected
-	 * as errors in the result. Emits a 'jobs:cancelled' event with the IDs of
+	 * Only cancels jobs in 'pending' status — the status guard is applied regardless
+	 * of what the filter specifies. Jobs in other states are silently skipped (not
+	 * matched by the query). Emits a 'jobs:cancelled' event with the count of
 	 * successfully cancelled jobs.
 	 *
 	 * @param filter - Selector for which jobs to cancel (name, status, date range)
-	 * @returns Result with count of cancelled jobs and any errors encountered
+	 * @returns Result with count of cancelled jobs (errors array always empty for bulk ops)
 	 *
 	 * @example Cancel all pending jobs for a queue
 	 * ```typescript
@@ -263,75 +264,54 @@ export class JobManager {
 	 * ```
 	 */
 	async cancelJobs(filter: JobSelector): Promise<BulkOperationResult> {
-		const baseQuery = buildSelectorQuery(filter);
-		const errors: Array<{ jobId: string; error: string }> = [];
-		const cancelledIds: string[] = [];
-
-		// Find all matching jobs and stream them to avoid memory pressure
-		const cursor = this.ctx.collection.find(baseQuery);
-
-		for await (const doc of cursor) {
-			const jobId = doc._id.toString();
-
-			if (doc['status'] !== JobStatus.PENDING && doc['status'] !== JobStatus.CANCELLED) {
-				errors.push({
-					jobId,
-					error: `Cannot cancel job in status '${doc['status']}'`,
-				});
-				continue;
-			}
+		const query = buildSelectorQuery(filter);
 
-			// Skip already cancelled jobs (idempotent)
-			if (doc['status'] === JobStatus.CANCELLED) {
-				cancelledIds.push(jobId);
-				continue;
+		// Enforce allowed status, but respect explicit status filters
+		if (filter.status !== undefined) {
+			const requested = Array.isArray(filter.status) ? filter.status : [filter.status];
+			if (!requested.includes(JobStatus.PENDING)) {
+				return { count: 0, errors: [] };
 			}
+		}
+		query['status'] = JobStatus.PENDING;
 
-			// Atomically update to cancelled
-			const result = await this.ctx.collection.findOneAndUpdate(
-				{ _id: doc._id, status: JobStatus.PENDING },
-				{
-					$set: {
-						status: JobStatus.CANCELLED,
-						updatedAt: new Date(),
-					},
+		try {
+			const result = await this.ctx.collection.updateMany(query, {
+				$set: {
+					status: JobStatus.CANCELLED,
+					updatedAt: new Date(),
 				},
-				{ returnDocument: 'after' },
-			);
+			});
+
+			const count = result.modifiedCount;
 
-			if (result) {
-				cancelledIds.push(jobId);
-			} else {
-				// Race condition: status changed
-				errors.push({
-					jobId,
-					error: 'Job status changed during cancellation',
-				});
+			if (count > 0) {
+				this.ctx.emit('jobs:cancelled', { count });
 			}
-		}
 
-		if (cancelledIds.length > 0) {
-			this.ctx.emit('jobs:cancelled', {
-				jobIds: cancelledIds,
-				count: cancelledIds.length,
-			});
+			return { count, errors: [] };
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during cancelJobs';
+			throw new ConnectionError(
+				`Failed to cancel jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
 		}
-
-		return {
-			count: cancelledIds.length,
-			errors,
-		};
 	}
 
 	/**
-	 * Retry multiple jobs matching the given filter.
+	 * Retry multiple jobs matching the given filter via a single pipeline-style updateMany call.
 	 *
-	 * Only retries jobs in 'failed' or 'cancelled' status. Jobs in other states
-	 * are collected as errors in the result. Emits a 'jobs:retried' event with
-	 * the IDs of successfully retried jobs.
+	 * Only retries jobs in 'failed' or 'cancelled' status — the status guard is applied
+	 * regardless of what the filter specifies. Jobs in other states are silently skipped.
+	 * Uses `$rand` for per-document staggered `nextRunAt` to avoid thundering herd on retry.
+	 * Emits a 'jobs:retried' event with the count of successfully retried jobs.
 	 *
 	 * @param filter - Selector for which jobs to retry (name, status, date range)
-	 * @returns Result with count of retried jobs and any errors encountered
+	 * @returns Result with count of retried jobs (errors array always empty for bulk ops)
 	 *
 	 * @example Retry all failed jobs
 	 * ```typescript
@@ -342,67 +322,60 @@ export class JobManager {
 	 * ```
 	 */
 	async retryJobs(filter: JobSelector): Promise<BulkOperationResult> {
-		const baseQuery = buildSelectorQuery(filter);
-		const errors: Array<{ jobId: string; error: string }> = [];
-		const retriedIds: string[] = [];
-
-		const cursor = this.ctx.collection.find(baseQuery);
-
-		for await (const doc of cursor) {
-			const jobId = doc._id.toString();
+		const query = buildSelectorQuery(filter);
 
-			if (doc['status'] !== JobStatus.FAILED && doc['status'] !== JobStatus.CANCELLED) {
-				errors.push({
-					jobId,
-					error: `Cannot retry job in status '${doc['status']}'`,
-				});
-				continue;
+		// Enforce allowed statuses, but respect explicit status filters
+		const retryable = [JobStatus.FAILED, JobStatus.CANCELLED] as const;
+		if (filter.status !== undefined) {
+			const requested = Array.isArray(filter.status) ? filter.status : [filter.status];
+			const allowed = requested.filter(
+				(status): status is (typeof retryable)[number] =>
+					status === JobStatus.FAILED || status === JobStatus.CANCELLED,
+			);
+			if (allowed.length === 0) {
+				return { count: 0, errors: [] };
 			}
+			query['status'] = allowed.length === 1 ? allowed[0] : { $in: allowed };
+		} else {
+			query['status'] = { $in: retryable };
+		}
 
-			const result = await this.ctx.collection.findOneAndUpdate(
-				{
-					_id: doc._id,
-					status: { $in: [JobStatus.FAILED, JobStatus.CANCELLED] },
-				},
+		const spreadWindowMs = 30_000; // 30s max spread for staggered retry
+
+		try {
+			const result = await this.ctx.collection.updateMany(query, [
 				{
 					$set: {
 						status: JobStatus.PENDING,
 						failCount: 0,
-						nextRunAt: new Date(),
+						nextRunAt: {
+							$add: [new Date(), { $multiply: [{ $rand: {} }, spreadWindowMs] }],
+						},
 						updatedAt: new Date(),
 					},
-					$unset: {
-						failReason: '',
-						lockedAt: '',
-						claimedBy: '',
-						lastHeartbeat: '',
-						heartbeatInterval: '',
-					},
 				},
-				{ returnDocument: 'after' },
-			);
+				{
+					$unset: ['failReason', 'lockedAt', 'claimedBy', 'lastHeartbeat', 'heartbeatInterval'],
+				},
+			]);
 
-			if (result) {
-				retriedIds.push(jobId);
-			} else {
-				errors.push({
-					jobId,
-					error: 'Job status changed during retry attempt',
-				});
+			const count = result.modifiedCount;
+
+			if (count > 0) {
+				this.ctx.emit('jobs:retried', { count });
 			}
-		}
 
-		if (retriedIds.length > 0) {
-			this.ctx.emit('jobs:retried', {
-				jobIds: retriedIds,
-				count: retriedIds.length,
-			});
+			return { count, errors: [] };
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during retryJobs';
+			throw new ConnectionError(
+				`Failed to retry jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
 		}
-
-		return {
-			count: retriedIds.length,
-			errors,
-		};
 	}
 
 	/**
@@ -428,16 +401,27 @@ export class JobManager {
 	async deleteJobs(filter: JobSelector): Promise<BulkOperationResult> {
 		const query = buildSelectorQuery(filter);
 
-		// Use deleteMany for efficiency
-		const result = await this.ctx.collection.deleteMany(query);
+		try {
+			// Use deleteMany for efficiency
+			const result = await this.ctx.collection.deleteMany(query);
 
-		if (result.deletedCount > 0) {
-			this.ctx.emit('jobs:deleted', { count: result.deletedCount });
-		}
+			if (result.deletedCount > 0) {
+				this.ctx.emit('jobs:deleted', { count: result.deletedCount });
+			}
 
-		return {
-			count: result.deletedCount,
-			errors: [],
-		};
+			return {
+				count: result.deletedCount,
+				errors: [],
+			};
+		} catch (error) {
+			if (error instanceof MonqueError) {
+				throw error;
+			}
+			const message = error instanceof Error ? error.message : 'Unknown error during deleteJobs';
+			throw new ConnectionError(
+				`Failed to delete jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
 	}
 }

package/src/scheduler/services/job-query.ts

@@ -16,6 +16,11 @@ import { AggregationTimeoutError, ConnectionError } from '@/shared';
 import { buildSelectorQuery, decodeCursor, encodeCursor } from '../helpers.js';
 import type { SchedulerContext } from './types.js';
 
+interface StatsCacheEntry {
+	data: QueueStats;
+	expiresAt: number;
+}
+
 /**
  * Internal service for job query operations.
  *
@@ -25,6 +30,9 @@ import type { SchedulerContext } from './types.js';
  * @internal Not part of public API - use Monque class methods instead.
  */
 export class JobQueryService {
+	private readonly statsCache = new Map<string, StatsCacheEntry>();
+	private static readonly MAX_CACHE_SIZE = 100;
+
 	constructor(private readonly ctx: SchedulerContext) {}
 
 	/**
@@ -264,12 +272,24 @@ export class JobQueryService {
 		};
 	}
 
+	/**
+	 * Clear all cached getQueueStats() results.
+	 * Called on scheduler stop() for clean state on restart.
+	 * @internal
+	 */
+	clearStatsCache(): void {
+		this.statsCache.clear();
+	}
+
 	/**
 	 * Get aggregate statistics for the job queue.
 	 *
 	 * Uses MongoDB aggregation pipeline for efficient server-side calculation.
 	 * Returns counts per status and optional average processing duration for completed jobs.
 	 *
+	 * Results are cached per unique filter with a configurable TTL (default 5s).
+	 * Set `statsCacheTtlMs: 0` to disable caching.
+	 *
 	 * @param filter - Optional filter to scope statistics by job name
 	 * @returns Promise resolving to queue statistics
 	 * @throws {AggregationTimeoutError} If aggregation exceeds 30 second timeout
@@ -288,6 +308,16 @@ export class JobQueryService {
 	 * ```
 	 */
 	async getQueueStats(filter?: Pick<JobSelector, 'name'>): Promise<QueueStats> {
+		const ttl = this.ctx.options.statsCacheTtlMs;
+		const cacheKey = filter?.name ?? '';
+
+		if (ttl > 0) {
+			const cached = this.statsCache.get(cacheKey);
+			if (cached && cached.expiresAt > Date.now()) {
+				return { ...cached.data };
+			}
+		}
+
 		const matchStage: Document = {};
 
 		if (filter?.name) {
@@ -350,48 +380,63 @@ export class JobQueryService {
 				total: 0,
 			};
 
-			if (!result) {
-				return stats;
-			}
+			if (result) {
+				// Map status counts to stats
+				const statusCounts = result['statusCounts'] as Array<{ _id: string; count: number }>;
+				for (const entry of statusCounts) {
+					const status = entry._id;
+					const count = entry.count;
+
+					switch (status) {
+						case JobStatus.PENDING:
+							stats.pending = count;
+							break;
+						case JobStatus.PROCESSING:
+							stats.processing = count;
+							break;
+						case JobStatus.COMPLETED:
+							stats.completed = count;
+							break;
+						case JobStatus.FAILED:
+							stats.failed = count;
+							break;
+						case JobStatus.CANCELLED:
+							stats.cancelled = count;
+							break;
+					}
+				}
 
-			// Map status counts to stats
-			const statusCounts = result['statusCounts'] as Array<{ _id: string; count: number }>;
-			for (const entry of statusCounts) {
-				const status = entry._id;
-				const count = entry.count;
-
-				switch (status) {
-					case JobStatus.PENDING:
-						stats.pending = count;
-						break;
-					case JobStatus.PROCESSING:
-						stats.processing = count;
-						break;
-					case JobStatus.COMPLETED:
-						stats.completed = count;
-						break;
-					case JobStatus.FAILED:
-						stats.failed = count;
-						break;
-					case JobStatus.CANCELLED:
-						stats.cancelled = count;
-						break;
+				// Extract total
+				const totalResult = result['total'] as Array<{ count: number }>;
+				if (totalResult.length > 0 && totalResult[0]) {
+					stats.total = totalResult[0].count;
 				}
-			}
 
-			// Extract total
-			const totalResult = result['total'] as Array<{ count: number }>;
-			if (totalResult.length > 0 && totalResult[0]) {
-				stats.total = totalResult[0].count;
+				// Extract average processing duration
+				const avgDurationResult = result['avgDuration'] as Array<{ avgMs: number }>;
+				if (avgDurationResult.length > 0 && avgDurationResult[0]) {
+					const avgMs = avgDurationResult[0].avgMs;
+					if (typeof avgMs === 'number' && !Number.isNaN(avgMs)) {
+						stats.avgProcessingDurationMs = Math.round(avgMs);
+					}
+				}
 			}
 
-			// Extract average processing duration
-			const avgDurationResult = result['avgDuration'] as Array<{ avgMs: number }>;
-			if (avgDurationResult.length > 0 && avgDurationResult[0]) {
-				const avgMs = avgDurationResult[0].avgMs;
-				if (typeof avgMs === 'number' && !Number.isNaN(avgMs)) {
-					stats.avgProcessingDurationMs = Math.round(avgMs);
+			// Cache the result if TTL is enabled
+			if (ttl > 0) {
+				// Delete existing entry first so re-insertion moves it to end (Map insertion order = LRU)
+				this.statsCache.delete(cacheKey);
+				// LRU eviction: if cache is still full after removing existing key, evict the oldest entry
+				if (this.statsCache.size >= JobQueryService.MAX_CACHE_SIZE) {
+					const oldestKey = this.statsCache.keys().next().value;
+					if (oldestKey !== undefined) {
+						this.statsCache.delete(oldestKey);
+					}
 				}
+				this.statsCache.set(cacheKey, {
+					data: { ...stats },
+					expiresAt: Date.now() + ttl,
+				});
 			}
 
 			return stats;

package/src/scheduler/services/job-scheduler.ts

@@ -1,4 +1,4 @@
-import type { Document } from 'mongodb';
+import { BSON, type Document } from 'mongodb';
 
 import {
 	type EnqueueOptions,
@@ -7,7 +7,7 @@ import {
 	type PersistedJob,
 	type ScheduleOptions,
 } from '@/jobs';
-import { ConnectionError, getNextCronDate, MonqueError } from '@/shared';
+import { ConnectionError, getNextCronDate, MonqueError, PayloadTooLargeError } from '@/shared';
 
 import type { SchedulerContext } from './types.js';
 
@@ -22,6 +22,41 @@ import type { SchedulerContext } from './types.js';
 export class JobScheduler {
 	constructor(private readonly ctx: SchedulerContext) {}
 
+	/**
+	 * Validate that the job data payload does not exceed the configured maximum BSON byte size.
+	 *
+	 * @param data - The job data payload to validate
+	 * @throws {PayloadTooLargeError} If the payload exceeds `maxPayloadSize`
+	 */
+	private validatePayloadSize(data: unknown): void {
+		const maxSize = this.ctx.options.maxPayloadSize;
+		if (maxSize === undefined) {
+			return;
+		}
+
+		let size: number;
+		try {
+			size = BSON.calculateObjectSize({ data } as Document);
+		} catch (error) {
+			const cause = error instanceof Error ? error : new Error(String(error));
+			const sizeError = new PayloadTooLargeError(
+				`Failed to calculate job payload size: ${cause.message}`,
+				-1,
+				maxSize,
+			);
+			sizeError.cause = cause;
+			throw sizeError;
+		}
+
+		if (size > maxSize) {
+			throw new PayloadTooLargeError(
+				`Job payload exceeds maximum size: ${size} bytes > ${maxSize} bytes`,
+				size,
+				maxSize,
+			);
+		}
+	}
+
 	/**
 	 * Enqueue a job for processing.
 	 *
@@ -40,6 +75,7 @@ export class JobScheduler {
 	 * @param options - Scheduling and deduplication options
 	 * @returns Promise resolving to the created or existing job document
 	 * @throws {ConnectionError} If database operation fails or scheduler not initialized
+	 * @throws {PayloadTooLargeError} If payload exceeds configured `maxPayloadSize`
 	 *
 	 * @example Basic job enqueueing
 	 * ```typescript
@@ -67,6 +103,7 @@ export class JobScheduler {
 	 * ```
 	 */
 	async enqueue<T>(name: string, data: T, options: EnqueueOptions = {}): Promise<PersistedJob<T>> {
+		this.validatePayloadSize(data);
 		const now = new Date();
 		const job: Omit<Job<T>, '_id'> = {
 			name,
@@ -174,6 +211,7 @@ export class JobScheduler {
 	 * @returns Promise resolving to the created job document with `repeatInterval` set
 	 * @throws {InvalidCronError} If cron expression is invalid
 	 * @throws {ConnectionError} If database operation fails or scheduler not initialized
+	 * @throws {PayloadTooLargeError} If payload exceeds configured `maxPayloadSize`
 	 *
 	 * @example Hourly cleanup job
 	 * ```typescript
@@ -204,6 +242,8 @@ export class JobScheduler {
 		data: T,
 		options: ScheduleOptions = {},
 	): Promise<PersistedJob<T>> {
+		this.validatePayloadSize(data);
+
 		// Validate cron and get next run date (throws InvalidCronError if invalid)
 		const nextRunAt = getNextCronDate(cron);
 

package/src/scheduler/services/lifecycle-manager.ts

@@ -0,0 +1,154 @@
+import type { DeleteResult } from 'mongodb';
+
+import { JobStatus } from '@/jobs';
+import { toError } from '@/shared';
+
+import type { SchedulerContext } from './types.js';
+
+/**
+ * Default retention check interval (1 hour).
+ */
+const DEFAULT_RETENTION_INTERVAL = 3600_000;
+
+/**
+ * Callbacks for timer-driven operations.
+ *
+ * These are provided by the Monque facade to wire LifecycleManager's timers
+ * to JobProcessor methods without creating a direct dependency.
+ */
+interface TimerCallbacks {
+	/** Poll for pending jobs */
+	poll: () => Promise<void>;
+	/** Update heartbeats for claimed jobs */
+	updateHeartbeats: () => Promise<void>;
+}
+
+/**
+ * Manages scheduler lifecycle timers and job cleanup.
+ *
+ * Owns poll interval, heartbeat interval, cleanup interval, and the
+ * cleanupJobs logic. Extracted from Monque to keep the facade thin.
+ *
+ * @internal Not part of public API.
+ */
+export class LifecycleManager {
+	private readonly ctx: SchedulerContext;
+	private pollIntervalId: ReturnType<typeof setInterval> | null = null;
+	private heartbeatIntervalId: ReturnType<typeof setInterval> | null = null;
+	private cleanupIntervalId: ReturnType<typeof setInterval> | null = null;
+
+	constructor(ctx: SchedulerContext) {
+		this.ctx = ctx;
+	}
+
+	/**
+	 * Start all lifecycle timers.
+	 *
+	 * Sets up poll interval, heartbeat interval, and (if configured)
+	 * cleanup interval. Runs an initial poll immediately.
+	 *
+	 * @param callbacks - Functions to invoke on each timer tick
+	 */
+	startTimers(callbacks: TimerCallbacks): void {
+		// Set up polling as backup (runs at configured interval)
+		this.pollIntervalId = setInterval(() => {
+			callbacks.poll().catch((error: unknown) => {
+				this.ctx.emit('job:error', { error: toError(error) });
+			});
+		}, this.ctx.options.pollInterval);
+
+		// Start heartbeat interval for claimed jobs
+		this.heartbeatIntervalId = setInterval(() => {
+			callbacks.updateHeartbeats().catch((error: unknown) => {
+				this.ctx.emit('job:error', { error: toError(error) });
+			});
+		}, this.ctx.options.heartbeatInterval);
+
+		// Start cleanup interval if retention is configured
+		if (this.ctx.options.jobRetention) {
+			const interval = this.ctx.options.jobRetention.interval ?? DEFAULT_RETENTION_INTERVAL;
+
+			// Run immediately on start
+			this.cleanupJobs().catch((error: unknown) => {
+				this.ctx.emit('job:error', { error: toError(error) });
+			});
+
+			this.cleanupIntervalId = setInterval(() => {
+				this.cleanupJobs().catch((error: unknown) => {
+					this.ctx.emit('job:error', { error: toError(error) });
+				});
+			}, interval);
+		}
+
+		// Run initial poll immediately to pick up any existing jobs
+		callbacks.poll().catch((error: unknown) => {
+			this.ctx.emit('job:error', { error: toError(error) });
+		});
+	}
+
+	/**
+	 * Stop all lifecycle timers.
+	 *
+	 * Clears poll, heartbeat, and cleanup intervals.
+	 */
+	stopTimers(): void {
+		if (this.cleanupIntervalId) {
+			clearInterval(this.cleanupIntervalId);
+			this.cleanupIntervalId = null;
+		}
+
+		if (this.pollIntervalId) {
+			clearInterval(this.pollIntervalId);
+			this.pollIntervalId = null;
+		}
+
+		if (this.heartbeatIntervalId) {
+			clearInterval(this.heartbeatIntervalId);
+			this.heartbeatIntervalId = null;
+		}
+	}
+
+	/**
+	 * Clean up old completed and failed jobs based on retention policy.
+	 *
+	 * - Removes completed jobs older than `jobRetention.completed`
+	 * - Removes failed jobs older than `jobRetention.failed`
+	 *
+	 * The cleanup runs concurrently for both statuses if configured.
+	 *
+	 * @returns Promise resolving when all deletion operations complete
+	 */
+	async cleanupJobs(): Promise<void> {
+		if (!this.ctx.options.jobRetention) {
+			return;
+		}
+
+		const { completed, failed } = this.ctx.options.jobRetention;
+		const now = Date.now();
+		const deletions: Promise<DeleteResult>[] = [];
+
+		if (completed != null) {
+			const cutoff = new Date(now - completed);
+			deletions.push(
+				this.ctx.collection.deleteMany({
+					status: JobStatus.COMPLETED,
+					updatedAt: { $lt: cutoff },
+				}),
+			);
+		}
+
+		if (failed != null) {
+			const cutoff = new Date(now - failed);
+			deletions.push(
+				this.ctx.collection.deleteMany({
+					status: JobStatus.FAILED,
+					updatedAt: { $lt: cutoff },
+				}),
+			);
+		}
+
+		if (deletions.length > 0) {
+			await Promise.all(deletions);
+		}
+	}
+}