@monque/core 1.1.0 → 1.1.2

package/src/scheduler/services/job-processor.ts

@@ -0,0 +1,301 @@
+import { isPersistedJob, type Job, JobStatus, type PersistedJob } from '@/jobs';
+import { calculateBackoff, getNextCronDate } from '@/shared';
+import type { WorkerRegistration } from '@/workers';
+
+import type { SchedulerContext } from './types.js';
+
+/**
+ * Internal service for job processing and execution.
+ *
+ * Manages the poll loop, atomic job acquisition, handler execution,
+ * and job completion/failure with exponential backoff retry logic.
+ *
+ * @internal Not part of public API.
+ */
+export class JobProcessor {
+	constructor(private readonly ctx: SchedulerContext) {}
+
+	/**
+	 * Poll for available jobs and process them.
+	 *
+	 * Called at regular intervals (configured by `pollInterval`). For each registered worker,
+	 * attempts to acquire jobs up to the worker's available concurrency slots.
+	 * Aborts early if the scheduler is stopping (`isRunning` is false).
+	 */
+	async poll(): Promise<void> {
+		if (!this.ctx.isRunning()) {
+			return;
+		}
+
+		for (const [name, worker] of this.ctx.workers) {
+			// Check if worker has capacity
+			const availableSlots = worker.concurrency - worker.activeJobs.size;
+
+			if (availableSlots <= 0) {
+				continue;
+			}
+
+			// Try to acquire jobs up to available slots
+			for (let i = 0; i < availableSlots; i++) {
+				if (!this.ctx.isRunning()) {
+					return;
+				}
+				const job = await this.acquireJob(name);
+
+				if (job) {
+					this.processJob(job, worker).catch((error: unknown) => {
+						this.ctx.emit('job:error', { error: error as Error, job });
+					});
+				} else {
+					// No more jobs available for this worker
+					break;
+				}
+			}
+		}
+	}
+
+	/**
+	 * Atomically acquire a pending job for processing using the claimedBy pattern.
+	 *
+	 * Uses MongoDB's `findOneAndUpdate` with atomic operations to ensure only one scheduler
+	 * instance can claim a job. The query ensures the job is:
+	 * - In pending status
+	 * - Has nextRunAt <= now
+	 * - Is not claimed by another instance (claimedBy is null/undefined)
+	 *
+	 * Returns `null` immediately if scheduler is stopping (`isRunning` is false).
+	 *
+	 * @param name - The job type to acquire
+	 * @returns The acquired job with updated status, claimedBy, and heartbeat info, or `null` if no jobs available
+	 */
+	async acquireJob(name: string): Promise<PersistedJob | null> {
+		if (!this.ctx.isRunning()) {
+			return null;
+		}
+
+		const now = new Date();
+
+		const result = await this.ctx.collection.findOneAndUpdate(
+			{
+				name,
+				status: JobStatus.PENDING,
+				nextRunAt: { $lte: now },
+				$or: [{ claimedBy: null }, { claimedBy: { $exists: false } }],
+			},
+			{
+				$set: {
+					status: JobStatus.PROCESSING,
+					claimedBy: this.ctx.instanceId,
+					lockedAt: now,
+					lastHeartbeat: now,
+					heartbeatInterval: this.ctx.options.heartbeatInterval,
+					updatedAt: now,
+				},
+			},
+			{
+				sort: { nextRunAt: 1 },
+				returnDocument: 'after',
+			},
+		);
+
+		if (!this.ctx.isRunning()) {
+			return null;
+		}
+
+		if (!result) {
+			return null;
+		}
+
+		return this.ctx.documentToPersistedJob(result);
+	}
+
+	/**
+	 * Execute a job using its registered worker handler.
+	 *
+	 * Tracks the job as active during processing, emits lifecycle events, and handles
+	 * both success and failure cases. On success, calls `completeJob()`. On failure,
+	 * calls `failJob()` which implements exponential backoff retry logic.
+	 *
+	 * @param job - The job to process
+	 * @param worker - The worker registration containing the handler and active job tracking
+	 */
+	async processJob(job: PersistedJob, worker: WorkerRegistration): Promise<void> {
+		const jobId = job._id.toString();
+		worker.activeJobs.set(jobId, job);
+
+		const startTime = Date.now();
+		this.ctx.emit('job:start', job);
+
+		try {
+			await worker.handler(job);
+
+			// Job completed successfully
+			const duration = Date.now() - startTime;
+			await this.completeJob(job);
+			this.ctx.emit('job:complete', { job, duration });
+		} catch (error) {
+			// Job failed
+			const err = error instanceof Error ? error : new Error(String(error));
+			await this.failJob(job, err);
+
+			const willRetry = job.failCount + 1 < this.ctx.options.maxRetries;
+			this.ctx.emit('job:fail', { job, error: err, willRetry });
+		} finally {
+			worker.activeJobs.delete(jobId);
+		}
+	}
+
+	/**
+	 * Mark a job as completed successfully.
+	 *
+	 * For recurring jobs (with `repeatInterval`), schedules the next run based on the cron
+	 * expression and resets `failCount` to 0. For one-time jobs, sets status to `completed`.
+	 * Clears `lockedAt` and `failReason` fields in both cases.
+	 *
+	 * @param job - The job that completed successfully
+	 */
+	async completeJob(job: Job): Promise<void> {
+		if (!isPersistedJob(job)) {
+			return;
+		}
+
+		if (job.repeatInterval) {
+			// Recurring job - schedule next run
+			const nextRunAt = getNextCronDate(job.repeatInterval);
+			await this.ctx.collection.updateOne(
+				{ _id: job._id },
+				{
+					$set: {
+						status: JobStatus.PENDING,
+						nextRunAt,
+						failCount: 0,
+						updatedAt: new Date(),
+					},
+					$unset: {
+						lockedAt: '',
+						claimedBy: '',
+						lastHeartbeat: '',
+						heartbeatInterval: '',
+						failReason: '',
+					},
+				},
+			);
+		} else {
+			// One-time job - mark as completed
+			await this.ctx.collection.updateOne(
+				{ _id: job._id },
+				{
+					$set: {
+						status: JobStatus.COMPLETED,
+						updatedAt: new Date(),
+					},
+					$unset: {
+						lockedAt: '',
+						claimedBy: '',
+						lastHeartbeat: '',
+						heartbeatInterval: '',
+						failReason: '',
+					},
+				},
+			);
+			job.status = JobStatus.COMPLETED;
+		}
+	}
+
+	/**
+	 * Handle job failure with exponential backoff retry logic.
+	 *
+	 * Increments `failCount` and calculates next retry time using exponential backoff:
+	 * `nextRunAt = 2^failCount × baseRetryInterval` (capped by optional `maxBackoffDelay`).
+	 *
+	 * If `failCount >= maxRetries`, marks job as permanently `failed`. Otherwise, resets
+	 * to `pending` status for retry. Stores error message in `failReason` field.
+	 *
+	 * @param job - The job that failed
+	 * @param error - The error that caused the failure
+	 */
+	async failJob(job: Job, error: Error): Promise<void> {
+		if (!isPersistedJob(job)) {
+			return;
+		}
+
+		const newFailCount = job.failCount + 1;
+
+		if (newFailCount >= this.ctx.options.maxRetries) {
+			// Permanent failure
+			await this.ctx.collection.updateOne(
+				{ _id: job._id },
+				{
+					$set: {
+						status: JobStatus.FAILED,
+						failCount: newFailCount,
+						failReason: error.message,
+						updatedAt: new Date(),
+					},
+					$unset: {
+						lockedAt: '',
+						claimedBy: '',
+						lastHeartbeat: '',
+						heartbeatInterval: '',
+					},
+				},
+			);
+		} else {
+			// Schedule retry with exponential backoff
+			const nextRunAt = calculateBackoff(
+				newFailCount,
+				this.ctx.options.baseRetryInterval,
+				this.ctx.options.maxBackoffDelay,
+			);
+
+			await this.ctx.collection.updateOne(
+				{ _id: job._id },
+				{
+					$set: {
+						status: JobStatus.PENDING,
+						failCount: newFailCount,
+						failReason: error.message,
+						nextRunAt,
+						updatedAt: new Date(),
+					},
+					$unset: {
+						lockedAt: '',
+						claimedBy: '',
+						lastHeartbeat: '',
+						heartbeatInterval: '',
+					},
+				},
+			);
+		}
+	}
+
+	/**
+	 * Update heartbeats for all jobs claimed by this scheduler instance.
+	 *
+	 * This method runs periodically while the scheduler is running to indicate
+	 * that jobs are still being actively processed.
+	 *
+	 * `lastHeartbeat` is primarily an observability signal (monitoring/debugging).
+	 * Stale recovery is based on `lockedAt` + `lockTimeout`.
+	 */
+	async updateHeartbeats(): Promise<void> {
+		if (!this.ctx.isRunning()) {
+			return;
+		}
+
+		const now = new Date();
+
+		await this.ctx.collection.updateMany(
+			{
+				claimedBy: this.ctx.instanceId,
+				status: JobStatus.PROCESSING,
+			},
+			{
+				$set: {
+					lastHeartbeat: now,
+					updatedAt: now,
+				},
+			},
+		);
+	}
+}

package/src/scheduler/services/job-query.ts

@@ -0,0 +1,411 @@
+import type { Document, Filter, ObjectId, WithId } from 'mongodb';
+
+import {
+	CursorDirection,
+	type CursorDirectionType,
+	type CursorOptions,
+	type CursorPage,
+	type GetJobsFilter,
+	type JobSelector,
+	JobStatus,
+	type PersistedJob,
+	type QueueStats,
+} from '@/jobs';
+import { AggregationTimeoutError, ConnectionError } from '@/shared';
+
+import { buildSelectorQuery, decodeCursor, encodeCursor } from '../helpers.js';
+import type { SchedulerContext } from './types.js';
+
+/**
+ * Internal service for job query operations.
+ *
+ * Provides read-only access to jobs with filtering and cursor-based pagination.
+ * All queries use efficient index-backed access patterns.
+ *
+ * @internal Not part of public API - use Monque class methods instead.
+ */
+export class JobQueryService {
+	constructor(private readonly ctx: SchedulerContext) {}
+
+	/**
+	 * Get a single job by its MongoDB ObjectId.
+	 *
+	 * Useful for retrieving job details when you have a job ID from events,
+	 * logs, or stored references.
+	 *
+	 * @template T - The expected type of the job data payload
+	 * @param id - The job's ObjectId
+	 * @returns Promise resolving to the job if found, null otherwise
+	 * @throws {ConnectionError} If scheduler not initialized
+	 *
+	 * @example Look up job from event
+	 * ```typescript
+	 * monque.on('job:fail', async ({ job }) => {
+	 *   // Later, retrieve the job to check its status
+	 *   const currentJob = await monque.getJob(job._id);
+	 *   console.log(`Job status: ${currentJob?.status}`);
+	 * });
+	 * ```
+	 *
+	 * @example Admin endpoint
+	 * ```typescript
+	 * app.get('/jobs/:id', async (req, res) => {
+	 *   const job = await monque.getJob(new ObjectId(req.params.id));
+	 *   if (!job) {
+	 *     return res.status(404).json({ error: 'Job not found' });
+	 *   }
+	 *   res.json(job);
+	 * });
+	 * ```
+	 */
+	async getJob<T = unknown>(id: ObjectId): Promise<PersistedJob<T> | null> {
+		try {
+			const doc = await this.ctx.collection.findOne({ _id: id });
+			if (!doc) {
+				return null;
+			}
+			return this.ctx.documentToPersistedJob<T>(doc as WithId<Document>);
+		} catch (error) {
+			const message = error instanceof Error ? error.message : 'Unknown error during getJob';
+			throw new ConnectionError(
+				`Failed to get job: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
+	}
+
+	/**
+	 * Query jobs from the queue with optional filters.
+	 *
+	 * Provides read-only access to job data for monitoring, debugging, and
+	 * administrative purposes. Results are ordered by `nextRunAt` ascending.
+	 *
+	 * @template T - The expected type of the job data payload
+	 * @param filter - Optional filter criteria
+	 * @returns Promise resolving to array of matching jobs
+	 * @throws {ConnectionError} If scheduler not initialized
+	 *
+	 * @example Get all pending jobs
+	 * ```typescript
+	 * const pendingJobs = await monque.getJobs({ status: JobStatus.PENDING });
+	 * console.log(`${pendingJobs.length} jobs waiting`);
+	 * ```
+	 *
+	 * @example Get failed email jobs
+	 * ```typescript
+	 * const failedEmails = await monque.getJobs({
+	 *   name: 'send-email',
+	 *   status: JobStatus.FAILED,
+	 * });
+	 * for (const job of failedEmails) {
+	 *   console.error(`Job ${job._id} failed: ${job.failReason}`);
+	 * }
+	 * ```
+	 *
+	 * @example Paginated job listing
+	 * ```typescript
+	 * const page1 = await monque.getJobs({ limit: 50, skip: 0 });
+	 * const page2 = await monque.getJobs({ limit: 50, skip: 50 });
+	 * ```
+	 *
+	 * @example Use with type guards from @monque/core
+	 * ```typescript
+	 * import { isPendingJob, isRecurringJob } from '@monque/core';
+	 *
+	 * const jobs = await monque.getJobs();
+	 * const pendingRecurring = jobs.filter(job => isPendingJob(job) && isRecurringJob(job));
+	 * ```
+	 */
+	async getJobs<T = unknown>(filter: GetJobsFilter = {}): Promise<PersistedJob<T>[]> {
+		const query: Document = {};
+
+		if (filter.name !== undefined) {
+			query['name'] = filter.name;
+		}
+
+		if (filter.status !== undefined) {
+			if (Array.isArray(filter.status)) {
+				query['status'] = { $in: filter.status };
+			} else {
+				query['status'] = filter.status;
+			}
+		}
+
+		const limit = filter.limit ?? 100;
+		const skip = filter.skip ?? 0;
+
+		try {
+			const cursor = this.ctx.collection.find(query).sort({ nextRunAt: 1 }).skip(skip).limit(limit);
+
+			const docs = await cursor.toArray();
+			return docs.map((doc) => this.ctx.documentToPersistedJob<T>(doc));
+		} catch (error) {
+			const message = error instanceof Error ? error.message : 'Unknown error during getJobs';
+			throw new ConnectionError(
+				`Failed to query jobs: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
+	}
+
+	/**
+	 * Get a paginated list of jobs using opaque cursors.
+	 *
+	 * Provides stable pagination for large job lists. Supports forward and backward
+	 * navigation, filtering, and efficient database access via index-based cursor queries.
+	 *
+	 * @template T - The job data payload type
+	 * @param options - Pagination options (cursor, limit, direction, filter)
+	 * @returns Page of jobs with next/prev cursors
+	 * @throws {InvalidCursorError} If the provided cursor is malformed
+	 * @throws {ConnectionError} If database operation fails or scheduler not initialized
+	 *
+	 * @example List pending jobs
+	 * ```typescript
+	 * const page = await monque.getJobsWithCursor({
+	 *   limit: 20,
+	 *   filter: { status: 'pending' }
+	 * });
+	 * const jobs = page.jobs;
+	 *
+	 * // Get next page
+	 * if (page.hasNextPage) {
+	 *   const page2 = await monque.getJobsWithCursor({
+	 *     cursor: page.cursor,
+	 *     limit: 20
+	 *   });
+	 * }
+	 * ```
+	 */
+	async getJobsWithCursor<T = unknown>(options: CursorOptions = {}): Promise<CursorPage<T>> {
+		const limit = options.limit ?? 50;
+		// Default to forward if not specified.
+		const direction: CursorDirectionType = options.direction ?? CursorDirection.FORWARD;
+		let anchorId: ObjectId | null = null;
+
+		if (options.cursor) {
+			const decoded = decodeCursor(options.cursor);
+			anchorId = decoded.id;
+		}
+
+		// Build base query from filters
+		const query: Filter<Document> = options.filter ? buildSelectorQuery(options.filter) : {};
+
+		// Add cursor condition to query
+		const sortDir = direction === CursorDirection.FORWARD ? 1 : -1;
+
+		if (anchorId) {
+			if (direction === CursorDirection.FORWARD) {
+				query._id = { ...query._id, $gt: anchorId };
+			} else {
+				query._id = { ...query._id, $lt: anchorId };
+			}
+		}
+
+		// Fetch limit + 1 to detect hasNext/hasPrev
+		const fetchLimit = limit + 1;
+
+		// Sort: Always deterministic.
+		let docs: WithId<Document>[];
+		try {
+			docs = await this.ctx.collection
+				.find(query)
+				.sort({ _id: sortDir })
+				.limit(fetchLimit)
+				.toArray();
+		} catch (error) {
+			const message =
+				error instanceof Error ? error.message : 'Unknown error during getJobsWithCursor';
+			throw new ConnectionError(
+				`Failed to query jobs with cursor: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
+
+		let hasMore = false;
+		if (docs.length > limit) {
+			hasMore = true;
+			docs.pop(); // Remove the extra item
+		}
+
+		if (direction === CursorDirection.BACKWARD) {
+			docs.reverse();
+		}
+
+		const jobs = docs.map((doc) => this.ctx.documentToPersistedJob<T>(doc as WithId<Document>));
+
+		let nextCursor: string | null = null;
+
+		if (jobs.length > 0) {
+			const lastJob = jobs[jobs.length - 1];
+			// Check for existence to satisfy strict null checks/noUncheckedIndexedAccess
+			if (lastJob) {
+				nextCursor = encodeCursor(lastJob._id, direction);
+			}
+		}
+
+		let hasNextPage = false;
+		let hasPreviousPage = false;
+
+		// Determine availability of next/prev pages
+		if (direction === CursorDirection.FORWARD) {
+			hasNextPage = hasMore;
+			hasPreviousPage = !!anchorId;
+		} else {
+			hasNextPage = !!anchorId;
+			hasPreviousPage = hasMore;
+		}
+
+		return {
+			jobs,
+			cursor: nextCursor,
+			hasNextPage,
+			hasPreviousPage,
+		};
+	}
+
+	/**
+	 * Get aggregate statistics for the job queue.
+	 *
+	 * Uses MongoDB aggregation pipeline for efficient server-side calculation.
+	 * Returns counts per status and optional average processing duration for completed jobs.
+	 *
+	 * @param filter - Optional filter to scope statistics by job name
+	 * @returns Promise resolving to queue statistics
+	 * @throws {AggregationTimeoutError} If aggregation exceeds 30 second timeout
+	 * @throws {ConnectionError} If database operation fails
+	 *
+	 * @example Get overall queue statistics
+	 * ```typescript
+	 * const stats = await monque.getQueueStats();
+	 * console.log(`Pending: ${stats.pending}, Failed: ${stats.failed}`);
+	 * ```
+	 *
+	 * @example Get statistics for a specific job type
+	 * ```typescript
+	 * const emailStats = await monque.getQueueStats({ name: 'send-email' });
+	 * console.log(`${emailStats.total} email jobs in queue`);
+	 * ```
+	 */
+	async getQueueStats(filter?: Pick<JobSelector, 'name'>): Promise<QueueStats> {
+		const matchStage: Document = {};
+
+		if (filter?.name) {
+			matchStage['name'] = filter.name;
+		}
+
+		const pipeline: Document[] = [
+			// Optional match stage for filtering by name
+			...(Object.keys(matchStage).length > 0 ? [{ $match: matchStage }] : []),
+			// Facet to calculate counts and avg processing duration in parallel
+			{
+				$facet: {
+					// Count by status
+					statusCounts: [
+						{
+							$group: {
+								_id: '$status',
+								count: { $sum: 1 },
+							},
+						},
+					],
+					// Calculate average job lifetime for completed jobs.
+					// Uses createdAt → updatedAt (total lifetime = queue wait + processing)
+					// since completeJob() unsets lockedAt, making pure processing time unavailable.
+					avgDuration: [
+						{
+							$match: {
+								status: JobStatus.COMPLETED,
+							},
+						},
+						{
+							$group: {
+								_id: null,
+								avgMs: {
+									$avg: {
+										$subtract: ['$updatedAt', '$createdAt'],
+									},
+								},
+							},
+						},
+					],
+					// Total count
+					total: [{ $count: 'count' }],
+				},
+			},
+		];
+
+		try {
+			const results = await this.ctx.collection.aggregate(pipeline, { maxTimeMS: 30000 }).toArray();
+
+			const result = results[0];
+
+			// Initialize with zeros
+			const stats: QueueStats = {
+				pending: 0,
+				processing: 0,
+				completed: 0,
+				failed: 0,
+				cancelled: 0,
+				total: 0,
+			};
+
+			if (!result) {
+				return stats;
+			}
+
+			// Map status counts to stats
+			const statusCounts = result['statusCounts'] as Array<{ _id: string; count: number }>;
+			for (const entry of statusCounts) {
+				const status = entry._id;
+				const count = entry.count;
+
+				switch (status) {
+					case JobStatus.PENDING:
+						stats.pending = count;
+						break;
+					case JobStatus.PROCESSING:
+						stats.processing = count;
+						break;
+					case JobStatus.COMPLETED:
+						stats.completed = count;
+						break;
+					case JobStatus.FAILED:
+						stats.failed = count;
+						break;
+					case JobStatus.CANCELLED:
+						stats.cancelled = count;
+						break;
+				}
+			}
+
+			// Extract total
+			const totalResult = result['total'] as Array<{ count: number }>;
+			if (totalResult.length > 0 && totalResult[0]) {
+				stats.total = totalResult[0].count;
+			}
+
+			// Extract average processing duration
+			const avgDurationResult = result['avgDuration'] as Array<{ avgMs: number }>;
+			if (avgDurationResult.length > 0 && avgDurationResult[0]) {
+				const avgMs = avgDurationResult[0].avgMs;
+				if (typeof avgMs === 'number' && !Number.isNaN(avgMs)) {
+					stats.avgProcessingDurationMs = Math.round(avgMs);
+				}
+			}
+
+			return stats;
+		} catch (error) {
+			// Check for timeout error
+			if (error instanceof Error && error.message.includes('exceeded time limit')) {
+				throw new AggregationTimeoutError();
+			}
+
+			const message = error instanceof Error ? error.message : 'Unknown error during getQueueStats';
+			throw new ConnectionError(
+				`Failed to get queue stats: ${message}`,
+				error instanceof Error ? { cause: error } : undefined,
+			);
+		}
+	}
+}