mongo-job-scheduler 0.1.16 → 0.1.17

package/dist/core/scheduler.js

@@ -66,6 +66,7 @@ class Scheduler {
             dedupeKey: options.dedupeKey,
             priority: options.priority,
             concurrency: options.concurrency,
+            lockVersion: 0,
             createdAt: now,
             updatedAt: now,
         };
@@ -111,6 +112,7 @@ class Scheduler {
                 dedupeKey: options.dedupeKey,
                 priority: options.priority,
                 concurrency: options.concurrency,
+                lockVersion: 0,
             };
             if (isNaN(job.nextRunAt.getTime())) {
                 throw new Error("Invalid Date provided for runAt");

package/dist/store/in-memory-job-store.js

@@ -24,6 +24,7 @@ class InMemoryJobStore {
             ...job,
             _id: id,
             priority: job.priority ?? 5,
+            lockVersion: job.lockVersion ?? 0,
             createdAt: job.createdAt ?? new Date(),
             updatedAt: job.updatedAt ?? new Date(),
         };
@@ -65,6 +66,8 @@ class InMemoryJobStore {
                 job.status = "running";
                 job.lockedAt = now;
                 job.lockedBy = workerId;
+                job.lockUntil = new Date(now.getTime() + lockTimeoutMs);
+                job.lockVersion = (job.lockVersion ?? 0) + 1;
                 job.updatedAt = new Date();
                 job.lastRunAt = now;
                 return { ...job };
@@ -117,6 +120,7 @@ class InMemoryJobStore {
                 job.status = "pending";
                 job.lockedAt = undefined;
                 job.lockedBy = undefined;
+                job.lockUntil = undefined;
                 job.updatedAt = new Date();
                 recovered++;
             }

package/dist/store/mongo/mongo-job-store.d.ts

@@ -10,9 +10,6 @@ export declare class MongoJobStore implements JobStore {
     private readonly collection;
     private readonly defaultLockTimeoutMs;
     constructor(db: Db, options?: MongoJobStoreOptions);
-    /**
-     * Create necessary indexes for optimal query performance
-     */
     private ensureIndexes;
     create(job: Job): Promise<Job>;
     createBulk(jobs: Job[]): Promise<Job[]>;

package/dist/store/mongo/mongo-job-store.js

@@ -10,33 +10,23 @@ class MongoJobStore {
             console.error("Failed to create indexes:", err);
         });
     }
-    /**
-     * Create necessary indexes for optimal query performance
-     */
     async ensureIndexes() {
         await Promise.all([
-            // Primary index for job polling (findAndLockNext) with priority
             this.collection.createIndex({ status: 1, priority: 1, nextRunAt: 1 }, { background: true }),
-            // Index for deduplication
             this.collection.createIndex({ dedupeKey: 1 }, { unique: true, sparse: true, background: true }),
-            // Index for stale lock recovery
-            this.collection.createIndex({ lockedAt: 1 }, { sparse: true, background: true }),
-            // Index for concurrency counting
+            this.collection.createIndex({ lockUntil: 1 }, { sparse: true, background: true }),
             this.collection.createIndex({ name: 1, status: 1 }, { background: true }),
         ]);
     }
-    // --------------------------------------------------
-    // CREATE
-    // --------------------------------------------------
     async create(job) {
         const now = new Date();
-        // IMPORTANT: strip _id completely
         const { _id, ...jobWithoutId } = job;
         const doc = {
             ...jobWithoutId,
             status: job.status ?? "pending",
             attempts: job.attempts ?? 0,
             priority: job.priority ?? 5,
+            lockVersion: job.lockVersion ?? 0,
             createdAt: now,
             updatedAt: now,
         };
@@ -44,7 +34,6 @@ class MongoJobStore {
             delete doc.dedupeKey;
         }
         if (job.dedupeKey) {
-            // Idempotent insert
             const result = await this.collection.findOneAndUpdate({ dedupeKey: job.dedupeKey }, { $setOnInsert: doc }, { upsert: true, returnDocument: "after" });
             return result;
         }
@@ -54,13 +43,13 @@ class MongoJobStore {
     async createBulk(jobs) {
         const now = new Date();
         const docs = jobs.map((job) => {
-            // IMPORTANT: strip _id completely
             const { _id, ...jobWithoutId } = job;
             const doc = {
                 ...jobWithoutId,
                 status: job.status ?? "pending",
                 attempts: job.attempts ?? 0,
                 priority: job.priority ?? 5,
+                lockVersion: job.lockVersion ?? 0,
                 createdAt: now,
                 updatedAt: now,
             };
@@ -77,73 +66,150 @@ class MongoJobStore {
             _id: result.insertedIds[index],
         }));
     }
-    // --------------------------------------------------
-    // ATOMIC FIND & LOCK (with concurrency support)
-    // --------------------------------------------------
+    // Atomic find & lock with version-based optimistic locking
     async findAndLockNext(options) {
         const { now, workerId, lockTimeoutMs } = options;
-        const lockExpiry = new Date(now.getTime() - lockTimeoutMs);
-        // Base query for runnable jobs
-        const baseQuery = {
-            status: "pending",
-            nextRunAt: { $lte: now },
+        const lockUntil = new Date(now.getTime() + lockTimeoutMs);
+        // Fast path: jobs without concurrency limits
+        const simpleQuery = {
             $or: [
-                { lockedAt: { $exists: false } },
-                { lockedAt: { $lte: lockExpiry } },
+                // Pending jobs (not locked)
+                {
+                    status: "pending",
+                    nextRunAt: { $lte: now },
+                    $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                },
+                // Stale running jobs (lock expired - crash recovery)
+                {
+                    status: "running",
+                    nextRunAt: { $lte: now },
+                    lockUntil: { $lte: now },
+                },
+            ],
+            $and: [
+                { $or: [{ concurrency: { $exists: false } }, { concurrency: null }] },
             ],
         };
-        // Try to find and lock a job, respecting concurrency limits
-        // We use a loop to handle cases where a job has a concurrency limit
-        const maxAttempts = 10; // Prevent infinite loops
+        const simpleResult = await this.collection.findOneAndUpdate(simpleQuery, {
+            $set: {
+                lockedAt: now,
+                lockedBy: workerId,
+                lockUntil: lockUntil,
+                status: "running",
+                lastRunAt: now,
+                updatedAt: now,
+            },
+            $inc: { lockVersion: 1 },
+        }, {
+            sort: { priority: 1, nextRunAt: 1 },
+            returnDocument: "after",
+        });
+        if (simpleResult) {
+            return simpleResult;
+        }
+        // Now handle jobs with concurrency limits
+        // We need to check concurrency before locking
+        const maxAttempts = 20;
+        const checkedNames = new Set();
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
-            // Find candidate without locking first
-            const candidate = await this.collection.findOne(baseQuery, {
+            // Find a candidate with concurrency limit that we haven't checked yet
+            const concurrencyQuery = {
+                $or: [
+                    {
+                        status: "pending",
+                        nextRunAt: { $lte: now },
+                        // Pending jobs should not have lockedBy set
+                        $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                    },
+                    {
+                        // Stale running jobs (lock expired)
+                        status: "running",
+                        nextRunAt: { $lte: now },
+                        lockUntil: { $lte: now },
+                    },
+                ],
+                concurrency: { $exists: true, $gt: 0 },
+            };
+            if (checkedNames.size > 0) {
+                concurrencyQuery.name = { $nin: Array.from(checkedNames) };
+            }
+            const candidate = await this.collection.findOne(concurrencyQuery, {
                 sort: { priority: 1, nextRunAt: 1 },
-                skip: attempt, // Skip previously checked candidates
+                projection: { name: 1, concurrency: 1, lockVersion: 1 },
             });
             if (!candidate) {
-                return null; // No more candidates
+                return null; // No more candidates with concurrency limits
             }
-            // Check concurrency limit if defined
-            if (candidate.concurrency !== undefined && candidate.concurrency > 0) {
-                const runningCount = await this.collection.countDocuments({
-                    name: candidate.name,
-                    status: "running",
-                });
-                if (runningCount >= candidate.concurrency) {
-                    // At concurrency limit, skip this job and try next
-                    continue;
-                }
+            const runningCount = await this.collection.countDocuments({
+                name: candidate.name,
+                status: "running",
+            });
+            if (runningCount >= candidate.concurrency) {
+                // At limit for this job name, skip all jobs with this name
+                checkedNames.add(candidate.name);
+                continue;
             }
-            // Attempt atomic lock on this specific job
-            const result = await this.collection.findOneAndUpdate({
-                _id: candidate._id,
-                status: "pending", // Re-verify status
+            const lockResult = await this.collection.findOneAndUpdate({
+                name: candidate.name,
+                concurrency: candidate.concurrency,
                 $or: [
-                    { lockedAt: { $exists: false } },
-                    { lockedAt: { $lte: lockExpiry } },
+                    {
+                        status: "pending",
+                        $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                    },
+                    {
+                        status: "running",
+                        lockUntil: { $lte: now },
+                    },
                 ],
+                nextRunAt: { $lte: now },
             }, {
                 $set: {
                     lockedAt: now,
                     lockedBy: workerId,
+                    lockUntil: lockUntil,
                     status: "running",
                     lastRunAt: now,
                     updatedAt: now,
                 },
+                $inc: { lockVersion: 1 },
             }, {
+                sort: { priority: 1, nextRunAt: 1 },
                 returnDocument: "after",
             });
-            if (result) {
-                return result;
+            if (lockResult) {
+                // Verify concurrency wasn't exceeded by race condition
+                const currentRunning = await this.collection.countDocuments({
+                    name: lockResult.name,
+                    status: "running",
+                });
+                if (currentRunning > lockResult.concurrency) {
+                    // We exceeded concurrency - release this job back to pending
+                    await this.collection.updateOne({
+                        _id: lockResult._id,
+                        lockedBy: workerId,
+                        lockVersion: lockResult.lockVersion,
+                    }, {
+                        $set: {
+                            status: "pending",
+                            updatedAt: new Date(),
+                        },
+                        $unset: {
+                            lockedAt: "",
+                            lockedBy: "",
+                            lockUntil: "",
+                            lastRunAt: "",
+                        },
+                    });
+                    continue;
+                }
+                return lockResult;
             }
-            // Job was taken by another worker, try next
+            // Lock failed (another worker got it), try next job name
+            checkedNames.add(candidate.name);
         }
         return null;
     }
-    // --------------------------------------------------
-    // MARK COMPLETED
-    // --------------------------------------------------
     async markCompleted(id) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -153,12 +219,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // MARK FAILED
-    // --------------------------------------------------
     async markFailed(id, error) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -169,12 +233,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // RESCHEDULE
-    // --------------------------------------------------
     async reschedule(id, nextRunAt, updates) {
         const result = await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -186,12 +248,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // CANCEL
-    // --------------------------------------------------
     async cancel(id) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -201,6 +261,7 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
@@ -210,14 +271,14 @@ class MongoJobStore {
             return null;
         return doc;
     }
-    // --------------------------------------------------
-    // RECOVER STALE JOBS
-    // --------------------------------------------------
     async recoverStaleJobs(options) {
         const { now, lockTimeoutMs } = options;
         const expiry = new Date(now.getTime() - lockTimeoutMs);
         const result = await this.collection.updateMany({
-            lockedAt: { $lte: expiry },
+            $or: [
+                { lockUntil: { $lte: now } },
+                { lockUntil: { $exists: false }, lockedAt: { $lte: expiry } },
+            ],
         }, {
             $set: {
                 status: "pending",
@@ -226,6 +287,7 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
         return result.modifiedCount;
@@ -240,7 +302,9 @@ class MongoJobStore {
             $set: {
                 lockedAt: now,
                 updatedAt: now,
+                lockUntil: new Date(now.getTime() + this.defaultLockTimeoutMs),
             },
+            $inc: { lockVersion: 1 },
         });
         if (result.matchedCount === 0) {
             throw new Error("Job lock lost or owner changed");
@@ -268,9 +332,6 @@ class MongoJobStore {
             $set.concurrency = updates.concurrency;
         await this.collection.updateOne({ _id: id }, { $set });
     }
-    // --------------------------------------------------
-    // COUNT RUNNING (for concurrency limits)
-    // --------------------------------------------------
     async countRunning(jobName) {
         return this.collection.countDocuments({
             name: jobName,

package/dist/types/job.d.ts

@@ -11,6 +11,15 @@ export interface Job<Data = unknown> {
     lastScheduledAt?: Date;
     lockedAt?: Date;
     lockedBy?: string;
+    /**
+     * Lock expiry time. Job can be taken by another worker after this time.
+     */
+    lockUntil?: Date;
+    /**
+     * Optimistic locking version. Incremented on each lock acquisition.
+     * Prevents race conditions in distributed environments.
+     */
+    lockVersion?: number;
     attempts: number;
     lastError?: string;
     retry?: RetryOptions | number;

package/dist/worker/worker.js

@@ -44,7 +44,6 @@ class Worker {
     }
     async loop() {
         while (this.running) {
-            // stop requested before poll
             if (!this.running)
                 break;
             const job = await this.store.findAndLockNext({
@@ -52,7 +51,6 @@ class Worker {
                 workerId: this.workerId,
                 lockTimeoutMs: this.lockTimeout,
             });
-            // stop requested after polling
             if (!this.running)
                 break;
             if (!job) {
@@ -65,24 +63,7 @@ class Worker {
     async execute(job) {
         this.emitter.emitSafe("job:start", job);
         const now = Date.now();
-        // ---------------------------
-        // CRON: pre-schedule BEFORE execution
-        // ---------------------------
-        if (job.repeat?.cron) {
-            let base = job.lastScheduledAt ?? job.nextRunAt ?? new Date(now);
-            let next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
-            // skip missed cron slots
-            while (next.getTime() <= now) {
-                base = next;
-                next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
-            }
-            // persist schedule immediately
-            job.lastScheduledAt = next;
-            await this.store.reschedule(job._id, next);
-        }
-        // ---------------------------
-        // HEARTBEAT
-        // ---------------------------
+        // Heartbeat to prevent lock expiry during long jobs
         const heartbeatIntervalMs = Math.max(50, this.lockTimeout / 2);
         const heartbeatParams = {
             jobId: job._id,
@@ -105,16 +86,42 @@ class Worker {
         };
         const heartbeatPromise = heartbeatLoop();
         try {
+            // Verify we still own the lock before any modifications
+            // (another worker might have stolen it via stale recovery)
             const current = await this.store.findById(job._id);
-            if (current && current.status === "cancelled") {
+            if (!current) {
+                stopHeartbeat = true;
+                return;
+            }
+            if (current.status === "cancelled") {
                 this.emitter.emitSafe("job:complete", job);
-                stopHeartbeat = true; // stop fast
+                stopHeartbeat = true;
+                return;
+            }
+            if (current.lockedBy !== this.workerId) {
+                this.emitter.emitSafe("worker:error", new Error(`Lock stolen for job ${job._id}: owned by ${current.lockedBy}, we are ${this.workerId}`));
+                stopHeartbeat = true;
                 return;
             }
+            if (current.status !== "running") {
+                this.emitter.emitSafe("worker:error", new Error(`Job ${job._id} is no longer running (status: ${current.status})`));
+                stopHeartbeat = true;
+                return;
+            }
+            // CRON: pre-schedule before execution (after lock verification)
+            if (job.repeat?.cron) {
+                let base = job.lastScheduledAt ?? job.nextRunAt ?? new Date(now);
+                let next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
+                // skip missed cron slots
+                while (next.getTime() <= now) {
+                    base = next;
+                    next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
+                }
+                job.lastScheduledAt = next;
+                await this.store.reschedule(job._id, next);
+            }
             await this.handler(job);
-            // ---------------------------
-            // INTERVAL: schedule AFTER execution
-            // ---------------------------
+            // INTERVAL: schedule after execution
             if (job.repeat?.every != null) {
                 const next = new Date(Date.now() + Math.max(job.repeat.every, 100));
                 await this.store.reschedule(job._id, next);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mongo-job-scheduler",
-  "version": "0.1.16",
+  "version": "0.1.17",
   "description": "Production-grade MongoDB-backed job scheduler with retries, cron, timezone support, and crash recovery",
   "license": "MIT",
   "author": "Darshan Bhut",
@@ -43,7 +43,6 @@
     "build": "tsc -p tsconfig.build.json",
     "test": "jest",
     "test:mongo": "jest tests/mongo",
-    "test:stress": "jest tests/stress",
     "prepublishOnly": "npm run build && npm test"
   },
   "dependencies": {