mongo-job-scheduler 0.1.15 → 0.1.17

package/README.md

@@ -11,6 +11,7 @@ A production-grade MongoDB-backed job scheduler for Node.js with distributed loc
 - ✅ **Distributed locking** — safe for multiple instances
 - ✅ **Atomic job execution** — no double processing
 - ✅ **Job priority** — process important jobs first
+- ✅ **Concurrency limits** — rate-limit job execution
 - ✅ **Automatic retries** — with configurable backoff
 - ✅ **Cron scheduling** — timezone-aware, non-drifting
 - ✅ **Interval jobs** — repeated execution
@@ -207,6 +208,27 @@ await scheduler.updateJob(jobId, { priority: 2 });
 
 > **Priority Scale**: 1 (highest) → 10 (lowest). Jobs with equal priority run in FIFO order by `nextRunAt`.
 
+### Concurrency Limits
+
+Limit how many instances of a job type can run simultaneously (useful for rate-limiting API calls):
+
+```typescript
+// Max 5 concurrent "api-sync" jobs globally
+await scheduler.schedule({
+  name: "api-sync",
+  concurrency: 5,
+});
+
+// Max 2 concurrent "webhook" jobs
+await scheduler.schedule({
+  name: "webhook",
+  data: { url: "https://..." },
+  concurrency: 2,
+});
+```
+
+> **Note**: Concurrency is enforced globally across all workers. Jobs exceeding the limit wait until a slot frees up.
+
 ### Retries with Backoff
 
 ```typescript

package/dist/core/scheduler.js

@@ -42,6 +42,12 @@ class Scheduler {
                 throw new Error("Priority must be an integer between 1 and 10");
             }
         }
+        // Concurrency validation
+        if (options.concurrency !== undefined) {
+            if (!Number.isInteger(options.concurrency) || options.concurrency < 1) {
+                throw new Error("Concurrency must be a positive integer");
+            }
+        }
         // ------------------------
         // Normalize run time
         // ------------------------
@@ -59,6 +65,8 @@ class Scheduler {
             repeat: options.repeat,
             dedupeKey: options.dedupeKey,
             priority: options.priority,
+            concurrency: options.concurrency,
+            lockVersion: 0,
             createdAt: now,
             updatedAt: now,
         };
@@ -88,6 +96,12 @@ class Scheduler {
                     throw new Error("Priority must be an integer between 1 and 10");
                 }
             }
+            // Concurrency validation
+            if (options.concurrency !== undefined) {
+                if (!Number.isInteger(options.concurrency) || options.concurrency < 1) {
+                    throw new Error("Concurrency must be a positive integer");
+                }
+            }
             const job = {
                 name: options.name,
                 data: options.data,
@@ -97,6 +111,8 @@ class Scheduler {
                 retry: options.retry,
                 dedupeKey: options.dedupeKey,
                 priority: options.priority,
+                concurrency: options.concurrency,
+                lockVersion: 0,
             };
             if (isNaN(job.nextRunAt.getTime())) {
                 throw new Error("Invalid Date provided for runAt");

package/dist/store/in-memory-job-store.d.ts

@@ -25,5 +25,6 @@ export declare class InMemoryJobStore implements JobStore {
     findById(jobId: unknown): Promise<Job | null>;
     renewLock(jobId: unknown, workerId: string): Promise<void>;
     update(jobId: unknown, updates: JobUpdates): Promise<void>;
+    countRunning(jobName: string): Promise<number>;
     findAll(query: JobQuery): Promise<Job[]>;
 }

package/dist/store/in-memory-job-store.js

@@ -24,6 +24,7 @@ class InMemoryJobStore {
             ...job,
             _id: id,
             priority: job.priority ?? 5,
+            lockVersion: job.lockVersion ?? 0,
             createdAt: job.createdAt ?? new Date(),
             updatedAt: job.updatedAt ?? new Date(),
         };
@@ -54,9 +55,19 @@ class InMemoryJobStore {
                     now.getTime() - job.lockedAt.getTime() < lockTimeoutMs) {
                     continue;
                 }
+                // Check concurrency limit if defined
+                if (job.concurrency !== undefined && job.concurrency > 0) {
+                    const runningCount = Array.from(this.jobs.values()).filter((j) => j.name === job.name && j.status === "running").length;
+                    if (runningCount >= job.concurrency) {
+                        // At concurrency limit, skip this job
+                        continue;
+                    }
+                }
                 job.status = "running";
                 job.lockedAt = now;
                 job.lockedBy = workerId;
+                job.lockUntil = new Date(now.getTime() + lockTimeoutMs);
+                job.lockVersion = (job.lockVersion ?? 0) + 1;
                 job.updatedAt = new Date();
                 job.lastRunAt = now;
                 return { ...job };
@@ -109,6 +120,7 @@ class InMemoryJobStore {
                 job.status = "pending";
                 job.lockedAt = undefined;
                 job.lockedBy = undefined;
+                job.lockUntil = undefined;
                 job.updatedAt = new Date();
                 recovered++;
             }
@@ -165,8 +177,14 @@ class InMemoryJobStore {
         if (updates.priority !== undefined) {
             job.priority = updates.priority;
         }
+        if (updates.concurrency !== undefined) {
+            job.concurrency = updates.concurrency;
+        }
         job.updatedAt = new Date();
     }
+    async countRunning(jobName) {
+        return Array.from(this.jobs.values()).filter((j) => j.name === jobName && j.status === "running").length;
+    }
     async findAll(query) {
         let jobs = Array.from(this.jobs.values());
         // Filter

package/dist/store/job-store.d.ts

@@ -60,6 +60,10 @@ export interface JobStore {
      * Find all jobs matching query
      */
     findAll(query: JobQuery): Promise<Job[]>;
+    /**
+     * Count running jobs by name (for concurrency limits)
+     */
+    countRunning(jobName: string): Promise<number>;
 }
 import { RetryOptions } from "../types/retry";
 import { RepeatOptions } from "../types/repeat";
@@ -72,4 +76,5 @@ export interface JobUpdates {
     status?: JobStatus;
     attempts?: number;
     priority?: number;
+    concurrency?: number;
 }

package/dist/store/mongo/mongo-job-store.d.ts

@@ -10,9 +10,6 @@ export declare class MongoJobStore implements JobStore {
     private readonly collection;
     private readonly defaultLockTimeoutMs;
     constructor(db: Db, options?: MongoJobStoreOptions);
-    /**
-     * Create necessary indexes for optimal query performance
-     */
     private ensureIndexes;
     create(job: Job): Promise<Job>;
     createBulk(jobs: Job[]): Promise<Job[]>;
@@ -35,5 +32,6 @@ export declare class MongoJobStore implements JobStore {
     }): Promise<number>;
     renewLock(id: ObjectId, workerId: string): Promise<void>;
     update(id: ObjectId, updates: JobUpdates): Promise<void>;
+    countRunning(jobName: string): Promise<number>;
     findAll(query: JobQuery): Promise<Job[]>;
 }

package/dist/store/mongo/mongo-job-store.js

@@ -10,31 +10,23 @@ class MongoJobStore {
             console.error("Failed to create indexes:", err);
         });
     }
-    /**
-     * Create necessary indexes for optimal query performance
-     */
     async ensureIndexes() {
         await Promise.all([
-            // Primary index for job polling (findAndLockNext) with priority
             this.collection.createIndex({ status: 1, priority: 1, nextRunAt: 1 }, { background: true }),
-            // Index for deduplication
             this.collection.createIndex({ dedupeKey: 1 }, { unique: true, sparse: true, background: true }),
-            // Index for stale lock recovery
-            this.collection.createIndex({ lockedAt: 1 }, { sparse: true, background: true }),
+            this.collection.createIndex({ lockUntil: 1 }, { sparse: true, background: true }),
+            this.collection.createIndex({ name: 1, status: 1 }, { background: true }),
         ]);
     }
-    // --------------------------------------------------
-    // CREATE
-    // --------------------------------------------------
     async create(job) {
         const now = new Date();
-        // IMPORTANT: strip _id completely
         const { _id, ...jobWithoutId } = job;
         const doc = {
             ...jobWithoutId,
             status: job.status ?? "pending",
             attempts: job.attempts ?? 0,
             priority: job.priority ?? 5,
+            lockVersion: job.lockVersion ?? 0,
             createdAt: now,
             updatedAt: now,
         };
@@ -42,7 +34,6 @@ class MongoJobStore {
             delete doc.dedupeKey;
         }
         if (job.dedupeKey) {
-            // Idempotent insert
             const result = await this.collection.findOneAndUpdate({ dedupeKey: job.dedupeKey }, { $setOnInsert: doc }, { upsert: true, returnDocument: "after" });
             return result;
         }
@@ -52,13 +43,13 @@ class MongoJobStore {
     async createBulk(jobs) {
         const now = new Date();
         const docs = jobs.map((job) => {
-            // IMPORTANT: strip _id completely
             const { _id, ...jobWithoutId } = job;
             const doc = {
                 ...jobWithoutId,
                 status: job.status ?? "pending",
                 attempts: job.attempts ?? 0,
                 priority: job.priority ?? 5,
+                lockVersion: job.lockVersion ?? 0,
                 createdAt: now,
                 updatedAt: now,
             };
@@ -75,36 +66,150 @@ class MongoJobStore {
             _id: result.insertedIds[index],
         }));
     }
-    // --------------------------------------------------
-    // ATOMIC FIND & LOCK
-    // --------------------------------------------------
+    // Atomic find & lock with version-based optimistic locking
     async findAndLockNext(options) {
         const { now, workerId, lockTimeoutMs } = options;
-        const lockExpiry = new Date(now.getTime() - lockTimeoutMs);
-        const result = await this.collection.findOneAndUpdate({
-            status: "pending",
-            nextRunAt: { $lte: now },
+        const lockUntil = new Date(now.getTime() + lockTimeoutMs);
+        // Fast path: jobs without concurrency limits
+        const simpleQuery = {
             $or: [
-                { lockedAt: { $exists: false } },
-                { lockedAt: { $lte: lockExpiry } },
+                // Pending jobs (not locked)
+                {
+                    status: "pending",
+                    nextRunAt: { $lte: now },
+                    $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                },
+                // Stale running jobs (lock expired - crash recovery)
+                {
+                    status: "running",
+                    nextRunAt: { $lte: now },
+                    lockUntil: { $lte: now },
+                },
             ],
-        }, {
+            $and: [
+                { $or: [{ concurrency: { $exists: false } }, { concurrency: null }] },
+            ],
+        };
+        const simpleResult = await this.collection.findOneAndUpdate(simpleQuery, {
             $set: {
                 lockedAt: now,
                 lockedBy: workerId,
+                lockUntil: lockUntil,
                 status: "running",
                 lastRunAt: now,
                 updatedAt: now,
             },
+            $inc: { lockVersion: 1 },
         }, {
             sort: { priority: 1, nextRunAt: 1 },
             returnDocument: "after",
         });
-        return result;
+        if (simpleResult) {
+            return simpleResult;
+        }
+        // Now handle jobs with concurrency limits
+        // We need to check concurrency before locking
+        const maxAttempts = 20;
+        const checkedNames = new Set();
+        for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            // Find a candidate with concurrency limit that we haven't checked yet
+            const concurrencyQuery = {
+                $or: [
+                    {
+                        status: "pending",
+                        nextRunAt: { $lte: now },
+                        // Pending jobs should not have lockedBy set
+                        $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                    },
+                    {
+                        // Stale running jobs (lock expired)
+                        status: "running",
+                        nextRunAt: { $lte: now },
+                        lockUntil: { $lte: now },
+                    },
+                ],
+                concurrency: { $exists: true, $gt: 0 },
+            };
+            if (checkedNames.size > 0) {
+                concurrencyQuery.name = { $nin: Array.from(checkedNames) };
+            }
+            const candidate = await this.collection.findOne(concurrencyQuery, {
+                sort: { priority: 1, nextRunAt: 1 },
+                projection: { name: 1, concurrency: 1, lockVersion: 1 },
+            });
+            if (!candidate) {
+                return null; // No more candidates with concurrency limits
+            }
+            const runningCount = await this.collection.countDocuments({
+                name: candidate.name,
+                status: "running",
+            });
+            if (runningCount >= candidate.concurrency) {
+                // At limit for this job name, skip all jobs with this name
+                checkedNames.add(candidate.name);
+                continue;
+            }
+            const lockResult = await this.collection.findOneAndUpdate({
+                name: candidate.name,
+                concurrency: candidate.concurrency,
+                $or: [
+                    {
+                        status: "pending",
+                        $or: [{ lockedBy: { $exists: false } }, { lockedBy: null }],
+                    },
+                    {
+                        status: "running",
+                        lockUntil: { $lte: now },
+                    },
+                ],
+                nextRunAt: { $lte: now },
+            }, {
+                $set: {
+                    lockedAt: now,
+                    lockedBy: workerId,
+                    lockUntil: lockUntil,
+                    status: "running",
+                    lastRunAt: now,
+                    updatedAt: now,
+                },
+                $inc: { lockVersion: 1 },
+            }, {
+                sort: { priority: 1, nextRunAt: 1 },
+                returnDocument: "after",
+            });
+            if (lockResult) {
+                // Verify concurrency wasn't exceeded by race condition
+                const currentRunning = await this.collection.countDocuments({
+                    name: lockResult.name,
+                    status: "running",
+                });
+                if (currentRunning > lockResult.concurrency) {
+                    // We exceeded concurrency - release this job back to pending
+                    await this.collection.updateOne({
+                        _id: lockResult._id,
+                        lockedBy: workerId,
+                        lockVersion: lockResult.lockVersion,
+                    }, {
+                        $set: {
+                            status: "pending",
+                            updatedAt: new Date(),
+                        },
+                        $unset: {
+                            lockedAt: "",
+                            lockedBy: "",
+                            lockUntil: "",
+                            lastRunAt: "",
+                        },
+                    });
+                    continue;
+                }
+                return lockResult;
+            }
+            // Lock failed (another worker got it), try next job name
+            checkedNames.add(candidate.name);
+        }
+        return null;
     }
-    // --------------------------------------------------
-    // MARK COMPLETED
-    // --------------------------------------------------
     async markCompleted(id) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -114,12 +219,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // MARK FAILED
-    // --------------------------------------------------
     async markFailed(id, error) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -130,12 +233,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // RESCHEDULE
-    // --------------------------------------------------
     async reschedule(id, nextRunAt, updates) {
         const result = await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -147,12 +248,10 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
-    // --------------------------------------------------
-    // CANCEL
-    // --------------------------------------------------
     async cancel(id) {
         await this.collection.updateOne({ _id: id }, {
             $set: {
@@ -162,6 +261,7 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
     }
@@ -171,14 +271,14 @@ class MongoJobStore {
             return null;
         return doc;
     }
-    // --------------------------------------------------
-    // RECOVER STALE JOBS
-    // --------------------------------------------------
     async recoverStaleJobs(options) {
         const { now, lockTimeoutMs } = options;
         const expiry = new Date(now.getTime() - lockTimeoutMs);
         const result = await this.collection.updateMany({
-            lockedAt: { $lte: expiry },
+            $or: [
+                { lockUntil: { $lte: now } },
+                { lockUntil: { $exists: false }, lockedAt: { $lte: expiry } },
+            ],
         }, {
             $set: {
                 status: "pending",
@@ -187,6 +287,7 @@ class MongoJobStore {
             $unset: {
                 lockedAt: "",
                 lockedBy: "",
+                lockUntil: "",
             },
         });
         return result.modifiedCount;
@@ -201,7 +302,9 @@ class MongoJobStore {
             $set: {
                 lockedAt: now,
                 updatedAt: now,
+                lockUntil: new Date(now.getTime() + this.defaultLockTimeoutMs),
             },
+            $inc: { lockVersion: 1 },
         });
         if (result.matchedCount === 0) {
             throw new Error("Job lock lost or owner changed");
@@ -225,8 +328,16 @@ class MongoJobStore {
             $set.attempts = updates.attempts;
         if (updates.priority !== undefined)
             $set.priority = updates.priority;
+        if (updates.concurrency !== undefined)
+            $set.concurrency = updates.concurrency;
         await this.collection.updateOne({ _id: id }, { $set });
     }
+    async countRunning(jobName) {
+        return this.collection.countDocuments({
+            name: jobName,
+            status: "running",
+        });
+    }
     async findAll(query) {
         const filter = {};
         if (query.name) {

package/dist/types/job.d.ts

@@ -11,6 +11,15 @@ export interface Job<Data = unknown> {
     lastScheduledAt?: Date;
     lockedAt?: Date;
     lockedBy?: string;
+    /**
+     * Lock expiry time. Job can be taken by another worker after this time.
+     */
+    lockUntil?: Date;
+    /**
+     * Optimistic locking version. Incremented on each lock acquisition.
+     * Prevents race conditions in distributed environments.
+     */
+    lockVersion?: number;
     attempts: number;
     lastError?: string;
     retry?: RetryOptions | number;
@@ -21,6 +30,11 @@ export interface Job<Data = unknown> {
      * Default: 5
      */
     priority?: number;
+    /**
+     * Max concurrent running jobs with this name.
+     * undefined = no limit.
+     */
+    concurrency?: number;
     createdAt: Date;
     updatedAt: Date;
 }

package/dist/types/schedule.d.ts

@@ -25,4 +25,9 @@ export interface ScheduleOptions<T = unknown> {
      * Default: 5
      */
     priority?: number;
+    /**
+     * Max concurrent running jobs with this name.
+     * Useful for rate-limiting external API calls.
+     */
+    concurrency?: number;
 }

package/dist/worker/worker.js

@@ -44,7 +44,6 @@ class Worker {
     }
     async loop() {
         while (this.running) {
-            // stop requested before poll
             if (!this.running)
                 break;
             const job = await this.store.findAndLockNext({
@@ -52,7 +51,6 @@ class Worker {
                 workerId: this.workerId,
                 lockTimeoutMs: this.lockTimeout,
             });
-            // stop requested after polling
             if (!this.running)
                 break;
             if (!job) {
@@ -65,24 +63,7 @@ class Worker {
     async execute(job) {
         this.emitter.emitSafe("job:start", job);
         const now = Date.now();
-        // ---------------------------
-        // CRON: pre-schedule BEFORE execution
-        // ---------------------------
-        if (job.repeat?.cron) {
-            let base = job.lastScheduledAt ?? job.nextRunAt ?? new Date(now);
-            let next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
-            // skip missed cron slots
-            while (next.getTime() <= now) {
-                base = next;
-                next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
-            }
-            // persist schedule immediately
-            job.lastScheduledAt = next;
-            await this.store.reschedule(job._id, next);
-        }
-        // ---------------------------
-        // HEARTBEAT
-        // ---------------------------
+        // Heartbeat to prevent lock expiry during long jobs
         const heartbeatIntervalMs = Math.max(50, this.lockTimeout / 2);
         const heartbeatParams = {
             jobId: job._id,
@@ -105,16 +86,42 @@ class Worker {
         };
         const heartbeatPromise = heartbeatLoop();
         try {
+            // Verify we still own the lock before any modifications
+            // (another worker might have stolen it via stale recovery)
             const current = await this.store.findById(job._id);
-            if (current && current.status === "cancelled") {
+            if (!current) {
+                stopHeartbeat = true;
+                return;
+            }
+            if (current.status === "cancelled") {
                 this.emitter.emitSafe("job:complete", job);
-                stopHeartbeat = true; // stop fast
+                stopHeartbeat = true;
+                return;
+            }
+            if (current.lockedBy !== this.workerId) {
+                this.emitter.emitSafe("worker:error", new Error(`Lock stolen for job ${job._id}: owned by ${current.lockedBy}, we are ${this.workerId}`));
+                stopHeartbeat = true;
                 return;
             }
+            if (current.status !== "running") {
+                this.emitter.emitSafe("worker:error", new Error(`Job ${job._id} is no longer running (status: ${current.status})`));
+                stopHeartbeat = true;
+                return;
+            }
+            // CRON: pre-schedule before execution (after lock verification)
+            if (job.repeat?.cron) {
+                let base = job.lastScheduledAt ?? job.nextRunAt ?? new Date(now);
+                let next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
+                // skip missed cron slots
+                while (next.getTime() <= now) {
+                    base = next;
+                    next = (0, repeat_1.getNextRunAt)(job.repeat, base, this.defaultTimezone);
+                }
+                job.lastScheduledAt = next;
+                await this.store.reschedule(job._id, next);
+            }
             await this.handler(job);
-            // ---------------------------
-            // INTERVAL: schedule AFTER execution
-            // ---------------------------
+            // INTERVAL: schedule after execution
             if (job.repeat?.every != null) {
                 const next = new Date(Date.now() + Math.max(job.repeat.every, 100));
                 await this.store.reschedule(job._id, next);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mongo-job-scheduler",
-  "version": "0.1.15",
+  "version": "0.1.17",
   "description": "Production-grade MongoDB-backed job scheduler with retries, cron, timezone support, and crash recovery",
   "license": "MIT",
   "author": "Darshan Bhut",
@@ -43,7 +43,6 @@
     "build": "tsc -p tsconfig.build.json",
     "test": "jest",
     "test:mongo": "jest tests/mongo",
-    "test:stress": "jest tests/stress",
     "prepublishOnly": "npm run build && npm test"
   },
   "dependencies": {