agenda 5.0.0 → 6.0.0

package/dist/JobProcessingQueue.js

@@ -0,0 +1,101 @@
+/**
+ * @class
+ */
+export class JobProcessingQueue {
+    agenda;
+    _queue;
+    constructor(agenda) {
+        this.agenda = agenda;
+        this._queue = [];
+    }
+    get length() {
+        return this._queue.length;
+    }
+    getQueue() {
+        return this._queue;
+    }
+    /**
+     * Pops and returns last queue element (next job to be processed) without checking concurrency.
+     * @returns {Job} Next Job to be processed
+     */
+    pop() {
+        return this._queue.pop();
+    }
+    /**
+     * Inserts job in first queue position
+     * @param {Job} job job to add to queue
+     * @returns {undefined}
+     */
+    /*
+    push(job: Job): void {
+        this._queue.push(job);
+    } */
+    remove(job) {
+        let removeJobIndex = this._queue.indexOf(job);
+        if (removeJobIndex === -1) {
+            // lookup by id
+            removeJobIndex = this._queue.findIndex(j => j.attrs._id?.toString() === job.attrs._id?.toString());
+        }
+        if (removeJobIndex === -1) {
+            throw new Error(`cannot find job ${job.attrs._id} in processing queue?`);
+        }
+        this._queue.splice(removeJobIndex, 1);
+    }
+    /**
+     * Inserts job in queue where it will be order from left to right in decreasing
+     * order of nextRunAt and priority (in case of same nextRunAt), if all values
+     * are even the first jobs to be introduced will have priority
+     * @param {Job} job job to add to queue
+     * @returns {undefined}
+     */
+    insert(job) {
+        const matchIndex = this._queue.findIndex(element => {
+            if (element.attrs.nextRunAt &&
+                job.attrs.nextRunAt &&
+                element.attrs.nextRunAt.getTime() <= job.attrs.nextRunAt.getTime()) {
+                if (element.attrs.nextRunAt.getTime() === job.attrs.nextRunAt.getTime()) {
+                    if (element.attrs.priority >= job.attrs.priority) {
+                        return true;
+                    }
+                }
+                else {
+                    return true;
+                }
+            }
+            return false;
+        });
+        if (matchIndex === -1) {
+            // put on left side of the queue
+            this._queue.unshift(job);
+        }
+        else {
+            this._queue.splice(matchIndex, 0, job);
+        }
+    }
+    /**
+     * Returns (does not pop, element remains in queue) first element (always from the right)
+     * that can be processed (not blocked by concurrency execution)
+     * @param {Object} jobStatus current status of jobs
+     * @returns {Job} Next Job to be processed
+     */
+    returnNextConcurrencyFreeJob(jobStatus, handledJobs) {
+        const next = Object.keys(this._queue).reverse().find(i => {
+            const def = this.agenda.definitions[this._queue[i].attrs.name];
+            const status = jobStatus[this._queue[i].attrs.name];
+            // check if we have a definition
+            // if there is no status available, we are good to go
+            // if there is no max concurrency defined (0), we are also good to go
+            // and if concurrency limit is not reached yet (actual running jobs is lower than max concurrency)
+            if (def &&
+                !handledJobs.includes(this._queue[i].attrs._id) &&
+                (!status || !def.concurrency || status.running < def.concurrency)) {
+                return true;
+            }
+            return false;
+        });
+        return next !== undefined
+            ? this._queue[next]
+            : undefined;
+    }
+}
+//# sourceMappingURL=JobProcessingQueue.js.map

package/dist/JobProcessingQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JobProcessingQueue.js","sourceRoot":"","sources":["../src/JobProcessingQueue.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,MAAM,OAAO,kBAAkB;IAGV;IAFZ,MAAM,CAAQ;IAEtB,YAAoB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QACjC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,QAAQ;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,GAAG;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH;;;QAGI;IAEJ,MAAM,CAAC,GAAQ;QACd,IAAI,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC3B,eAAe;YACf,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CACrC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,CAC1D,CAAC;QACH,CAAC;QACD,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,CAAC,KAAK,CAAC,GAAG,uBAAuB,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,GAAQ;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;YAClD,IACC,OAAO,CAAC,KAAK,CAAC,SAAS;gBACvB,GAAG,CAAC,KAAK,CAAC,SAAS;gBACnB,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,EACjE,CAAC;gBACF,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;oBACzE,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;wBAClD,OAAO,IAAI,CAAC;oBACb,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,OAAO,IAAI,CAAC;gBACb,CAAC;YACF,CAAC;YAED,OAAO,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,gCAAgC;YAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACH,4BAA4B,CAC3B,SAMC,EACD,WAAmC;QAEnC,MAAM,IAAI,GAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAyB,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;YACjF,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpD,gCAAgC;YAChC,qDAAqD;YACrD,qEAAqE;YACrE,kGAAkG;YAClG,IACC,GAAG;gBACH,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC,EAChE,CAAC;gBACF,OAAO,IAAI,CAAC;YACb,CAAC;YACD,OAAO,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,KAAK,SAAS;YACxB,CAAC,CAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAgE;YACnF,CAAC,CAAC,SAAS,CAAC;IACd,CAAC;CACD"}

package/dist/JobProcessor.d.ts

@@ -0,0 +1,70 @@
+import type { AgendaStatus } from './types/AgendaStatus.js';
+import type { Agenda, JobWithId } from './index.js';
+import type { NotificationChannel } from './types/NotificationChannel.js';
+/**
+ * @class
+ * Process methods for jobs
+ */
+export declare class JobProcessor {
+    private agenda;
+    private maxConcurrency;
+    private totalLockLimit;
+    private processEvery;
+    private jobStatus;
+    private localQueueProcessing;
+    getStatus(fullDetails?: boolean): Promise<AgendaStatus>;
+    private nextScanAt;
+    private jobQueue;
+    private runningJobs;
+    private lockedJobs;
+    private isJobQueueFilling;
+    private isRunning;
+    private processInterval?;
+    private notificationChannel?;
+    private notificationUnsubscribe?;
+    constructor(agenda: Agenda, maxConcurrency: number, totalLockLimit: number, processEvery: number, notificationChannel?: NotificationChannel);
+    stop(): JobWithId[];
+    /**
+     * Waits for all currently running jobs to finish, then stops the processor.
+     * Unlike stop(), this method does not immediately return - it waits for graceful completion.
+     * @returns Promise that resolves when all running jobs have completed
+     */
+    drain(): Promise<void>;
+    /**
+     * Handle incoming job notification - triggers immediate processing
+     */
+    private handleNotification;
+    process(): Promise<void>;
+    /**
+     * Returns true if a job of the specified name can be locked.
+     * Considers maximum locked jobs at any time if self._lockLimit is > 0
+     * Considers maximum locked jobs of the specified name at any time if jobDefinition.lockLimit is > 0
+     * @param {String} name name of job to check if we should lock or not
+     * @returns {boolean} whether or not you should lock job
+     */
+    shouldLock(name: string): boolean;
+    /**
+     * Internal method that adds a job to be processed to the local queue
+     * @param job - Job to enqueue
+     */
+    private enqueueJob;
+    private findAndLockNextJob;
+    /**
+     * Internal method used to fill a queue with jobs that can be run
+     * @param {String} name fill a queue with specific job name
+     * @returns {undefined}
+     */
+    private jobQueueFilling;
+    /**
+     * Internal method that processes any jobs in the local queue (array)
+     * handledJobs keeps list of already processed jobs
+     * @returns {undefined}
+     */
+    private jobProcessing;
+    /**
+     * Internal method that tries to run a job and if it fails, retries again!
+     * @returns {boolean} processed a job or not
+     */
+    private runOrRetry;
+    private updateStatus;
+}

package/dist/JobProcessor.js

@@ -0,0 +1,458 @@
+import debug from 'debug';
+import packageJson from '../package.json' with { type: 'json' };
+import { Job } from './Job.js';
+import { JobProcessingQueue } from './JobProcessingQueue.js';
+const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+const log = debug('agenda:jobProcessor');
+const agendaVersion = packageJson.version;
+const MAX_SAFE_32BIT_INTEGER = 2 ** 31; // Math.pow(2,31);
+/**
+ * @class
+ * Process methods for jobs
+ */
+export class JobProcessor {
+    agenda;
+    maxConcurrency;
+    totalLockLimit;
+    processEvery;
+    jobStatus = {};
+    localQueueProcessing = 0;
+    async getStatus(fullDetails = false) {
+        const jobStatus = Object.keys(this.jobStatus).reduce((obj, key) => {
+            const status = this.jobStatus[key];
+            obj[key] = {
+                running: status?.running ?? 0,
+                locked: status?.locked ?? 0,
+                config: this.agenda.definitions[key]
+            };
+            return obj;
+        }, {});
+        return {
+            version: agendaVersion,
+            queueName: this.agenda.attrs.name,
+            totalQueueSizeDB: await this.agenda.db.getQueueSize(),
+            config: {
+                totalLockLimit: this.totalLockLimit,
+                maxConcurrency: this.maxConcurrency,
+                processEvery: this.processEvery
+            },
+            backend: {
+                // Placeholder - will be overwritten by Agenda.getRunningStats()
+                name: 'unknown',
+                hasNotificationChannel: false
+            },
+            internal: {
+                localQueueProcessing: this.localQueueProcessing
+            },
+            jobStatus,
+            queuedJobs: !fullDetails
+                ? this.jobQueue.length
+                : this.jobQueue.getQueue().map(job => ({
+                    ...job.toJson(),
+                    canceled: job.getCanceledMessage()
+                })),
+            runningJobs: !fullDetails
+                ? this.runningJobs.length
+                : this.runningJobs.map(job => ({
+                    ...job.toJson(),
+                    canceled: job.getCanceledMessage()
+                })),
+            lockedJobs: !fullDetails
+                ? this.lockedJobs.length
+                : this.lockedJobs.map(job => ({
+                    ...job.toJson(),
+                    canceled: job.getCanceledMessage()
+                }))
+        };
+    }
+    nextScanAt = new Date();
+    jobQueue;
+    runningJobs = [];
+    lockedJobs = [];
+    isJobQueueFilling = new Map();
+    isRunning = true;
+    processInterval;
+    notificationChannel;
+    notificationUnsubscribe;
+    constructor(agenda, maxConcurrency, totalLockLimit, processEvery, notificationChannel) {
+        this.agenda = agenda;
+        this.maxConcurrency = maxConcurrency;
+        this.totalLockLimit = totalLockLimit;
+        this.processEvery = processEvery;
+        this.notificationChannel = notificationChannel;
+        this.jobQueue = new JobProcessingQueue(this.agenda);
+        log('creating interval to call processJobs every [%dms]', processEvery);
+        this.processInterval = setInterval(() => this.process(), processEvery);
+        this.process();
+        // Subscribe to job notifications for real-time processing
+        if (this.notificationChannel) {
+            log('subscribing to notification channel');
+            this.notificationUnsubscribe = this.notificationChannel.subscribe(this.handleNotification.bind(this));
+        }
+    }
+    stop() {
+        log.extend('stop')('stop job processor', this.isRunning);
+        this.isRunning = false;
+        if (this.processInterval) {
+            clearInterval(this.processInterval);
+            this.processInterval = undefined;
+        }
+        // Unsubscribe from notifications
+        if (this.notificationUnsubscribe) {
+            log.extend('stop')('unsubscribing from notification channel');
+            this.notificationUnsubscribe();
+            this.notificationUnsubscribe = undefined;
+        }
+        // Return both locked and running jobs so they can all be unlocked
+        // Running jobs are also "locked" in the database (they have lockedAt set),
+        // they just moved from lockedJobs to runningJobs when processing started
+        return [...this.lockedJobs, ...this.runningJobs];
+    }
+    /**
+     * Waits for all currently running jobs to finish, then stops the processor.
+     * Unlike stop(), this method does not immediately return - it waits for graceful completion.
+     * @returns Promise that resolves when all running jobs have completed
+     */
+    async drain() {
+        log.extend('drain')('drain called, clearing interval for processJobs()');
+        // Stop accepting new jobs
+        if (this.processInterval) {
+            clearInterval(this.processInterval);
+            this.processInterval = undefined;
+        }
+        // Unsubscribe from notifications to stop receiving new job notifications
+        if (this.notificationUnsubscribe) {
+            log.extend('drain')('unsubscribing from notification channel');
+            this.notificationUnsubscribe();
+            this.notificationUnsubscribe = undefined;
+        }
+        // If no jobs are running, resolve immediately
+        if (this.runningJobs.length === 0) {
+            log.extend('drain')('no running jobs, resolving immediately');
+            this.isRunning = false;
+            return;
+        }
+        log.extend('drain')('waiting for %d running jobs to finish', this.runningJobs.length);
+        // Wait for all running jobs to complete
+        return new Promise(resolve => {
+            const checkComplete = () => {
+                if (this.runningJobs.length === 0) {
+                    log.extend('drain')('all jobs completed, resolving');
+                    this.isRunning = false;
+                    resolve();
+                }
+            };
+            // Listen for 'complete' events from the agenda
+            const completeListener = () => {
+                // Running jobs are removed after the event is emitted,
+                // so we check on next tick
+                setImmediate(checkComplete);
+            };
+            this.agenda.on('complete', completeListener);
+            // Also check immediately in case jobs finished between our check and listener setup
+            checkComplete();
+        });
+    }
+    /**
+     * Handle incoming job notification - triggers immediate processing
+     */
+    async handleNotification(notification) {
+        if (!this.isRunning) {
+            log.extend('handleNotification')('JobProcessor not running, ignoring notification');
+            return;
+        }
+        // Only process notifications for jobs we have definitions for
+        if (!this.agenda.definitions[notification.jobName]) {
+            log.extend('handleNotification')('no definition for job [%s], ignoring notification', notification.jobName);
+            return;
+        }
+        // Check if the job should run soon (within the next scan interval)
+        if (notification.nextRunAt && notification.nextRunAt > this.nextScanAt) {
+            log.extend('handleNotification')('job [%s:%s] nextRunAt is after nextScanAt, will be picked up by regular scan', notification.jobName, notification.jobId);
+            return;
+        }
+        log.extend('handleNotification')('received notification for job [%s:%s], triggering immediate queue fill', notification.jobName, notification.jobId);
+        // Fill the queue for this job type and process
+        await this.jobQueueFilling(notification.jobName);
+        this.jobProcessing();
+    }
+    // processJobs
+    async process() {
+        // Make sure an interval has actually been set
+        // Prevents race condition with 'Agenda.stop' and already scheduled run
+        if (!this.isRunning) {
+            log.extend('process')('JobProcessor got stopped already, returning');
+            return;
+        }
+        log.extend('process')('starting to process jobs');
+        // Go through each jobName set in 'Agenda.process' and fill the queue with the next jobs
+        await Promise.all(Object.keys(this.agenda.definitions).map(async (jobName) => {
+            log.extend('process')('queuing up job to process: [%s]', jobName);
+            await this.jobQueueFilling(jobName);
+        }));
+        this.jobProcessing();
+    }
+    /**
+     * Returns true if a job of the specified name can be locked.
+     * Considers maximum locked jobs at any time if self._lockLimit is > 0
+     * Considers maximum locked jobs of the specified name at any time if jobDefinition.lockLimit is > 0
+     * @param {String} name name of job to check if we should lock or not
+     * @returns {boolean} whether or not you should lock job
+     */
+    shouldLock(name) {
+        const jobDefinition = this.agenda.definitions[name];
+        let shouldLock = true;
+        // global lock limit
+        if (this.totalLockLimit && this.lockedJobs.length >= this.totalLockLimit) {
+            shouldLock = false;
+        }
+        // job specific lock limit
+        const status = this.jobStatus[name];
+        if (jobDefinition.lockLimit && status && status.locked >= jobDefinition.lockLimit) {
+            shouldLock = false;
+        }
+        log.extend('shouldLock')('job [%s] lock status: shouldLock = %s', name, shouldLock, `${status?.locked} >= ${jobDefinition?.lockLimit}`, `${this.lockedJobs.length} >= ${this.totalLockLimit}`);
+        return shouldLock;
+    }
+    /**
+     * Internal method that adds a job to be processed to the local queue
+     * @param job - Job to enqueue
+     */
+    enqueueJob(job) {
+        this.jobQueue.insert(job);
+    }
+    async findAndLockNextJob(jobName, definition) {
+        const lockDeadline = new Date(Date.now().valueOf() - definition.lockLifetime);
+        log.extend('findAndLockNextJob')(`looking for lockable jobs for ${jobName} (lock dead line = ${lockDeadline})`);
+        // Find ONE and ONLY ONE job and set the 'lockedAt' time so that job begins to be processed
+        const result = await this.agenda.db.getNextJobToRun(jobName, this.nextScanAt, lockDeadline, undefined, { lastModifiedBy: this.agenda.attrs.name || undefined });
+        if (result) {
+            log.extend('findAndLockNextJob')('found a job available to lock, creating a new job on Agenda with id [%s]', result._id);
+            return new Job(this.agenda, result, true);
+        }
+        return undefined;
+    }
+    /**
+     * Internal method used to fill a queue with jobs that can be run
+     * @param {String} name fill a queue with specific job name
+     * @returns {undefined}
+     */
+    async jobQueueFilling(name) {
+        this.isJobQueueFilling.set(name, true);
+        try {
+            // Don't lock because of a limit we have set (lockLimit, etc)
+            if (!this.shouldLock(name)) {
+                log.extend('jobQueueFilling')('lock limit reached in queue filling for [%s]', name);
+                return;
+            }
+            // Set the date of the next time we are going to run _processEvery function
+            const now = new Date();
+            this.nextScanAt = new Date(now.valueOf() + this.processEvery);
+            // For this job name, find the next job to run and lock it!
+            const job = await this.findAndLockNextJob(name, this.agenda.definitions[name]);
+            // Still have the job?
+            // 1. Add it to lock list
+            // 2. Add count of locked jobs
+            // 3. Queue the job to actually be run now that it is locked
+            // 4. Recursively run this same method we are in to check for more available jobs of same type!
+            if (job) {
+                if (job.attrs.name !== name) {
+                    throw new Error(`got different job name: ${job.attrs.name} (actual) !== ${name} (expected)`);
+                }
+                // Before en-queing job make sure we haven't exceed our lock limits
+                if (!this.shouldLock(name)) {
+                    log.extend('jobQueueFilling')('lock limit reached before job was returned. Releasing lock on [%s]', name);
+                    this.agenda.db.unlockJob(job.attrs);
+                    return;
+                }
+                log.extend('jobQueueFilling')('[%s:%s] job locked while filling queue', name, job.attrs._id);
+                this.updateStatus(name, 'locked', +1);
+                this.lockedJobs.push(job);
+                this.enqueueJob(job);
+                await this.jobQueueFilling(name);
+            }
+            else {
+                log.extend('jobQueueFilling')('Cannot lock job [%s]', name);
+            }
+        }
+        catch (error) {
+            log.extend('jobQueueFilling')('[%s] job lock failed while filling queue', name, error);
+            this.agenda.emit('error', error);
+        }
+        finally {
+            this.isJobQueueFilling.delete(name);
+        }
+    }
+    /**
+     * Internal method that processes any jobs in the local queue (array)
+     * handledJobs keeps list of already processed jobs
+     * @returns {undefined}
+     */
+    async jobProcessing(handledJobs = []) {
+        // Ensure we have jobs
+        if (this.jobQueue.length === 0) {
+            return;
+        }
+        this.localQueueProcessing += 1;
+        try {
+            const now = new Date();
+            // Check if there is any job that is not blocked by concurrency
+            const job = this.jobQueue.returnNextConcurrencyFreeJob(this.jobStatus, handledJobs);
+            if (!job) {
+                log.extend('jobProcessing')('[%s:%s] there is no job to process');
+                return;
+            }
+            this.jobQueue.remove(job);
+            if (!(await job.isExpired())) {
+                // check if job has expired (and therefore probably got picked up again by another queue in the meantime)
+                // before it even has started to run
+                log.extend('jobProcessing')('[%s:%s] there is a job to process (priority = %d)', job.attrs.name, job.attrs._id, job.attrs.priority, job.gotTimerToExecute);
+                // If the 'nextRunAt' time is older than the current time, run the job
+                // Otherwise, setTimeout that gets called at the time of 'nextRunAt'
+                if (!job.attrs.nextRunAt || job.attrs.nextRunAt <= now) {
+                    log.extend('jobProcessing')('[%s:%s] nextRunAt is in the past, run the job immediately', job.attrs.name, job.attrs._id);
+                    this.runOrRetry(job);
+                }
+                else {
+                    const runIn = job.attrs.nextRunAt.getTime() - now.getTime();
+                    if (runIn > this.processEvery) {
+                        // this job is not in the near future, remove it (it will be picked up later)
+                        log.extend('runOrRetry')('[%s:%s] job is too far away, freeing it up', job.attrs.name, job.attrs._id);
+                        let lockedJobIndex = this.lockedJobs.indexOf(job);
+                        if (lockedJobIndex === -1) {
+                            // lookup by id
+                            lockedJobIndex = this.lockedJobs.findIndex(j => j.attrs._id?.toString() === job.attrs._id?.toString());
+                        }
+                        if (lockedJobIndex === -1) {
+                            throw new Error(`cannot find job ${job.attrs._id} in locked jobs queue?`);
+                        }
+                        this.lockedJobs.splice(lockedJobIndex, 1);
+                        this.updateStatus(job.attrs.name, 'locked', -1);
+                    }
+                    else {
+                        log.extend('jobProcessing')('[%s:%s] nextRunAt is in the future, calling setTimeout(%d)', job.attrs.name, job.attrs._id, runIn);
+                        // re add to queue (puts it at the right position in the queue)
+                        this.jobQueue.insert(job);
+                        // ensure every job gets a timer to run at the near future time (but also ensure this time is set only once)
+                        if (!job.gotTimerToExecute) {
+                            job.gotTimerToExecute = true;
+                            setTimeout(() => {
+                                this.jobProcessing();
+                            }, runIn > MAX_SAFE_32BIT_INTEGER ? MAX_SAFE_32BIT_INTEGER : runIn); // check if runIn is higher than unsined 32 bit int, if so, use this time to recheck,
+                            // because setTimeout will run in an overflow otherwise and reprocesses immediately
+                        }
+                    }
+                }
+            }
+            handledJobs.push(job.attrs._id);
+            if (job && this.localQueueProcessing < this.maxConcurrency) {
+                // additionally run again and check if there are more jobs that we can process right now (as long concurrency not reached)
+                setImmediate(() => this.jobProcessing(handledJobs));
+            }
+        }
+        finally {
+            this.localQueueProcessing -= 1;
+        }
+    }
+    /**
+     * Internal method that tries to run a job and if it fails, retries again!
+     * @returns {boolean} processed a job or not
+     */
+    async runOrRetry(job) {
+        if (!this.isRunning) {
+            // const a = new Error();
+            // console.log('STACK', a.stack);
+            log.extend('runOrRetry')('JobProcessor got stopped already while calling runOrRetry, returning!');
+            return;
+        }
+        const jobDefinition = this.agenda.definitions[job.attrs.name];
+        const status = this.jobStatus[job.attrs.name];
+        if ((!jobDefinition.concurrency || !status || status.running < jobDefinition.concurrency) &&
+            this.runningJobs.length < this.maxConcurrency) {
+            // Add to local "running" queue
+            this.runningJobs.push(job);
+            this.updateStatus(job.attrs.name, 'running', 1);
+            let jobIsRunning = true;
+            try {
+                log.extend('runOrRetry')('[%s:%s] processing job', job.attrs.name, job.attrs._id);
+                // check if the job is still alive
+                const checkIfJobIsStillAlive = async () => {
+                    // check every "this.agenda.definitions[job.attrs.name].lockLifetime / 2"" (or at mininum every processEvery)
+                    const timeout = Math.max(this.processEvery / 2, this.agenda.definitions[job.attrs.name].lockLifetime / 2);
+                    do {
+                        await delay(timeout);
+                        // when job is not running anymore, just finish
+                        if (!jobIsRunning) {
+                            log.extend('runOrRetry')('[%s:%s] checkIfJobIsStillAlive detected job is not running anymore. stopping check.', job.attrs.name, job.attrs._id);
+                            return;
+                        }
+                        if (await job.isExpired()) {
+                            log.extend('runOrRetry')('[%s:%s] checkIfJobIsStillAlive detected an expired job, killing it.', job.attrs.name, job.attrs._id);
+                            throw new Error(`execution of '${job.attrs.name}' canceled, execution took more than ${this.agenda.definitions[job.attrs.name].lockLifetime}ms. Call touch() for long running jobs to keep them alive.`);
+                        }
+                        if (!job.attrs.lockedAt) {
+                            log.extend('runOrRetry')('[%s:%s] checkIfJobIsStillAlive detected a job without a lockedAt value, killing it.', job.attrs.name, job.attrs._id);
+                            throw new Error(`execution of '${job.attrs.name}' canceled, no lockedAt date found. Ensure to call touch() for long running jobs to keep them alive.`);
+                        }
+                    } while (jobIsRunning);
+                };
+                // CALL THE ACTUAL METHOD TO PROCESS THE JOB!!!
+                await Promise.race([job.run(), checkIfJobIsStillAlive()]);
+                log.extend('runOrRetry')('[%s:%s] processing job successfull', job.attrs.name, job.attrs._id);
+                // Job isn't in running jobs so throw an error
+                if (!this.runningJobs.includes(job)) {
+                    log.extend('runOrRetry')('[%s] callback was called, job must have been marked as complete already', job.attrs._id);
+                    throw new Error(`callback already called - job ${job.attrs.name} already marked complete`);
+                }
+            }
+            catch (error) {
+                job.cancel(error instanceof Error ? error : undefined);
+                log.extend('runOrRetry')('[%s:%s] processing job failed', job.attrs.name, job.attrs._id, error);
+                this.agenda.emit('error', error instanceof Error ? error : new Error(String(error)));
+            }
+            finally {
+                jobIsRunning = false;
+                // Remove the job from the running queue
+                let runningJobIndex = this.runningJobs.indexOf(job);
+                if (runningJobIndex === -1) {
+                    // lookup by id
+                    runningJobIndex = this.runningJobs.findIndex(j => j.attrs._id?.toString() === job.attrs._id?.toString());
+                }
+                if (runningJobIndex === -1) {
+                    // eslint-disable-next-line no-unsafe-finally
+                    throw new Error(`cannot find job ${job.attrs._id} in running jobs queue?`);
+                }
+                this.runningJobs.splice(runningJobIndex, 1);
+                this.updateStatus(job.attrs.name, 'running', -1);
+                // Remove the job from the locked queue
+                let lockedJobIndex = this.lockedJobs.indexOf(job);
+                if (lockedJobIndex === -1) {
+                    // lookup by id
+                    lockedJobIndex = this.lockedJobs.findIndex(j => j.attrs._id?.toString() === job.attrs._id?.toString());
+                }
+                if (lockedJobIndex === -1) {
+                    // eslint-disable-next-line no-unsafe-finally
+                    throw new Error(`cannot find job ${job.attrs._id} in locked jobs queue?`);
+                }
+                this.lockedJobs.splice(lockedJobIndex, 1);
+                this.updateStatus(job.attrs.name, 'locked', -1);
+            }
+            // Re-process jobs now that one has finished
+            setImmediate(() => this.jobProcessing());
+            return;
+        }
+        // Run the job later
+        log.extend('runOrRetry')('[%s:%s] concurrency preventing immediate run, pushing job to top of queue', job.attrs.name, job.attrs._id);
+        this.enqueueJob(job);
+    }
+    updateStatus(name, key, number) {
+        if (!this.jobStatus[name]) {
+            this.jobStatus[name] = {
+                locked: 0,
+                running: 0
+            };
+        }
+        this.jobStatus[name][key] += number;
+    }
+}
+//# sourceMappingURL=JobProcessor.js.map