@tachybase/plugin-adapter-bullmq 1.3.17 → 1.3.18

package/dist/node_modules/bullmq/dist/esm/classes/scripts.js

@@ -49,7 +49,7 @@ export class Scripts {
         }
         return Number.isInteger(result);
     }
-    addDelayedJob(client, job, encodedOpts, args) {
+    addDelayedJobArgs(job, encodedOpts, args) {
         const queueKeys = this.queue.keys;
         const keys = [
             queueKeys.marker,
@@ -60,35 +60,49 @@ export class Scripts {
             queueKeys.events,
         ];
         keys.push(pack(args), job.data, encodedOpts);
-        return this.execCommand(client, 'addDelayedJob', keys);
+        return keys;
     }
-    addPrioritizedJob(client, job, encodedOpts, args) {
+    addDelayedJob(client, job, encodedOpts, args) {
+        const argsList = this.addDelayedJobArgs(job, encodedOpts, args);
+        return this.execCommand(client, 'addDelayedJob', argsList);
+    }
+    addPrioritizedJobArgs(job, encodedOpts, args) {
         const queueKeys = this.queue.keys;
         const keys = [
             queueKeys.marker,
             queueKeys.meta,
             queueKeys.id,
             queueKeys.prioritized,
+            queueKeys.delayed,
             queueKeys.completed,
             queueKeys.active,
             queueKeys.events,
             queueKeys.pc,
         ];
         keys.push(pack(args), job.data, encodedOpts);
-        return this.execCommand(client, 'addPrioritizedJob', keys);
+        return keys;
     }
-    addParentJob(client, job, encodedOpts, args) {
+    addPrioritizedJob(client, job, encodedOpts, args) {
+        const argsList = this.addPrioritizedJobArgs(job, encodedOpts, args);
+        return this.execCommand(client, 'addPrioritizedJob', argsList);
+    }
+    addParentJobArgs(job, encodedOpts, args) {
         const queueKeys = this.queue.keys;
         const keys = [
             queueKeys.meta,
             queueKeys.id,
+            queueKeys.delayed,
             queueKeys.completed,
             queueKeys.events,
         ];
         keys.push(pack(args), job.data, encodedOpts);
-        return this.execCommand(client, 'addParentJob', keys);
+        return keys;
     }
-    addStandardJob(client, job, encodedOpts, args) {
+    addParentJob(client, job, encodedOpts, args) {
+        const argsList = this.addParentJobArgs(job, encodedOpts, args);
+        return this.execCommand(client, 'addParentJob', argsList);
+    }
+    addStandardJobArgs(job, encodedOpts, args) {
         const queueKeys = this.queue.keys;
         const keys = [
             queueKeys.wait,
@@ -96,25 +110,29 @@ export class Scripts {
             queueKeys.meta,
             queueKeys.id,
             queueKeys.completed,
+            queueKeys.delayed,
             queueKeys.active,
             queueKeys.events,
             queueKeys.marker,
         ];
         keys.push(pack(args), job.data, encodedOpts);
-        return this.execCommand(client, 'addStandardJob', keys);
+        return keys;
+    }
+    addStandardJob(client, job, encodedOpts, args) {
+        const argsList = this.addStandardJobArgs(job, encodedOpts, args);
+        return this.execCommand(client, 'addStandardJob', argsList);
     }
-    async addJob(client, job, opts, jobId, parentOpts = {}) {
+    async addJob(client, job, opts, jobId, parentKeyOpts = {}) {
         const queueKeys = this.queue.keys;
-        const parent = job.parent
-            ? Object.assign(Object.assign({}, job.parent), { fpof: opts.fpof, rdof: opts.rdof, idof: opts.idof }) : null;
+        const parent = job.parent;
         const args = [
             queueKeys[''],
             typeof jobId !== 'undefined' ? jobId : '',
             job.name,
             job.timestamp,
             job.parentKey || null,
-            parentOpts.waitChildrenKey || null,
-            parentOpts.parentDependenciesKey || null,
+            parentKeyOpts.waitChildrenKey || null,
+            parentKeyOpts.parentDependenciesKey || null,
             parent,
             job.repeatJobKey,
             job.deduplicationId ? `${queueKeys.de}:${job.deduplicationId}` : null,
@@ -134,10 +152,10 @@ export class Scripts {
             encodedOpts = pack(opts);
         }
         let result;
-        if (parentOpts.waitChildrenKey) {
+        if (parentKeyOpts.waitChildrenKey) {
             result = await this.addParentJob(client, job, encodedOpts, args);
         }
-        else if (typeof opts.delay == 'number') {
+        else if (typeof opts.delay == 'number' && opts.delay > 0) {
             result = await this.addDelayedJob(client, job, encodedOpts, args);
         }
         else if (opts.priority) {
@@ -149,7 +167,7 @@ export class Scripts {
         if (result < 0) {
             throw this.finishedErrors({
                 code: result,
-                parentKey: parentOpts.parentKey,
+                parentKey: parentKeyOpts.parentKey,
                 command: 'addJob',
             });
         }
@@ -191,11 +209,23 @@ export class Scripts {
         const args = this.addRepeatableJobArgs(customKey, nextMillis, opts, legacyCustomKey);
         return this.execCommand(client, 'addRepeatableJob', args);
     }
-    async addJobScheduler(client, jobSchedulerId, nextMillis, templateData, templateOpts, opts) {
+    async addJobScheduler(jobSchedulerId, nextMillis, templateData, templateOpts, opts, delayedJobOpts, 
+    // The job id of the job that produced this next iteration
+    producerId) {
+        const client = await this.queue.client;
         const queueKeys = this.queue.keys;
         const keys = [
             queueKeys.repeat,
             queueKeys.delayed,
+            queueKeys.wait,
+            queueKeys.paused,
+            queueKeys.meta,
+            queueKeys.prioritized,
+            queueKeys.marker,
+            queueKeys.id,
+            queueKeys.events,
+            queueKeys.pc,
+            queueKeys.active,
         ];
         const args = [
             nextMillis,
@@ -203,7 +233,10 @@ export class Scripts {
             jobSchedulerId,
             templateData,
             pack(templateOpts),
+            pack(delayedJobOpts),
+            Date.now(),
             queueKeys[''],
+            producerId ? this.queue.toKey(producerId) : '',
         ];
         return this.execCommand(client, 'addJobScheduler', keys.concat(args));
     }
@@ -216,8 +249,35 @@ export class Scripts {
         ];
         return this.execCommand(client, 'updateRepeatableJobMillis', args);
     }
-    async updateJobSchedulerNextMillis(client, jobSchedulerId, nextMillis) {
-        return client.zadd(this.queue.keys.repeat, nextMillis, jobSchedulerId);
+    async updateJobSchedulerNextMillis(jobSchedulerId, nextMillis, templateData, delayedJobOpts, 
+    // The job id of the job that produced this next iteration - TODO: remove in next breaking change
+    producerId) {
+        const client = await this.queue.client;
+        const queueKeys = this.queue.keys;
+        const keys = [
+            queueKeys.repeat,
+            queueKeys.delayed,
+            queueKeys.wait,
+            queueKeys.paused,
+            queueKeys.meta,
+            queueKeys.prioritized,
+            queueKeys.marker,
+            queueKeys.id,
+            queueKeys.events,
+            queueKeys.pc,
+            producerId ? this.queue.toKey(producerId) : '',
+            queueKeys.active,
+        ];
+        const args = [
+            nextMillis,
+            jobSchedulerId,
+            templateData,
+            pack(delayedJobOpts),
+            Date.now(),
+            queueKeys[''],
+            producerId,
+        ];
+        return this.execCommand(client, 'updateJobScheduler', keys.concat(args));
     }
     removeRepeatableArgs(legacyRepeatJobId, repeatConcatOptions, repeatJobKey) {
         const queueKeys = this.queue.keys;
@@ -250,8 +310,8 @@ export class Scripts {
         return this.execCommand(client, 'removeJobScheduler', keys.concat(args));
     }
     removeArgs(jobId, removeChildren) {
-        const keys = ['', 'meta'].map(name => this.queue.toKey(name));
-        const args = [jobId, removeChildren ? 1 : 0];
+        const keys = [jobId, 'repeat'].map(name => this.queue.toKey(name));
+        const args = [jobId, removeChildren ? 1 : 0, this.queue.toKey('')];
         return keys.concat(args);
     }
     async remove(jobId, removeChildren) {
@@ -267,6 +327,16 @@ export class Scripts {
         }
         return result;
     }
+    async removeUnprocessedChildren(jobId) {
+        const client = await this.queue.client;
+        const args = [
+            this.queue.toKey(jobId),
+            this.queue.keys.meta,
+            this.queue.toKey(''),
+            jobId,
+        ];
+        await this.execCommand(client, 'removeUnprocessedChildren', args);
+    }
     async extendLock(jobId, token, duration, client) {
         client = client || (await this.queue.client);
         const args = [
@@ -336,7 +406,7 @@ export class Scripts {
         return result;
     }
     moveToFinishedArgs(job, val, propVal, shouldRemove, target, token, timestamp, fetchNext = true, fieldsToUpdate) {
-        var _a, _b, _c, _d, _e, _f;
+        var _a, _b, _c, _d, _e, _f, _g;
         const queueKeys = this.queue.keys;
         const opts = this.queue.opts;
         const workerKeepJobs = target === 'completed' ? opts.removeOnComplete : opts.removeOnFail;
@@ -357,6 +427,7 @@ export class Scripts {
             queueKeys[''],
             pack({
                 token,
+                name: opts.name,
                 keepJobs,
                 limiter: opts.limiter,
                 lockDuration: opts.lockDuration,
@@ -365,8 +436,9 @@ export class Scripts {
                     ? (_c = opts.metrics) === null || _c === void 0 ? void 0 : _c.maxDataPoints
                     : '',
                 fpof: !!((_d = job.opts) === null || _d === void 0 ? void 0 : _d.failParentOnFailure),
-                idof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.ignoreDependencyOnFailure),
-                rdof: !!((_f = job.opts) === null || _f === void 0 ? void 0 : _f.removeDependencyOnFailure),
+                cpof: !!((_e = job.opts) === null || _e === void 0 ? void 0 : _e.continueParentOnFailure),
+                idof: !!((_f = job.opts) === null || _f === void 0 ? void 0 : _f.ignoreDependencyOnFailure),
+                rdof: !!((_g = job.opts) === null || _g === void 0 ? void 0 : _g.removeDependencyOnFailure),
             }),
             fieldsToUpdate ? pack(objectToFlatArray(fieldsToUpdate)) : void 0,
         ];
@@ -404,11 +476,11 @@ export class Scripts {
         const keys = [
             queueKeys.wait,
             queueKeys.paused,
-            delayed ? queueKeys.delayed : '',
+            queueKeys.delayed,
             queueKeys.prioritized,
             queueKeys.repeat,
         ];
-        const args = [queueKeys['']];
+        const args = [queueKeys[''], delayed ? '1' : '0'];
         return keys.concat(args);
     }
     async drain(delayed) {
@@ -483,6 +555,23 @@ export class Scripts {
         const args = this.getCountsPerPriorityArgs(priorities);
         return await this.execCommand(client, 'getCountsPerPriority', args);
     }
+    getDependencyCountsArgs(jobId, types) {
+        const keys = [
+            `${jobId}:processed`,
+            `${jobId}:dependencies`,
+            `${jobId}:failed`,
+            `${jobId}:unsuccessful`,
+        ].map(name => {
+            return this.queue.toKey(name);
+        });
+        const args = types;
+        return keys.concat(args);
+    }
+    async getDependencyCounts(jobId, types) {
+        const client = await this.queue.client;
+        const args = this.getDependencyCountsArgs(jobId, types);
+        return await this.execCommand(client, 'getDependencyCounts', args);
+    }
     moveToCompletedArgs(job, returnvalue, removeOnComplete, token, fetchNext = false) {
         const timestamp = Date.now();
         return this.moveToFinishedArgs(job, returnvalue, 'returnvalue', removeOnComplete, 'completed', token, timestamp, fetchNext);
@@ -597,11 +686,14 @@ export class Scripts {
         const timestamp = Date.now();
         const childKey = getParentKey(opts.child);
         const keys = [
-            `${jobId}:lock`,
             'active',
             'waiting-children',
             jobId,
+            `${jobId}:dependencies`,
+            `${jobId}:unsuccessful`,
             'stalled',
+            'failed',
+            'events',
         ].map(name => {
             return this.queue.toKey(name);
         });
@@ -610,6 +702,7 @@ export class Scripts {
             childKey !== null && childKey !== void 0 ? childKey : '',
             JSON.stringify(timestamp),
             jobId,
+            this.queue.toKey(''),
         ]);
     }
     isMaxedArgs() {
@@ -699,7 +792,7 @@ export class Scripts {
         const args = this.getJobSchedulerArgs(id);
         return this.execCommand(client, 'getJobScheduler', args);
     }
-    retryJobArgs(jobId, lifo, token, fieldsToUpdate) {
+    retryJobArgs(jobId, lifo, token, opts = {}) {
         const keys = [
             this.queue.keys.active,
             this.queue.keys.wait,
@@ -720,12 +813,14 @@ export class Scripts {
             pushCmd,
             jobId,
             token,
-            fieldsToUpdate ? pack(objectToFlatArray(fieldsToUpdate)) : void 0,
+            opts.fieldsToUpdate
+                ? pack(objectToFlatArray(opts.fieldsToUpdate))
+                : void 0,
         ]);
     }
-    async retryJob(jobId, lifo, token, fieldsToUpdate) {
+    async retryJob(jobId, lifo, token = '0', opts = {}) {
         const client = await this.queue.client;
-        const args = this.retryJobArgs(jobId, lifo, token, fieldsToUpdate);
+        const args = this.retryJobArgs(jobId, lifo, token, opts);
         const result = await this.execCommand(client, 'retryJob', args);
         if (result < 0) {
             throw this.finishedErrors({
@@ -863,7 +958,6 @@ export class Scripts {
             this.queue.keys.stalled,
             this.queue.keys.wait,
             this.queue.keys.active,
-            this.queue.keys.failed,
             this.queue.keys['stalled-check'],
             this.queue.keys.meta,
             this.queue.keys.paused,
@@ -901,14 +995,12 @@ export class Scripts {
      * @param jobId - Job id
      * @returns
      */
-    async moveJobFromActiveToWait(jobId, token) {
+    async moveJobFromActiveToWait(jobId, token = '0') {
         const client = await this.queue.client;
-        const lockKey = `${this.queue.toKey(jobId)}:lock`;
         const keys = [
             this.queue.keys.active,
             this.queue.keys.wait,
             this.queue.keys.stalled,
-            lockKey,
             this.queue.keys.paused,
             this.queue.keys.meta,
             this.queue.keys.limiter,
@@ -917,8 +1009,16 @@ export class Scripts {
             this.queue.keys.events,
         ];
         const args = [jobId, token, this.queue.toKey(jobId)];
-        const pttl = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
-        return pttl < 0 ? 0 : pttl;
+        const result = await this.execCommand(client, 'moveJobFromActiveToWait', keys.concat(args));
+        if (result < 0) {
+            throw this.finishedErrors({
+                code: result,
+                jobId,
+                command: 'moveJobFromActiveToWait',
+                state: 'active',
+            });
+        }
+        return result;
     }
     async obliterate(opts) {
         const client = await this.queue.client;
@@ -940,7 +1040,7 @@ export class Scripts {
     }
     /**
      * Paginate a set or hash keys.
-     * @param opts
+     * @param opts - options to define the pagination behaviour
      *
      */
     async paginate(key, opts) {
@@ -1003,7 +1103,7 @@ export class Scripts {
                 return new Error(`Missing lock for job ${jobId}. ${command}`);
             case ErrorCode.JobNotInState:
                 return new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
-            case ErrorCode.JobPendingDependencies:
+            case ErrorCode.JobPendingChildren:
                 return new Error(`Job ${jobId} has pending dependencies. ${command}`);
             case ErrorCode.ParentJobNotExist:
                 return new Error(`Missing key for parent job ${parentKey}. ${command}`);
@@ -1013,6 +1113,8 @@ export class Scripts {
                 return new Error(`The parent job ${parentKey} cannot be replaced. ${command}`);
             case ErrorCode.JobBelongsToJobScheduler:
                 return new Error(`Job ${jobId} belongs to a job scheduler and cannot be removed directly. ${command}`);
+            case ErrorCode.JobFailedChildren:
+                return new Error(`Job ${jobId} has failed children. ${command}`);
             default:
                 return new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
         }

package/dist/node_modules/bullmq/dist/esm/classes/worker.d.ts

@@ -1,7 +1,8 @@
 /// <reference types="node" />
 import { URL } from 'url';
 import { AbortController } from 'node-abort-controller';
-import { GetNextJobOptions, IoredisListener, JobJsonRaw, Processor, RedisClient, WorkerOptions } from '../interfaces';
+import { GetNextJobOptions, IoredisListener, JobJsonRaw, Processor, RedisClient, Span, WorkerOptions } from '../interfaces';
+import { JobProgress } from '../types';
 import { QueueBase } from './queue-base';
 import { Repeat } from './repeat';
 import { Job } from './job';
@@ -68,7 +69,7 @@ export interface WorkerListener<DataType = any, ResultType = any, NameType exten
      * progress or any other data from within a processor to the rest of the
      * world.
      */
-    progress: (job: Job<DataType, ResultType, NameType>, progress: number | object) => void;
+    progress: (job: Job<DataType, ResultType, NameType>, progress: JobProgress) => void;
     /**
      * Listen to 'ready' event.
      *
@@ -100,7 +101,6 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
     readonly opts: WorkerOptions;
     readonly id: string;
     private abortDelayController;
-    private asyncFifoQueue;
     private blockingConnection;
     private blockUntil;
     private _concurrency;
@@ -108,14 +108,14 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
     private drained;
     private extendLocksTimer;
     private limitUntil;
-    private resumeWorker;
     private stalledCheckStopper?;
     private waiting;
     private _repeat;
     protected _jobScheduler: JobScheduler;
-    protected paused: Promise<void>;
+    protected paused: boolean;
     protected processFn: Processor<DataType, ResultType, NameType>;
     protected running: boolean;
+    protected mainLoopRunning: Promise<void> | null;
     static RateLimitError(): Error;
     constructor(name: string, processor?: string | URL | null | Processor<DataType, ResultType, NameType>, opts?: WorkerOptions, Connection?: typeof RedisConnection);
     emit<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, ...args: Parameters<WorkerListener<DataType, ResultType, NameType>[U]>): boolean;
@@ -136,6 +136,13 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
     get repeat(): Promise<Repeat>;
     get jobScheduler(): Promise<JobScheduler>;
     run(): Promise<void>;
+    private waitForRateLimit;
+    /**
+     * This is the main loop in BullMQ. Its goals are to fetch jobs from the queue
+     * as efficiently as possible, providing concurrency and minimal unnecessary calls
+     * to Redis.
+     */
+    private mainLoop;
     /**
      * Returns a promise that resolves to the next job in queue.
      * @param token - worker token to be assigned to retrieved job
@@ -150,10 +157,11 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
      */
     rateLimit(expireTimeMs: number): Promise<void>;
     get minimumBlockTimeout(): number;
+    private isRateLimited;
     protected moveToActive(client: RedisClient, token: string, name?: string): Promise<Job<DataType, ResultType, NameType>>;
     private waitForJob;
     protected getBlockTimeout(blockUntil: number): number;
-    protected getLimitUntil(limitUntil: number): number;
+    protected getRateLimitDelay(delay: number): number;
     /**
      *
      * This function is exposed only for testing purposes.
@@ -165,6 +173,20 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
         job: Job;
         ts: number;
     }>): Promise<void | Job<DataType, ResultType, NameType>>;
+    protected handleCompleted(result: ResultType, job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback: () => boolean, jobsInProgress: Set<{
+        job: Job;
+        ts: number;
+    }>, inProgressItem: {
+        job: Job;
+        ts: number;
+    }, span?: Span): Promise<Job<DataType, ResultType, NameType>>;
+    protected handleFailed(err: Error, job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback: () => boolean, jobsInProgress: Set<{
+        job: Job;
+        ts: number;
+    }>, inProgressItem: {
+        job: Job;
+        ts: number;
+    }, span?: Span): Promise<Job<DataType, ResultType, NameType>>;
     /**
      *
      * Pauses the processing of this queue only for this worker.
@@ -226,6 +248,5 @@ export declare class Worker<DataType = any, ResultType = any, NameType extends s
     private retryIfFailed;
     protected extendLocks(jobs: Job[]): Promise<void>;
     private moveStalledJobsToWait;
-    private notifyFailedJobs;
     private moveLimitedBackToWait;
 }