@tachybase/plugin-adapter-bullmq 1.3.16 → 1.3.18

package/dist/node_modules/bullmq/dist/esm/classes/child-processor.js

@@ -7,6 +7,7 @@ var ChildStatus;
     ChildStatus[ChildStatus["Terminating"] = 2] = "Terminating";
     ChildStatus[ChildStatus["Errored"] = 3] = "Errored";
 })(ChildStatus || (ChildStatus = {}));
+const RESPONSE_TIMEOUT = process.env.NODE_ENV === 'test' ? 500 : 5000;
 /**
  * ChildProcessor
  *
@@ -15,8 +16,9 @@ var ChildStatus;
  *
  */
 export class ChildProcessor {
-    constructor(send) {
+    constructor(send, receiver) {
         this.send = send;
+        this.receiver = receiver;
     }
     async init(processorFile) {
         let processor;
@@ -104,7 +106,7 @@ export class ChildProcessor {
     wrapJob(job, send) {
         const wrappedJob = Object.assign(Object.assign({}, job), { data: JSON.parse(job.data || '{}'), opts: job.opts, returnValue: JSON.parse(job.returnvalue || '{}'), 
             /*
-             * Emulate the real job `updateProgress` function, should works as `progress` function.
+             * Proxy `updateProgress` function, should works as `progress` function.
              */
             async updateProgress(progress) {
                 // Locally store reference to new progress value
@@ -117,7 +119,7 @@ export class ChildProcessor {
                 });
             }, 
             /*
-             * Emulate the real job `log` function.
+             * Proxy job `log` function.
              */
             log: async (row) => {
                 await send({
@@ -126,7 +128,7 @@ export class ChildProcessor {
                 });
             }, 
             /*
-             * Emulate the real job `moveToDelayed` function.
+             * Proxy `moveToDelayed` function.
              */
             moveToDelayed: async (timestamp, token) => {
                 await send({
@@ -135,7 +137,16 @@ export class ChildProcessor {
                 });
             }, 
             /*
-             * Emulate the real job `updateData` function.
+             * Proxy `moveToWait` function.
+             */
+            moveToWait: async (token) => {
+                await send({
+                    cmd: ParentCommand.MoveToWait,
+                    value: { token },
+                });
+            }, 
+            /*
+             * Proxy `updateData` function.
              */
             updateData: async (data) => {
                 await send({
@@ -143,8 +154,51 @@ export class ChildProcessor {
                     value: data,
                 });
                 wrappedJob.data = data;
+            }, 
+            /**
+             * Proxy `getChildrenValues` function.
+             */
+            getChildrenValues: async () => {
+                const requestId = Math.random().toString(36).substring(2, 15);
+                await send({
+                    requestId,
+                    cmd: ParentCommand.GetChildrenValues,
+                });
+                return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getChildrenValues');
+            }, 
+            /**
+             * Proxy `getIgnoredChildrenFailures` function.
+             *
+             * This method sends a request to retrieve the failures of ignored children
+             * and waits for a response from the parent process.
+             *
+             * @returns - A promise that resolves with the ignored children failures.
+             * The exact structure of the returned data depends on the parent process implementation.
+             */
+            getIgnoredChildrenFailures: async () => {
+                const requestId = Math.random().toString(36).substring(2, 15);
+                await send({
+                    requestId,
+                    cmd: ParentCommand.GetIgnoredChildrenFailures,
+                });
+                return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getIgnoredChildrenFailures');
             } });
         return wrappedJob;
     }
 }
+const waitResponse = async (requestId, receiver, timeout, cmd) => {
+    return new Promise((resolve, reject) => {
+        const listener = (msg) => {
+            if (msg.requestId === requestId) {
+                resolve(msg.value);
+                receiver.off('message', listener);
+            }
+        };
+        receiver.on('message', listener);
+        setTimeout(() => {
+            receiver.off('message', listener);
+            reject(new Error(`TimeoutError: ${cmd} timed out in (${timeout}ms)`));
+        }, timeout);
+    });
+};
 //# sourceMappingURL=child-processor.js.map

package/dist/node_modules/bullmq/dist/esm/classes/child.js

@@ -44,7 +44,9 @@ export class Child extends EventEmitter {
             return this.childProcess.pid;
         }
         else if (this.worker) {
-            return this.worker.threadId;
+            // Worker threads pids can become negative when they are terminated
+            // so we need to use the absolute value to index the retained object
+            return Math.abs(this.worker.threadId);
         }
         else {
             throw new Error('No child process or worker thread');

package/dist/node_modules/bullmq/dist/esm/classes/errors/index.d.ts

@@ -1,4 +1,5 @@
 export * from './delayed-error';
-export * from './unrecoverable-error';
 export * from './rate-limit-error';
+export * from './unrecoverable-error';
 export * from './waiting-children-error';
+export * from './waiting-error';

package/dist/node_modules/bullmq/dist/esm/classes/errors/index.js

@@ -1,5 +1,6 @@
 export * from './delayed-error';
-export * from './unrecoverable-error';
 export * from './rate-limit-error';
+export * from './unrecoverable-error';
 export * from './waiting-children-error';
+export * from './waiting-error';
 //# sourceMappingURL=index.js.map

package/dist/node_modules/bullmq/dist/esm/classes/errors/waiting-error.d.ts

@@ -0,0 +1,10 @@
+export declare const WAITING_ERROR = "bullmq:movedToWait";
+/**
+ * WaitingError
+ *
+ * Error to be thrown when job is moved to wait or prioritized state
+ * from job in active state.
+ */
+export declare class WaitingError extends Error {
+    constructor(message?: string);
+}

package/dist/node_modules/bullmq/dist/esm/classes/errors/waiting-error.js

@@ -0,0 +1,15 @@
+export const WAITING_ERROR = 'bullmq:movedToWait';
+/**
+ * WaitingError
+ *
+ * Error to be thrown when job is moved to wait or prioritized state
+ * from job in active state.
+ */
+export class WaitingError extends Error {
+    constructor(message = WAITING_ERROR) {
+        super(message);
+        this.name = this.constructor.name;
+        Object.setPrototypeOf(this, new.target.prototype);
+    }
+}
+//# sourceMappingURL=waiting-error.js.map

package/dist/node_modules/bullmq/dist/esm/classes/flow-producer.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 import { EventEmitter } from 'events';
 import { ChainableCommander } from 'ioredis';
-import { FlowJob, FlowQueuesOpts, FlowOpts, IoredisListener, QueueBaseOptions, RedisClient, Tracer, ContextManager } from '../interfaces';
+import { FlowJob, FlowQueuesOpts, FlowOpts, IoredisListener, ParentOptions, QueueBaseOptions, RedisClient, Tracer, ContextManager } from '../interfaces';
 import { Job } from './job';
 import { KeysMap, QueueKeys } from './queue-keys';
 import { RedisConnection } from './redis-connection';
@@ -9,10 +9,7 @@ export interface AddNodeOpts {
     multi: ChainableCommander;
     node: FlowJob;
     parent?: {
-        parentOpts: {
-            id: string;
-            queue: string;
-        };
+        parentOpts: ParentOptions;
         parentDependenciesKey: string;
     };
     /**
@@ -24,10 +21,7 @@ export interface AddChildrenOpts {
     multi: ChainableCommander;
     nodes: FlowJob[];
     parent: {
-        parentOpts: {
-            id: string;
-            queue: string;
-        };
+        parentOpts: ParentOptions;
         parentDependenciesKey: string;
     };
     queuesOpts?: FlowQueuesOpts;

package/dist/node_modules/bullmq/dist/esm/classes/flow-producer.js

@@ -18,7 +18,12 @@ export class FlowProducer extends EventEmitter {
         super();
         this.opts = opts;
         this.opts = Object.assign({ prefix: 'bull' }, opts);
-        this.connection = new Connection(opts.connection, isRedisInstance(opts.connection), false, opts.skipVersionCheck);
+        this.connection = new Connection(opts.connection, {
+            shared: isRedisInstance(opts.connection),
+            blocking: false,
+            skipVersionCheck: opts.skipVersionCheck,
+            skipWaitingForReady: opts.skipWaitingForReady,
+        });
         this.connection.on('error', (error) => this.emit('error', error));
         this.connection.on('close', () => {
             if (!this.closing) {
@@ -112,6 +117,7 @@ export class FlowProducer extends EventEmitter {
         const updatedOpts = Object.assign({
             depth: 10,
             maxChildren: 20,
+            prefix: this.opts.prefix,
         }, opts);
         const jobsTree = this.getNode(client, updatedOpts);
         return jobsTree;
@@ -166,7 +172,7 @@ export class FlowProducer extends EventEmitter {
         const queueOpts = queuesOpts && queuesOpts[node.queueName];
         const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
         const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || v4();
-        return trace(this.telemetry, SpanKind.PRODUCER, node.name, 'addNode', node.queueName, async (span, srcPropagationMedatada) => {
+        return trace(this.telemetry, SpanKind.PRODUCER, node.queueName, 'addNode', node.queueName, async (span, srcPropagationMedatada) => {
             var _a, _b;
             span === null || span === void 0 ? void 0 : span.setAttributes({
                 [TelemetryAttributes.JobName]: node.name,
@@ -253,19 +259,29 @@ export class FlowProducer extends EventEmitter {
         const queue = this.queueFromNode(node, new QueueKeys(node.prefix), node.prefix);
         const job = await this.Job.fromId(queue, node.id);
         if (job) {
-            const { processed = {}, unprocessed = [] } = await job.getDependencies({
+            const { processed = {}, unprocessed = [], failed = [], ignored = {}, } = await job.getDependencies({
+                failed: {
+                    count: node.maxChildren,
+                },
                 processed: {
                     count: node.maxChildren,
                 },
                 unprocessed: {
                     count: node.maxChildren,
                 },
+                ignored: {
+                    count: node.maxChildren,
+                },
             });
             const processedKeys = Object.keys(processed);
-            const childrenCount = processedKeys.length + unprocessed.length;
+            const ignoredKeys = Object.keys(ignored);
+            const childrenCount = processedKeys.length +
+                unprocessed.length +
+                ignoredKeys.length +
+                failed.length;
             const newDepth = node.depth - 1;
             if (childrenCount > 0 && newDepth) {
-                const children = await this.getChildren(client, [...processedKeys, ...unprocessed], newDepth, node.maxChildren);
+                const children = await this.getChildren(client, [...processedKeys, ...unprocessed, ...failed, ...ignoredKeys], newDepth, node.maxChildren);
                 return { job, children };
             }
             else {

package/dist/node_modules/bullmq/dist/esm/classes/index.d.ts

@@ -1,5 +1,6 @@
 export * from './async-fifo-queue';
 export * from './backoffs';
+export * from './child';
 export * from './child-pool';
 export * from './child-processor';
 export * from './errors';

package/dist/node_modules/bullmq/dist/esm/classes/index.js

@@ -1,5 +1,6 @@
 export * from './async-fifo-queue';
 export * from './backoffs';
+export * from './child';
 export * from './child-pool';
 export * from './child-processor';
 export * from './errors';

package/dist/node_modules/bullmq/dist/esm/classes/job-scheduler.d.ts

@@ -10,12 +10,12 @@ export declare class JobScheduler extends QueueBase {
         override: boolean;
         producerId?: string;
     }): Promise<Job<T, R, N> | undefined>;
-    private createNextJob;
+    private getNextJobOpts;
     removeJobScheduler(jobSchedulerId: string): Promise<number>;
     private getSchedulerData;
     private transformSchedulerData;
     private keyToData;
-    getScheduler<D = any>(id: string): Promise<JobSchedulerJson<D>>;
+    getScheduler<D = any>(id: string): Promise<JobSchedulerJson<D> | undefined>;
     private getTemplateFromJSON;
     getJobSchedulers<D = any>(start?: number, end?: number, asc?: boolean): Promise<JobSchedulerJson<D>[]>;
     getSchedulersCount(): Promise<number>;

package/dist/node_modules/bullmq/dist/esm/classes/job-scheduler.js

@@ -11,7 +11,7 @@ export class JobScheduler extends QueueBase {
             (opts.settings && opts.settings.repeatStrategy) || defaultRepeatStrategy;
     }
     async upsertJobScheduler(jobSchedulerId, repeatOpts, jobName, jobData, opts, { override, producerId }) {
-        const { every, pattern, offset } = repeatOpts;
+        const { every, limit, pattern, offset } = repeatOpts;
         if (pattern && every) {
             throw new Error('Both .pattern and .every options are defined for this repeatable job');
         }
@@ -33,51 +33,38 @@ export class JobScheduler extends QueueBase {
         // Check if we reached the end date of the repeatable job
         let now = Date.now();
         const { endDate } = repeatOpts;
-        if (!(typeof endDate === undefined) && now > new Date(endDate).getTime()) {
+        if (endDate && now > new Date(endDate).getTime()) {
             return;
         }
+        const prevMillis = opts.prevMillis || 0;
+        now = prevMillis < now ? now : prevMillis;
         // Check if we have a start date for the repeatable job
         const { startDate, immediately } = repeatOpts, filteredRepeatOpts = __rest(repeatOpts, ["startDate", "immediately"]);
+        let startMillis = (now > prevMillis ? now : prevMillis) + (offset || 0);
         if (startDate) {
-            const startMillis = new Date(startDate).getTime();
-            now = startMillis > now ? startMillis : now;
+            startMillis = new Date(startDate).getTime();
+            startMillis = startMillis > now ? startMillis : now;
         }
-        const prevMillis = opts.prevMillis || 0;
-        now = prevMillis < now ? now : prevMillis;
         let nextMillis;
-        let newOffset = offset;
+        let newOffset = null;
         if (every) {
-            const nextSlot = Math.floor(now / every) * every + every;
+            const prevSlot = Math.floor((startMillis - (offset || 0)) / every) * every;
+            newOffset = typeof offset === 'number' ? offset : startMillis - prevSlot;
+            const nextSlot = prevSlot + every;
             if (prevMillis || offset) {
-                nextMillis = nextSlot + (offset || 0);
+                nextMillis = nextSlot;
             }
             else {
-                nextMillis = now;
-                newOffset = every - (nextSlot - now);
-                // newOffset should always be positive, but as an extra safety check
-                newOffset = newOffset < 0 ? 0 : newOffset;
-            }
-            if (nextMillis < now) {
-                nextMillis = now;
+                nextMillis = prevSlot;
             }
         }
         else if (pattern) {
             nextMillis = await this.repeatStrategy(now, repeatOpts, jobName);
+            if (nextMillis < now) {
+                nextMillis = now;
+            }
         }
-        const multi = (await this.client).multi();
         if (nextMillis) {
-            if (override) {
-                this.scripts.addJobScheduler(multi, jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), Job.optsAsJSON(opts), {
-                    name: jobName,
-                    endDate: endDate ? new Date(endDate).getTime() : undefined,
-                    tz: repeatOpts.tz,
-                    pattern,
-                    every,
-                });
-            }
-            else {
-                this.scripts.updateJobSchedulerNextMillis(multi, jobSchedulerId, nextMillis);
-            }
             return this.trace(SpanKind.PRODUCER, 'add', `${this.name}.${jobName}`, async (span, srcPropagationMedatada) => {
                 var _a, _b;
                 let telemetry = opts.telemetry;
@@ -92,27 +79,42 @@ export class JobScheduler extends QueueBase {
                         };
                     }
                 }
-                const job = this.createNextJob(multi, jobName, nextMillis, jobSchedulerId, Object.assign(Object.assign({}, opts), { repeat: Object.assign(Object.assign({}, filteredRepeatOpts), { offset: newOffset }), telemetry }), jobData, iterationCount, producerId);
-                const results = await multi.exec(); // multi.exec returns an array of results [ err, result ][]
-                // Check if there are any errors
-                const erroredResult = results.find(result => result[0]);
-                if (erroredResult) {
-                    throw new Error(`Error upserting job scheduler ${jobSchedulerId} - ${erroredResult[0]}`);
+                const mergedOpts = this.getNextJobOpts(nextMillis, jobSchedulerId, Object.assign(Object.assign({}, opts), { repeat: filteredRepeatOpts, telemetry }), iterationCount, newOffset);
+                if (override) {
+                    const jobId = await this.scripts.addJobScheduler(jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), Job.optsAsJSON(opts), {
+                        name: jobName,
+                        endDate: endDate ? new Date(endDate).getTime() : undefined,
+                        tz: repeatOpts.tz,
+                        pattern,
+                        every,
+                        limit,
+                        offset: newOffset,
+                    }, Job.optsAsJSON(mergedOpts), producerId);
+                    const job = new this.Job(this, jobName, jobData, mergedOpts, jobId);
+                    job.id = jobId;
+                    span === null || span === void 0 ? void 0 : span.setAttributes({
+                        [TelemetryAttributes.JobSchedulerId]: jobSchedulerId,
+                        [TelemetryAttributes.JobId]: job.id,
+                    });
+                    return job;
+                }
+                else {
+                    const jobId = await this.scripts.updateJobSchedulerNextMillis(jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), Job.optsAsJSON(mergedOpts), producerId);
+                    if (jobId) {
+                        const job = new this.Job(this, jobName, jobData, mergedOpts, jobId);
+                        job.id = jobId;
+                        span === null || span === void 0 ? void 0 : span.setAttributes({
+                            [TelemetryAttributes.JobSchedulerId]: jobSchedulerId,
+                            [TelemetryAttributes.JobId]: job.id,
+                        });
+                        return job;
+                    }
                 }
-                // Get last result with the job id
-                const lastResult = results.pop();
-                job.id = lastResult[1];
-                span === null || span === void 0 ? void 0 : span.setAttributes({
-                    [TelemetryAttributes.JobSchedulerId]: jobSchedulerId,
-                    [TelemetryAttributes.JobId]: job.id,
-                });
-                return job;
             });
         }
     }
-    createNextJob(client, name, nextMillis, jobSchedulerId, opts, data, currentCount, 
-    // The job id of the job that produced this next iteration
-    producerId) {
+    getNextJobOpts(nextMillis, jobSchedulerId, opts, currentCount, offset) {
+        var _a;
         //
         // Generate unique job id for this iteration.
         //
@@ -121,16 +123,12 @@ export class JobScheduler extends QueueBase {
             nextMillis,
         });
         const now = Date.now();
-        const delay = nextMillis - now;
+        const delay = nextMillis + offset - now;
         const mergedOpts = Object.assign(Object.assign({}, opts), { jobId, delay: delay < 0 ? 0 : delay, timestamp: now, prevMillis: nextMillis, repeatJobKey: jobSchedulerId });
-        mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { count: currentCount });
-        const job = new this.Job(this, name, data, mergedOpts, jobId);
-        job.addJob(client);
-        if (producerId) {
-            const producerJobKey = this.toKey(producerId);
-            client.hset(producerJobKey, 'nrjid', job.id);
-        }
-        return job;
+        mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { offset, count: currentCount, endDate: ((_a = opts.repeat) === null || _a === void 0 ? void 0 : _a.endDate)
+                ? new Date(opts.repeat.endDate).getTime()
+                : undefined });
+        return mergedOpts;
     }
     async removeJobScheduler(jobSchedulerId) {
         return this.scripts.removeJobScheduler(jobSchedulerId);
@@ -139,15 +137,43 @@ export class JobScheduler extends QueueBase {
         const jobData = await client.hgetall(this.toKey('repeat:' + key));
         return this.transformSchedulerData(key, jobData, next);
     }
-    async transformSchedulerData(key, jobData, next) {
+    transformSchedulerData(key, jobData, next) {
         if (jobData) {
-            return Object.assign(Object.assign({ key, name: jobData.name, endDate: parseInt(jobData.endDate) || null, tz: jobData.tz || null, pattern: jobData.pattern || null, every: jobData.every || null }, (jobData.data || jobData.opts
-                ? {
-                    template: this.getTemplateFromJSON(jobData.data, jobData.opts),
-                }
-                : {})), { next });
+            const jobSchedulerData = {
+                key,
+                name: jobData.name,
+                next,
+            };
+            if (jobData.ic) {
+                jobSchedulerData.iterationCount = parseInt(jobData.ic);
+            }
+            if (jobData.limit) {
+                jobSchedulerData.limit = parseInt(jobData.limit);
+            }
+            if (jobData.endDate) {
+                jobSchedulerData.endDate = parseInt(jobData.endDate);
+            }
+            if (jobData.tz) {
+                jobSchedulerData.tz = jobData.tz;
+            }
+            if (jobData.pattern) {
+                jobSchedulerData.pattern = jobData.pattern;
+            }
+            if (jobData.every) {
+                jobSchedulerData.every = parseInt(jobData.every);
+            }
+            if (jobData.offset) {
+                jobSchedulerData.offset = parseInt(jobData.offset);
+            }
+            if (jobData.data || jobData.opts) {
+                jobSchedulerData.template = this.getTemplateFromJSON(jobData.data, jobData.opts);
+            }
+            return jobSchedulerData;
+        }
+        // TODO: remove this check and keyToData as it is here only to support legacy code
+        if (key.includes(':')) {
+            return this.keyToData(key, next);
         }
-        return this.keyToData(key, next);
     }
     keyToData(key, next) {
         const data = key.split(':');