bullmq 5.81.2 → 6.0.0

package/README.md

@@ -4,7 +4,7 @@
   <br/>
   <br/>
   <p>
-    The fastest, most reliable, Redis-based distributed queue for Node.js, Python, Rust, and more. <br/>
+    The fastest, most reliable, Redis-based distributed queue for Node.js, Python, Elixir, Rust, PHP, and more. <br/>
     Carefully written for rock solid stability and atomicity.
   </p>
   Read the <a href="https://docs.bullmq.io">documentation</a>
@@ -44,7 +44,7 @@ BullMQ is available natively in multiple languages:
 
 - **Node.js / Bun** — This repository (`npm install bullmq`)
 - **Python** — [`python/`](./python) directory (`pip install bullmq`)
-- **Rust** — [`rust/`](./rust) directory (`cargo add bullmq`)
+- **Rust** — [`rust/`](./rust) directory (`cargo add bullmq-official --rename bullmq`)
 - **Elixir** — [`elixir/`](./elixir) directory (`{:bullmq, "~> x.x"}`)
 - **PHP** — [`php/`](./php) directory
 

package/dist/cjs/classes/bun-redis-client.js

@@ -186,7 +186,13 @@ class BunRedisAdapter extends events_1.EventEmitter {
     // Connection lifecycle
     // ---------------------------------------------------------------
     async connect() {
-        if (this.raw.connected) {
+        const replaceRaw = this.hasConnected && (this.closed || !this.raw.connected);
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = null;
+        }
+        this.reconnecting = false;
+        if (this.raw.connected && !replaceRaw) {
             this.hasConnected = true;
             this.closed = false;
             this.closing = false;
@@ -199,7 +205,7 @@ class BunRedisAdapter extends events_1.EventEmitter {
             this.statusOverride = undefined;
             // If the raw client was previously closed, Bun doesn't support
             // reconnecting on the same instance. Create a fresh raw client.
-            if (this.hasConnected && !this.raw.connected) {
+            if (replaceRaw) {
                 const BunRedisClient = this.raw
                     .constructor;
                 this.raw = new BunRedisClient(this.raw.url);

package/dist/cjs/classes/finished-errors.js

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.finishedErrors = finishedErrors;
+const enums_1 = require("../enums");
+const unrecoverable_error_1 = require("./errors/unrecoverable-error");
+/**
+ * Builds the canonical `Error` for a negative status code returned by a backend
+ * operation (Lua script, SQL function, …). Shared by every backend so that the
+ * error messages a caller sees are identical regardless of the datastore.
+ *
+ * The resulting error carries the numeric `code` so callers can branch on it.
+ */
+function finishedErrors({ code, jobId, parentKey, command, state, }) {
+    let error;
+    switch (code) {
+        case enums_1.ErrorCode.JobNotExist:
+            error = new Error(`Missing key for job ${jobId}. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobLockNotExist:
+            error = new Error(`Missing lock for job ${jobId}. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobNotInState:
+            error = new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobPendingChildren:
+            error = new Error(`Job ${jobId} has pending dependencies. ${command}`);
+            break;
+        case enums_1.ErrorCode.ParentJobNotExist:
+            error = new Error(`Missing key for parent job ${parentKey}. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobLockMismatch:
+            error = new Error(`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`);
+            break;
+        case enums_1.ErrorCode.ParentJobCannotBeReplaced:
+            error = new Error(`The parent job ${parentKey} cannot be replaced. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobBelongsToJobScheduler:
+            error = new Error(`Job ${jobId} belongs to a job scheduler and cannot be removed directly. ${command}`);
+            break;
+        case enums_1.ErrorCode.JobHasFailedChildren:
+            error = new unrecoverable_error_1.UnrecoverableError(`Cannot complete job ${jobId} because it has at least one failed child. ${command}`);
+            break;
+        case enums_1.ErrorCode.SchedulerJobIdCollision:
+            error = new Error(`Cannot create job scheduler iteration - job ID already exists. ${command}`);
+            break;
+        case enums_1.ErrorCode.SchedulerJobSlotsBusy:
+            error = new Error(`Cannot create job scheduler iteration - current and next time slots already have jobs. ${command}`);
+            break;
+        default:
+            error = new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
+    }
+    // Add the code property to the error object
+    error.code = code;
+    return error;
+}

package/dist/cjs/classes/flow-producer.js

@@ -3,10 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.FlowProducer = void 0;
 const events_1 = require("events");
 const utils_1 = require("../utils");
-const create_scripts_1 = require("../utils/create-scripts");
+const create_backend_1 = require("../utils/create-backend");
 const job_1 = require("./job");
-const queue_keys_1 = require("./queue-keys");
-const redis_connection_1 = require("./redis-connection");
 const enums_1 = require("../enums");
 /**
  * This class allows to add jobs with dependencies between them in such
@@ -17,31 +15,23 @@ const enums_1 = require("../enums");
  * All Jobs can be in different queues, either children or parents,
  */
 class FlowProducer extends events_1.EventEmitter {
-    constructor(opts = { connection: {} }, Connection = redis_connection_1.RedisConnection) {
+    constructor(opts = { connection: {} }, backendFactory = (0, create_backend_1.getDefaultBackendFactory)()) {
         super();
         this.opts = opts;
-        /**
-         * Cache of lightweight queue-like objects keyed by `${prefix}:${queueName}`.
-         * Each entry carries a single shared Scripts instance that is reused by all
-         * jobs added to that queue during the FlowProducer's lifetime, avoiding a
-         * redundant Scripts allocation per node/job.
-         */
-        this.queues = new Map();
-        this.opts = Object.assign({ prefix: 'bull' }, opts);
-        this.connection = new Connection(opts.connection, {
-            shared: (0, utils_1.isRedisInstance)(opts.connection),
-            blocking: false,
-            skipVersionCheck: opts.skipVersionCheck,
-            skipWaitingForReady: opts.skipWaitingForReady,
+        this.opts = Object.assign({}, opts);
+        // The flow producer is not bound to a single queue: each flow entry carries
+        // its own queue identity, so the backend is created with an empty name.
+        this.backend = backendFactory('', this.opts);
+        this.backend.on('error', (error) => {
+            if (this.listenerCount('error') > 0) {
+                this.emit('error', error);
+            }
         });
-        (0, utils_1.forwardConnectionError)(this, this.connection);
-        this.connection.on('close', () => {
-            this.queues.clear();
+        this.backend.on('close', () => {
             if (!this.closing) {
                 this.emit('ioredis:close');
             }
         });
-        this.queueKeys = new queue_keys_1.QueueKeys(opts.prefix);
         if (opts === null || opts === void 0 ? void 0 : opts.telemetry) {
             this.telemetry = opts.telemetry;
         }
@@ -61,12 +51,6 @@ class FlowProducer extends events_1.EventEmitter {
         super.once(event, listener);
         return this;
     }
-    /**
-     * Returns a promise that resolves to a redis client. Normally used only by subclasses.
-     */
-    get client() {
-        return this.connection.client;
-    }
     /**
      * Helper to easily extend Job class calls.
      */
@@ -74,7 +58,19 @@ class FlowProducer extends events_1.EventEmitter {
         return job_1.Job;
     }
     waitUntilReady() {
-        return this.client;
+        return this.backend.waitUntilReady();
+    }
+    /**
+     * Returns the datastore backend that powers this flow producer.
+     *
+     * The backend owns its connection and exposes every datastore-agnostic
+     * operation through {@link IQueueBackend}. Datastore-specific escape hatches
+     * (e.g. the raw Redis client) live on the concrete backend implementation,
+     * and are exposed here when the flow producer is parameterized on that
+     * concrete backend type (the default is the Redis backend).
+     */
+    getBackend() {
+        return this.backend;
     }
     /**
      * Adds a flow.
@@ -87,13 +83,15 @@ class FlowProducer extends events_1.EventEmitter {
      * @param opts - options that will be applied to the flow object.
      */
     async add(flow, opts) {
-        var _a;
         if (this.closing) {
             return;
         }
-        const client = await this.connection.client;
-        const multi = client.multi();
-        const parentOpts = (_a = flow === null || flow === void 0 ? void 0 : flow.opts) === null || _a === void 0 ? void 0 : _a.parent;
+        this.validateFlowJobs([flow]);
+        // Ensure the backend (and thus the connection) is ready before building
+        // the per-node queue contexts used to create jobs.
+        await this.backend.waitUntilReady();
+        const flowOpts = flow === null || flow === void 0 ? void 0 : flow.opts;
+        const parentOpts = flowOpts && 'parent' in flowOpts ? flowOpts.parent : undefined;
         const parentKey = (0, utils_1.getParentKey)(parentOpts);
         const parentDependenciesKey = parentKey
             ? `${parentKey}:dependencies`
@@ -102,8 +100,9 @@ class FlowProducer extends events_1.EventEmitter {
             span === null || span === void 0 ? void 0 : span.setAttributes({
                 [enums_1.TelemetryAttributes.FlowName]: flow.name,
             });
+            const entries = [];
             const jobsTree = await this.addNode({
-                multi,
+                entries,
                 node: flow,
                 queuesOpts: opts === null || opts === void 0 ? void 0 : opts.queuesOptions,
                 parent: {
@@ -111,7 +110,7 @@ class FlowProducer extends events_1.EventEmitter {
                     parentDependenciesKey,
                 },
             });
-            const results = (await multi.exec());
+            const results = await this.backend.addFlow(entries);
             const [result] = results || [];
             if (result) {
                 const [err, jobId] = result;
@@ -124,9 +123,6 @@ class FlowProducer extends events_1.EventEmitter {
                 if (typeof jobId === 'string') {
                     jobsTree.job.id = jobId;
                 }
-                else if (typeof jobId === 'number') {
-                    jobsTree.job.id = jobId.toString();
-                }
             }
             return jobsTree;
         });
@@ -140,13 +136,13 @@ class FlowProducer extends events_1.EventEmitter {
         if (this.closing) {
             return;
         }
-        const client = await this.connection.client;
+        await this.backend.waitUntilReady();
         const updatedOpts = Object.assign({
             depth: 10,
             maxChildren: 20,
             prefix: this.opts.prefix,
         }, opts);
-        const jobsTree = this.getNode(client, updatedOpts);
+        const jobsTree = this.getNode(updatedOpts);
         return jobsTree;
     }
     /**
@@ -156,8 +152,9 @@ class FlowProducer extends events_1.EventEmitter {
      * Whenever the children of a given parent are completed, the parent
      * will be processed, being able to access the children's result data.
      *
-     * All Jobs can be in different queues, either children or parents.
-     * If a flow fails to be added, other flows in the batch may still be added.
+     * All Jobs can be in different queues, either children or parents,
+     * however this call would be atomic, either it fails and no jobs will
+     * be added to the queues, or it succeeds and all jobs will be added.
      *
      * @param flows - an array of objects with a tree-like structure where children jobs
      * will be processed before their parents.
@@ -166,36 +163,29 @@ class FlowProducer extends events_1.EventEmitter {
         if (this.closing) {
             return;
         }
-        const client = await this.connection.client;
-        const multi = client.multi();
+        this.validateFlowJobs(flows);
+        // Ensure the backend (and thus the connection) is ready before building
+        // the per-node queue contexts used to create jobs.
+        await this.backend.waitUntilReady();
         return (0, utils_1.trace)(this.telemetry, enums_1.SpanKind.PRODUCER, '', 'addBulkFlows', '', async (span) => {
-            var _a, _b;
             span === null || span === void 0 ? void 0 : span.setAttributes({
                 [enums_1.TelemetryAttributes.BulkCount]: flows.length,
                 [enums_1.TelemetryAttributes.BulkNames]: flows
                     .map(flow => flow.name)
                     .join(','),
             });
-            const jobsTrees = await this.addNodes(multi, flows);
-            const results = (await multi.exec());
+            const entries = [];
+            const jobsTrees = await this.addNodes(entries, flows);
+            const results = await this.backend.addFlow(entries);
             for (let index = 0; index < jobsTrees.length; ++index) {
                 const result = results === null || results === void 0 ? void 0 : results[index];
                 if (!result) {
                     continue;
                 }
                 const [err, jobId] = result;
-                if (err) {
-                    throw err;
-                }
-                if (typeof jobId === 'number' && jobId < 0) {
-                    throw this.toFlowError(jobId, (0, utils_1.getParentKey)((_b = (_a = flows[index]) === null || _a === void 0 ? void 0 : _a.opts) === null || _b === void 0 ? void 0 : _b.parent));
-                }
-                if (typeof jobId === 'string') {
+                if (!err && typeof jobId === 'string') {
                     jobsTrees[index].job.id = jobId;
                 }
-                else if (typeof jobId === 'number') {
-                    jobsTrees[index].job.id = jobId.toString();
-                }
             }
             return jobsTrees;
         });
@@ -211,10 +201,10 @@ class FlowProducer extends events_1.EventEmitter {
      * @param parent - parent data sent to children to create the "links" to their parent
      * @returns
      */
-    async addNode({ multi, node, parent, queuesOpts, }) {
+    async addNode({ entries, node, parent, queuesOpts, }) {
         var _a, _b;
         const prefix = node.prefix || this.opts.prefix;
-        const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(prefix), prefix);
+        const queue = this.queueFromNode(node, prefix);
         const queueOpts = queuesOpts && queuesOpts[node.queueName];
         const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
         const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || (0, utils_1.randomUUID)();
@@ -242,20 +232,21 @@ class FlowProducer extends events_1.EventEmitter {
             if (node.children && node.children.length > 0) {
                 // Create the parent job, it will be a job in status "waiting-children".
                 const parentId = jobId;
-                const queueKeysParent = new queue_keys_1.QueueKeys(node.prefix || this.opts.prefix);
-                await job.addJob(multi, {
+                await this.collectFlowEntry(entries, job, {
                     parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
                     addToWaitingChildren: true,
                     parentKey,
                 });
-                const parentDependenciesKey = `${queueKeysParent.toKey(node.queueName, parentId)}:dependencies`;
+                // Queue identity is owned by the backend (the `queue` object above is
+                // bound to this node's queue via the backend's `forQueue`).
+                const parentDependenciesKey = `${queue.toKey(parentId)}:dependencies`;
                 const children = await this.addChildren({
-                    multi,
+                    entries,
                     nodes: node.children,
                     parent: {
                         parentOpts: {
                             id: parentId,
-                            queue: queueKeysParent.getQueueQualifiedName(node.queueName),
+                            queue: queue.qualifiedName,
                         },
                         parentDependenciesKey,
                     },
@@ -264,7 +255,7 @@ class FlowProducer extends events_1.EventEmitter {
                 return { job, children };
             }
             else {
-                await job.addJob(multi, {
+                await this.collectFlowEntry(entries, job, {
                     parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
                     parentKey,
                 });
@@ -282,16 +273,24 @@ class FlowProducer extends events_1.EventEmitter {
      * @param nodes - the nodes representing jobs to be added to some queue
      * @returns
      */
-    addNodes(multi, nodes) {
+    /**
+     * Collects a single job insert for a flow, preserving the same await point
+     * as the previous transaction-based insert so that the relative order of
+     * entries (in particular, roots before their descendants) is unchanged.
+     */
+    async collectFlowEntry(entries, job, parentOpts) {
+        entries.push(job.toFlowEntry(parentOpts));
+    }
+    addNodes(entries, nodes) {
         return Promise.all(nodes.map(node => {
-            var _a;
-            const parentOpts = (_a = node === null || node === void 0 ? void 0 : node.opts) === null || _a === void 0 ? void 0 : _a.parent;
+            const nodeOpts = node === null || node === void 0 ? void 0 : node.opts;
+            const parentOpts = nodeOpts && 'parent' in nodeOpts ? nodeOpts.parent : undefined;
             const parentKey = (0, utils_1.getParentKey)(parentOpts);
             const parentDependenciesKey = parentKey
                 ? `${parentKey}:dependencies`
                 : undefined;
             return this.addNode({
-                multi,
+                entries,
                 node,
                 parent: {
                     parentOpts,
@@ -300,8 +299,8 @@ class FlowProducer extends events_1.EventEmitter {
             });
         }));
     }
-    async getNode(client, node) {
-        const queue = this.queueFromNode(node, new queue_keys_1.QueueKeys(node.prefix), node.prefix);
+    async getNode(node) {
+        const queue = this.queueFromNode(node, node.prefix);
         const job = await this.Job.fromId(queue, node.id);
         if (job) {
             const { processed = {}, unprocessed = [], failed = [], ignored = {}, } = await job.getDependencies({
@@ -326,7 +325,7 @@ class FlowProducer extends events_1.EventEmitter {
                 failed.length;
             const newDepth = node.depth - 1;
             if (childrenCount > 0 && newDepth) {
-                const children = await this.getChildren(client, [...processedKeys, ...unprocessed, ...failed, ...ignoredKeys], newDepth, node.maxChildren);
+                const children = await this.getChildren([...processedKeys, ...unprocessed, ...failed, ...ignoredKeys], newDepth, node.maxChildren);
                 return { job, children };
             }
             else {
@@ -334,13 +333,26 @@ class FlowProducer extends events_1.EventEmitter {
             }
         }
     }
-    addChildren({ multi, nodes, parent, queuesOpts }) {
-        return Promise.all(nodes.map(node => this.addNode({ multi, node, parent, queuesOpts })));
+    validateFlowJobs(nodes) {
+        for (const node of nodes) {
+            const children = node.children;
+            if (children && children.length > 0) {
+                const nodeOpts = node.opts;
+                const hasDeduplication = nodeOpts && 'deduplication' in nodeOpts && nodeOpts.deduplication;
+                if (hasDeduplication) {
+                    throw new Error('Deduplication options cannot be used on flow nodes with children');
+                }
+                this.validateFlowJobs(children);
+            }
+        }
+    }
+    addChildren({ entries, nodes, parent, queuesOpts }) {
+        return Promise.all(nodes.map(node => this.addNode({ entries, node, parent, queuesOpts })));
     }
-    getChildren(client, childrenKeys, depth, maxChildren) {
+    getChildren(childrenKeys, depth, maxChildren) {
         const getChild = (key) => {
-            const [prefix, queueName, id] = key.split(':');
-            return this.getNode(client, {
+            const { prefix, queueName, id } = this.backend.parseNodeKey(key);
+            return this.getNode({
                 id,
                 queueName,
                 prefix,
@@ -355,45 +367,31 @@ class FlowProducer extends events_1.EventEmitter {
      * required to create jobs in any queue.
      *
      * @param node - The flow node containing the queue name and other job options.
-     * @param queueKeys - The queue keys helper used to resolve key names.
-     * @param prefix - The Redis key prefix used for the queue.
-     * @returns A queue-like object with the client, keys, and options needed to create jobs.
+     * @param prefix - The key prefix for the queue (honored by the Redis backend only).
+     * @returns A queue-like object with the keys, identity and backend needed to create jobs.
      */
-    queueFromNode(node, queueKeys, prefix) {
-        const cacheKey = `${prefix}:${node.queueName}`;
-        const cached = this.queues.get(cacheKey);
-        if (cached) {
-            return cached;
-        }
-        const flowProducer = this;
-        const queue = {
-            client: this.connection.client,
+    queueFromNode(node, prefix) {
+        // Queue identity and key building are owned by the backend (the Redis
+        // backend encodes the key `prefix`; other backends format their own
+        // identity). The flow's own backend is queue-agnostic, so we ask it for a
+        // sibling bound to this node's queue.
+        const backend = this.backend.forQueue(node.queueName, prefix);
+        return {
             name: node.queueName,
-            keys: queueKeys.getKeys(node.queueName),
-            toKey: (type) => queueKeys.toKey(node.queueName, type),
+            keys: backend.keys,
+            toKey: (type) => backend.toKey(type),
             opts: { prefix, connection: {} },
-            qualifiedName: queueKeys.getQueueQualifiedName(node.queueName),
-            get closing() {
-                return flowProducer.closing;
+            qualifiedName: backend.qualifiedName,
+            closing: this.closing,
+            backend,
+            waitUntilReady: async () => {
+                await this.backend.waitUntilReady();
             },
-            waitUntilReady: async () => this.connection.client,
             removeListener: this.removeListener.bind(this),
             emit: this.emit.bind(this),
             on: this.on.bind(this),
-            get redisVersion() {
-                return flowProducer.connection.redisVersion;
-            },
-            get databaseType() {
-                return flowProducer.connection.databaseType;
-            },
             trace: async () => { },
         };
-        // Build the shared Scripts instance once per queue so that every job
-        // created from this queue-like object reuses it instead of allocating
-        // its own.
-        queue.scripts = (0, create_scripts_1.createScripts)(queue);
-        this.queues.set(cacheKey, queue);
-        return queue;
     }
     /**
      * Translates numeric addJob Lua error codes returned by root flow exec.
@@ -422,8 +420,7 @@ class FlowProducer extends events_1.EventEmitter {
      */
     async close() {
         if (!this.closing) {
-            this.queues.clear();
-            this.closing = this.connection.close();
+            this.closing = this.backend.close();
         }
         await this.closing;
     }
@@ -432,7 +429,7 @@ class FlowProducer extends events_1.EventEmitter {
      * Force disconnects a connection.
      */
     disconnect() {
-        return this.connection.disconnect();
+        return this.backend.disconnect();
     }
 }
 exports.FlowProducer = FlowProducer;

package/dist/cjs/classes/index.js

@@ -24,7 +24,6 @@ tslib_1.__exportStar(require("./queue-getters"), exports);
 tslib_1.__exportStar(require("./queue-keys"), exports);
 tslib_1.__exportStar(require("./queue"), exports);
 tslib_1.__exportStar(require("./redis-connection"), exports);
-tslib_1.__exportStar(require("./repeat"), exports);
 tslib_1.__exportStar(require("./sandbox"), exports);
-tslib_1.__exportStar(require("./scripts"), exports);
+tslib_1.__exportStar(require("./redis-queue-backend"), exports);
 tslib_1.__exportStar(require("./worker"), exports);