bunqueue 1.9.8 → 2.0.0

package/dist/client/flow.js

@@ -1,8 +1,11 @@
 /**
  * FlowProducer - Job chaining and pipelines
+ * BullMQ v5 compatible
  */
 import { getSharedManager } from './manager';
+import { TcpConnectionPool, getSharedPool, releaseSharedPool } from './tcpPool';
 import { jobId } from '../domain/types/job';
+const FORCE_EMBEDDED = process.env.BUNQUEUE_EMBEDDED === '1';
 /**
  * FlowProducer creates job flows with automatic dependencies.
  *
@@ -28,6 +31,453 @@ import { jobId } from '../domain/types/job';
  * ```
  */
 export class FlowProducer {
+    embedded;
+    tcp;
+    useSharedPool;
+    constructor(opts = {}) {
+        this.embedded = opts.embedded ?? FORCE_EMBEDDED;
+        if (this.embedded) {
+            this.tcp = null;
+            this.useSharedPool = false;
+        }
+        else {
+            const connOpts = opts.connection ?? {};
+            const poolSize = connOpts.poolSize ?? 4;
+            if (poolSize === 4 && !connOpts.token) {
+                this.tcp = getSharedPool({ host: connOpts.host, port: connOpts.port, poolSize });
+                this.useSharedPool = true;
+            }
+            else {
+                this.tcp = new TcpConnectionPool({
+                    host: connOpts.host ?? 'localhost',
+                    port: connOpts.port ?? 6789,
+                    token: connOpts.token,
+                    poolSize,
+                });
+                this.useSharedPool = false;
+            }
+        }
+    }
+    /** Close the connection pool (only if using dedicated pool) */
+    close() {
+        if (this.tcp && !this.useSharedPool) {
+            this.tcp.close();
+        }
+        else if (this.tcp && this.useSharedPool) {
+            releaseSharedPool(this.tcp);
+        }
+    }
+    /**
+     * Disconnect from the server (BullMQ v5 compatible).
+     * Alias for close().
+     */
+    disconnect() {
+        this.close();
+        return Promise.resolve();
+    }
+    /**
+     * Wait until the FlowProducer is ready (BullMQ v5 compatible).
+     * In embedded mode, resolves immediately. In TCP mode, ensures connection.
+     */
+    async waitUntilReady() {
+        if (this.embedded) {
+            // Embedded mode is always ready
+            return;
+        }
+        // TCP mode - ensure we can send a ping
+        if (this.tcp) {
+            await this.tcp.send({ cmd: 'Ping' });
+        }
+    }
+    // ============================================================================
+    // BullMQ v5 Compatible Methods
+    // ============================================================================
+    /**
+     * Add a flow (BullMQ v5 compatible).
+     *
+     * Children are processed BEFORE their parent. When all children complete,
+     * the parent becomes processable and can access children results via
+     * `job.getChildrenValues()`.
+     *
+     * @example
+     * ```typescript
+     * const flow = new FlowProducer();
+     *
+     * // Children execute first, then parent
+     * const { job, children } = await flow.add({
+     *   name: 'parent',
+     *   queueName: 'main',
+     *   data: { type: 'aggregate' },
+     *   children: [
+     *     { name: 'child1', queueName: 'main', data: { id: 1 } },
+     *     { name: 'child2', queueName: 'main', data: { id: 2 } },
+     *   ],
+     * });
+     * ```
+     */
+    async add(flow) {
+        return this.addFlowNode(flow, null);
+    }
+    /**
+     * Add multiple flows (BullMQ v5 compatible).
+     */
+    async addBulk(flows) {
+        const results = [];
+        for (const flow of flows) {
+            results.push(await this.add(flow));
+        }
+        return results;
+    }
+    /**
+     * Get a flow tree starting from a job (BullMQ v5 compatible).
+     *
+     * Retrieves the job and all its children recursively, building a JobNode tree.
+     *
+     * @example
+     * ```typescript
+     * const flow = new FlowProducer();
+     *
+     * // Get a previously created flow
+     * const node = await flow.getFlow({
+     *   id: 'job-id',
+     *   queueName: 'my-queue',
+     * });
+     *
+     * if (node) {
+     *   console.log('Parent:', node.job.name);
+     *   console.log('Children:', node.children?.length ?? 0);
+     * }
+     * ```
+     */
+    async getFlow(opts) {
+        const { id, queueName, depth, maxChildren } = opts;
+        if (this.embedded) {
+            return this.getFlowEmbedded(id, queueName, depth ?? Infinity, maxChildren);
+        }
+        return this.getFlowTcp(id, queueName, depth ?? Infinity, maxChildren);
+    }
+    /** Get flow in embedded mode */
+    async getFlowEmbedded(id, queueName, depth, maxChildren) {
+        const manager = getSharedManager();
+        const job = await manager.getJob(jobId(id));
+        if (job?.queue !== queueName) {
+            return null;
+        }
+        return this.buildJobNode(job, depth, maxChildren);
+    }
+    /** Get flow in TCP mode */
+    async getFlowTcp(id, queueName, depth, maxChildren) {
+        if (!this.tcp)
+            throw new Error('TCP connection not initialized');
+        const response = await this.tcp.send({
+            cmd: 'GetJob',
+            id,
+        });
+        if (!response.ok || !response.job) {
+            return null;
+        }
+        const jobData = response.job;
+        if (jobData.queue !== queueName) {
+            return null;
+        }
+        // Build job node from TCP response
+        return this.buildJobNodeFromTcp(jobData, depth, maxChildren);
+    }
+    /** Build JobNode recursively from internal job */
+    async buildJobNode(job, depth, maxChildren) {
+        const data = job.data;
+        const nameValue = data.name;
+        const name = typeof nameValue === 'string' ? nameValue : 'default';
+        const userData = this.extractUserDataFromInternal(data);
+        const jobObj = this.createJobObject(String(job.id), name, userData, job.queue);
+        // If depth exhausted or no children, return without children
+        if (depth <= 0 || job.childrenIds.length === 0) {
+            return { job: jobObj };
+        }
+        // Fetch children recursively
+        const manager = getSharedManager();
+        const childNodes = [];
+        const childrenToFetch = maxChildren ? job.childrenIds.slice(0, maxChildren) : job.childrenIds;
+        for (const childId of childrenToFetch) {
+            const childJob = await manager.getJob(childId);
+            if (childJob) {
+                const childNode = await this.buildJobNode(childJob, depth - 1, maxChildren);
+                childNodes.push(childNode);
+            }
+        }
+        return {
+            job: jobObj,
+            children: childNodes.length > 0 ? childNodes : undefined,
+        };
+    }
+    /** Build JobNode from TCP response */
+    async buildJobNodeFromTcp(jobData, depth, maxChildren) {
+        const id = String(jobData.id);
+        const queueName = String(jobData.queue);
+        const data = jobData.data;
+        const nameValue = data?.name;
+        const name = typeof nameValue === 'string' ? nameValue : 'default';
+        const userData = this.extractUserDataFromInternal(data ?? {});
+        const rawChildrenIds = data?.__childrenIds;
+        const childrenIds = Array.isArray(rawChildrenIds) ? rawChildrenIds : [];
+        const jobObj = this.createJobObject(id, name, userData, queueName);
+        // If depth exhausted or no children, return without children
+        if (depth <= 0 || childrenIds.length === 0) {
+            return { job: jobObj };
+        }
+        // Fetch children recursively via TCP
+        if (!this.tcp) {
+            return { job: jobObj };
+        }
+        const childNodes = [];
+        const childrenToFetch = maxChildren ? childrenIds.slice(0, maxChildren) : childrenIds;
+        for (const childId of childrenToFetch) {
+            const response = await this.tcp.send({ cmd: 'GetJob', id: childId });
+            if (response.ok && response.job) {
+                const childNode = await this.buildJobNodeFromTcp(response.job, depth - 1, maxChildren);
+                childNodes.push(childNode);
+            }
+        }
+        return {
+            job: jobObj,
+            children: childNodes.length > 0 ? childNodes : undefined,
+        };
+    }
+    /** Extract user data (remove internal fields) */
+    extractUserDataFromInternal(data) {
+        const result = {};
+        for (const [key, value] of Object.entries(data)) {
+            if (!key.startsWith('__') && key !== 'name') {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    /**
+     * Internal: Recursively add a flow node and its children.
+     * Children are created first with parent reference, then parent is created.
+     */
+    async addFlowNode(node, parentRef) {
+        // First, create all children recursively
+        const childNodes = [];
+        const childIds = [];
+        if (node.children && node.children.length > 0) {
+            // Create a placeholder parent ID for children to reference
+            // In BullMQ, children reference the parent, not the other way around
+            const tempParentRef = { id: 'pending', queue: node.queueName };
+            for (const child of node.children) {
+                const childNode = await this.addFlowNode(child, tempParentRef);
+                childNodes.push(childNode);
+                childIds.push(childNode.job.id);
+            }
+        }
+        // Create the job data with parent info if this is a child
+        const jobData = {
+            name: node.name,
+            ...node.data,
+        };
+        // Add parent reference if this job has a parent
+        if (parentRef) {
+            jobData.__parentId = parentRef.id;
+            jobData.__parentQueue = parentRef.queue;
+        }
+        // Add children IDs so parent knows its children
+        if (childIds.length > 0) {
+            jobData.__childrenIds = childIds;
+        }
+        // Push the job
+        const jobIdStr = await this.pushJobWithParent(node.queueName, jobData, node.opts ?? {}, parentRef, childIds);
+        // Create the Job object
+        const job = this.createJobObject(jobIdStr, node.name, node.data, node.queueName);
+        return {
+            job,
+            children: childNodes.length > 0 ? childNodes : undefined,
+        };
+    }
+    /** Push a job with parent/children tracking */
+    async pushJobWithParent(queueName, data, opts, parentRef, childIds) {
+        if (this.embedded) {
+            const manager = getSharedManager();
+            // Parse removeOnComplete/removeOnFail (can be boolean, number, or KeepJobs)
+            const removeOnComplete = typeof opts.removeOnComplete === 'boolean' ? opts.removeOnComplete : false;
+            const removeOnFail = typeof opts.removeOnFail === 'boolean' ? opts.removeOnFail : false;
+            const job = await manager.push(queueName, {
+                data,
+                priority: opts.priority,
+                delay: opts.delay,
+                maxAttempts: opts.attempts,
+                backoff: opts.backoff,
+                timeout: opts.timeout,
+                customId: opts.jobId,
+                removeOnComplete,
+                removeOnFail,
+                parentId: parentRef ? jobId(parentRef.id) : undefined,
+                // Note: childrenIds tracking happens via job.childrenIds field
+            });
+            // If this job has children, update the children's parent references
+            // with the actual parent ID (they were created with 'pending')
+            if (childIds.length > 0) {
+                for (const childIdStr of childIds) {
+                    await manager.updateJobParent(jobId(childIdStr), job.id);
+                }
+            }
+            return String(job.id);
+        }
+        // TCP mode
+        if (!this.tcp)
+            throw new Error('TCP connection not initialized');
+        const response = await this.tcp.send({
+            cmd: 'PUSH',
+            queue: queueName,
+            data,
+            priority: opts.priority,
+            delay: opts.delay,
+            maxAttempts: opts.attempts,
+            backoff: opts.backoff,
+            timeout: opts.timeout,
+            jobId: opts.jobId,
+            removeOnComplete: opts.removeOnComplete,
+            removeOnFail: opts.removeOnFail,
+            parentId: parentRef?.id,
+            childIds,
+        });
+        if (!response.ok) {
+            throw new Error(response.error ?? 'Failed to add job');
+        }
+        return response.id;
+    }
+    /** Create a simple Job object */
+    createJobObject(id, name, data, queueName) {
+        const ts = Date.now();
+        return {
+            id,
+            name,
+            data,
+            queueName,
+            attemptsMade: 0,
+            timestamp: ts,
+            progress: 0,
+            // BullMQ v5 properties
+            delay: 0,
+            processedOn: undefined,
+            finishedOn: undefined,
+            stacktrace: null,
+            stalledCounter: 0,
+            priority: 0,
+            parentKey: undefined,
+            opts: {},
+            token: undefined,
+            processedBy: undefined,
+            deduplicationId: undefined,
+            repeatJobKey: undefined,
+            attemptsStarted: 0,
+            // Methods
+            updateProgress: () => Promise.resolve(),
+            log: () => Promise.resolve(),
+            getState: () => Promise.resolve('waiting'),
+            remove: () => Promise.resolve(),
+            retry: () => Promise.resolve(),
+            getChildrenValues: () => Promise.resolve({}),
+            // BullMQ v5 state check methods
+            isWaiting: () => Promise.resolve(true),
+            isActive: () => Promise.resolve(false),
+            isDelayed: () => Promise.resolve(false),
+            isCompleted: () => Promise.resolve(false),
+            isFailed: () => Promise.resolve(false),
+            isWaitingChildren: () => Promise.resolve(false),
+            // BullMQ v5 mutation methods
+            updateData: () => Promise.resolve(),
+            promote: () => Promise.resolve(),
+            changeDelay: () => Promise.resolve(),
+            changePriority: () => Promise.resolve(),
+            extendLock: () => Promise.resolve(0),
+            clearLogs: () => Promise.resolve(),
+            // BullMQ v5 dependency methods
+            getDependencies: () => Promise.resolve({ processed: {}, unprocessed: [] }),
+            getDependenciesCount: () => Promise.resolve({ processed: 0, unprocessed: 0 }),
+            // BullMQ v5 serialization methods
+            toJSON: () => ({
+                id,
+                name,
+                data,
+                opts: {},
+                progress: 0,
+                delay: 0,
+                timestamp: ts,
+                attemptsMade: 0,
+                stacktrace: null,
+                queueQualifiedName: `bull:${queueName}`,
+            }),
+            asJSON: () => ({
+                id,
+                name,
+                data: JSON.stringify(data),
+                opts: '{}',
+                progress: '0',
+                delay: '0',
+                timestamp: String(ts),
+                attemptsMade: '0',
+                stacktrace: null,
+            }),
+            // BullMQ v5 move methods
+            moveToCompleted: () => Promise.resolve(null),
+            moveToFailed: () => Promise.resolve(),
+            moveToWait: () => Promise.resolve(false),
+            moveToDelayed: () => Promise.resolve(),
+            moveToWaitingChildren: () => Promise.resolve(false),
+            waitUntilFinished: () => Promise.resolve(undefined),
+            // BullMQ v5 additional methods
+            discard: () => { },
+            getFailedChildrenValues: () => Promise.resolve({}),
+            getIgnoredChildrenFailures: () => Promise.resolve({}),
+            removeChildDependency: () => Promise.resolve(false),
+            removeDeduplicationKey: () => Promise.resolve(false),
+            removeUnprocessedChildren: () => Promise.resolve(),
+        };
+    }
+    /** Push a job via embedded manager or TCP */
+    async pushJob(queueName, data, opts = {}, dependsOn) {
+        if (this.embedded) {
+            const manager = getSharedManager();
+            // Parse removeOnComplete/removeOnFail (can be boolean, number, or KeepJobs)
+            const removeOnComplete = typeof opts.removeOnComplete === 'boolean' ? opts.removeOnComplete : false;
+            const removeOnFail = typeof opts.removeOnFail === 'boolean' ? opts.removeOnFail : false;
+            const job = await manager.push(queueName, {
+                data,
+                priority: opts.priority,
+                delay: opts.delay,
+                maxAttempts: opts.attempts,
+                backoff: opts.backoff,
+                timeout: opts.timeout,
+                customId: opts.jobId,
+                removeOnComplete,
+                removeOnFail,
+                dependsOn: dependsOn?.map((id) => jobId(id)),
+            });
+            return String(job.id);
+        }
+        // TCP mode - tcp is guaranteed to exist when not embedded
+        if (!this.tcp)
+            throw new Error('TCP connection not initialized');
+        const response = await this.tcp.send({
+            cmd: 'PUSH',
+            queue: queueName,
+            data,
+            priority: opts.priority,
+            delay: opts.delay,
+            maxAttempts: opts.attempts,
+            backoff: opts.backoff,
+            timeout: opts.timeout,
+            jobId: opts.jobId,
+            removeOnComplete: opts.removeOnComplete,
+            removeOnFail: opts.removeOnFail,
+            dependsOn,
+        });
+        if (!response.ok) {
+            throw new Error(response.error ?? 'Failed to add job');
+        }
+        return response.id;
+    }
     /**
      * Add a chain of jobs where each depends on the previous.
      * Jobs execute sequentially: step[0] → step[1] → step[2] → ...
@@ -36,27 +486,21 @@ export class FlowProducer {
         if (steps.length === 0) {
             return { jobIds: [] };
         }
-        const manager = getSharedManager();
         const jobIds = [];
         let prevId = null;
-        for (const step of steps) {
-            const merged = step.opts ?? {};
-            const input = {
-                data: { name: step.name, __flowParentId: prevId, ...step.data },
-                priority: merged.priority,
-                delay: merged.delay,
-                maxAttempts: merged.attempts,
-                backoff: merged.backoff,
-                timeout: merged.timeout,
-                customId: merged.jobId,
-                removeOnComplete: merged.removeOnComplete,
-                removeOnFail: merged.removeOnFail,
-                dependsOn: prevId ? [jobId(prevId)] : undefined,
-            };
-            const job = await manager.push(step.queueName, input);
-            const id = String(job.id);
-            jobIds.push(id);
-            prevId = id;
+        try {
+            for (const step of steps) {
+                const merged = step.opts ?? {};
+                const data = { name: step.name, __flowParentId: prevId, ...step.data };
+                const id = await this.pushJob(step.queueName, data, merged, prevId ? [prevId] : undefined);
+                jobIds.push(id);
+                prevId = id;
+            }
+        }
+        catch (error) {
+            // Cleanup already-created jobs on failure
+            await this.cleanupJobs(jobIds);
+            throw error;
         }
         return { jobIds };
     }
@@ -72,42 +516,33 @@ export class FlowProducer {
      * ```
      */
     async addBulkThen(parallel, final) {
-        const manager = getSharedManager();
         // Create parallel jobs (no dependencies)
         const parallelIds = [];
-        for (const step of parallel) {
-            const merged = step.opts ?? {};
-            const job = await manager.push(step.queueName, {
-                data: { name: step.name, ...step.data },
-                priority: merged.priority,
-                delay: merged.delay,
-                maxAttempts: merged.attempts,
-                backoff: merged.backoff,
-                timeout: merged.timeout,
-                customId: merged.jobId,
-                removeOnComplete: merged.removeOnComplete,
-                removeOnFail: merged.removeOnFail,
-            });
-            parallelIds.push(String(job.id));
-        }
-        // Create final job with dependencies on all parallel jobs
-        const finalMerged = final.opts ?? {};
-        const finalJob = await manager.push(final.queueName, {
-            data: { name: final.name, __flowParentIds: parallelIds, ...final.data },
-            priority: finalMerged.priority,
-            delay: finalMerged.delay,
-            maxAttempts: finalMerged.attempts,
-            backoff: finalMerged.backoff,
-            timeout: finalMerged.timeout,
-            customId: finalMerged.jobId,
-            removeOnComplete: finalMerged.removeOnComplete,
-            removeOnFail: finalMerged.removeOnFail,
-            dependsOn: parallelIds.map((id) => jobId(id)),
-        });
-        return {
-            parallelIds,
-            finalId: String(finalJob.id),
-        };
+        try {
+            for (const step of parallel) {
+                const merged = step.opts ?? {};
+                const data = { name: step.name, ...step.data };
+                const id = await this.pushJob(step.queueName, data, merged);
+                parallelIds.push(id);
+            }
+            // Create final job with dependencies on all parallel jobs
+            const finalMerged = final.opts ?? {};
+            const finalData = {
+                name: final.name,
+                __flowParentIds: parallelIds,
+                ...final.data,
+            };
+            const finalId = await this.pushJob(final.queueName, finalData, finalMerged, parallelIds);
+            return {
+                parallelIds,
+                finalId,
+            };
+        }
+        catch (error) {
+            // Cleanup already-created parallel jobs on failure
+            await this.cleanupJobs(parallelIds);
+            throw error;
+        }
     }
     /**
      * Add a tree of jobs where children depend on parent.
@@ -115,25 +550,20 @@ export class FlowProducer {
      */
     async addTree(root) {
         const jobIds = [];
-        await this.addTreeNode(root, null, jobIds);
+        try {
+            await this.addTreeNode(root, null, jobIds);
+        }
+        catch (error) {
+            // Cleanup already-created jobs on failure
+            await this.cleanupJobs(jobIds);
+            throw error;
+        }
         return { jobIds };
     }
     async addTreeNode(step, parentId, jobIds) {
-        const manager = getSharedManager();
         const merged = step.opts ?? {};
-        const job = await manager.push(step.queueName, {
-            data: { name: step.name, __flowParentId: parentId, ...step.data },
-            priority: merged.priority,
-            delay: merged.delay,
-            maxAttempts: merged.attempts,
-            backoff: merged.backoff,
-            timeout: merged.timeout,
-            customId: merged.jobId,
-            removeOnComplete: merged.removeOnComplete,
-            removeOnFail: merged.removeOnFail,
-            dependsOn: parentId ? [jobId(parentId)] : undefined,
-        });
-        const id = String(job.id);
+        const data = { name: step.name, __flowParentId: parentId, ...step.data };
+        const id = await this.pushJob(step.queueName, data, merged, parentId ? [parentId] : undefined);
         jobIds.push(id);
         // Create children with this job as parent
         if (step.children) {
@@ -143,18 +573,58 @@ export class FlowProducer {
         }
         return id;
     }
+    /**
+     * Cleanup jobs that were created before a failure occurred.
+     * Cancels each job to prevent orphaned jobs in the queue.
+     */
+    async cleanupJobs(jobIds) {
+        if (jobIds.length === 0)
+            return;
+        if (this.embedded) {
+            const manager = getSharedManager();
+            const cleanupPromises = jobIds.map(async (id) => {
+                try {
+                    await manager.cancel(jobId(id));
+                }
+                catch {
+                    // Ignore errors during cleanup - job may already be processed or removed
+                }
+            });
+            await Promise.all(cleanupPromises);
+        }
+        else if (this.tcp) {
+            // TCP mode cleanup
+            const cleanupPromises = jobIds.map(async (id) => {
+                try {
+                    await this.tcp?.send({ cmd: 'Cancel', id });
+                }
+                catch {
+                    // Ignore errors during cleanup
+                }
+            });
+            await Promise.all(cleanupPromises);
+        }
+    }
     /**
      * Get the result of a completed parent job.
      * Call this from within a worker to access the previous step's result.
+     * Note: Only works in embedded mode.
      */
     getParentResult(parentId) {
+        if (!this.embedded) {
+            throw new Error('getParentResult is only available in embedded mode');
+        }
         const manager = getSharedManager();
         return manager.getResult(jobId(parentId));
     }
     /**
      * Get results from multiple parent jobs (for merge scenarios).
+     * Note: Only works in embedded mode.
      */
     getParentResults(parentIds) {
+        if (!this.embedded) {
+            throw new Error('getParentResults is only available in embedded mode');
+        }
         const manager = getSharedManager();
         const results = new Map();
         for (const id of parentIds) {