@onlineapps/conn-infra-mq 1.1.0

package/src/layers/ForkJoinHandler.js

@@ -0,0 +1,312 @@
+'use strict';
+
+/**
+ * ForkJoinHandler - Manages fork-join parallel processing patterns
+ * Handles accumulator queues and result collection from parallel branches
+ */
+class ForkJoinHandler {
+  constructor(mqClient, queueManager, config = {}) {
+    this.client = mqClient;
+    this.queueManager = queueManager;
+    this.config = {
+      defaultTimeout: config.defaultTimeout || 30000,
+      accumulatorPrefix: config.accumulatorPrefix || 'fork',
+      ...config
+    };
+    this.activeAccumulators = new Map();
+  }
+
+  /**
+   * Create accumulator queue for collecting fork-join results
+   * @param {string} workflowId - Workflow identifier
+   * @param {string} stepId - Step identifier
+   * @param {number} expectedCount - Number of expected results
+   */
+  async createAccumulator(workflowId, stepId, expectedCount) {
+    const queueName = `${this.config.accumulatorPrefix}.${workflowId}.${stepId}.accumulator`;
+
+    // Create temporary queue for accumulation
+    await this.queueManager.ensureQueue(queueName, {
+      durable: false,
+      autoDelete: true,
+      expires: this.config.defaultTimeout * 2, // Double the timeout for safety
+      exclusive: false
+    });
+
+    // Track accumulator
+    this.activeAccumulators.set(queueName, {
+      workflowId,
+      stepId,
+      expectedCount,
+      collectedCount: 0,
+      results: [],
+      createdAt: Date.now()
+    });
+
+    return queueName;
+  }
+
+  /**
+   * Fork - distribute work to multiple queues
+   * @param {Array} branches - Array of { queue, message } objects
+   * @param {Object} context - Common context for all branches
+   */
+  async fork(branches, context = {}) {
+    const forkId = `fork-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+    const promises = [];
+
+    for (let i = 0; i < branches.length; i++) {
+      const branch = branches[i];
+      const message = {
+        ...branch.message,
+        _fork: {
+          id: forkId,
+          branch: i,
+          total: branches.length,
+          context
+        }
+      };
+
+      promises.push(
+        this.client.publish(message, {
+          queue: branch.queue,
+          ...branch.options
+        })
+      );
+    }
+
+    await Promise.all(promises);
+    return forkId;
+  }
+
+  /**
+   * Collect results from accumulator queue
+   * @param {string} queueName - Accumulator queue name
+   * @param {Function} onComplete - Callback when all results collected
+   * @param {Object} options - Collection options
+   */
+  async collectResults(queueName, onComplete, options = {}) {
+    const accumulator = this.activeAccumulators.get(queueName);
+    if (!accumulator) {
+      throw new Error(`Accumulator ${queueName} not found`);
+    }
+
+    const timeout = options.timeout || this.config.defaultTimeout;
+    const timeoutHandle = setTimeout(() => {
+      this.handleTimeout(queueName, onComplete);
+    }, timeout);
+
+    // Consume with prefetch 1 for atomic result collection
+    await this.client.consume(async (result, rawMsg) => {
+      accumulator.results.push(result);
+      accumulator.collectedCount++;
+
+      // Acknowledge immediately
+      await this.client.ack(rawMsg);
+
+      // Check if all results collected
+      if (accumulator.collectedCount >= accumulator.expectedCount) {
+        clearTimeout(timeoutHandle);
+        await this.handleComplete(queueName, onComplete);
+      }
+    }, {
+      queue: queueName,
+      prefetch: 1,
+      noAck: false
+    });
+
+    return {
+      queueName,
+      expectedCount: accumulator.expectedCount
+    };
+  }
+
+  /**
+   * Join - wait for all results and combine them
+   * @param {string} queueName - Accumulator queue name
+   * @param {Function} joinStrategy - Function to combine results
+   * @param {Object} options - Join options
+   */
+  async join(queueName, joinStrategy, options = {}) {
+    return new Promise((resolve, reject) => {
+      const timeout = options.timeout || this.config.defaultTimeout;
+
+      this.collectResults(queueName, async (error, results) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+
+        try {
+          const joined = await joinStrategy(results);
+          resolve(joined);
+        } catch (err) {
+          reject(err);
+        }
+      }, { timeout });
+    });
+  }
+
+  /**
+   * Fork-join convenience method
+   * @param {Array} branches - Branches to fork to
+   * @param {Function} joinStrategy - How to combine results
+   * @param {Object} options - Fork-join options
+   */
+  async forkJoin(branches, joinStrategy, options = {}) {
+    const workflowId = options.workflowId || `wf-${Date.now()}`;
+    const stepId = options.stepId || 'step-1';
+
+    // Create accumulator
+    const accumulatorQueue = await this.createAccumulator(
+      workflowId,
+      stepId,
+      branches.length
+    );
+
+    // Modify branches to send results to accumulator
+    const modifiedBranches = branches.map(branch => ({
+      ...branch,
+      message: {
+        ...branch.message,
+        _replyTo: accumulatorQueue
+      }
+    }));
+
+    // Fork work
+    const forkId = await this.fork(modifiedBranches);
+
+    // Join results
+    const result = await this.join(accumulatorQueue, joinStrategy, options);
+
+    // Cleanup
+    await this.cleanupAccumulator(accumulatorQueue);
+
+    return {
+      forkId,
+      result,
+      branchCount: branches.length
+    };
+  }
+
+  /**
+   * Handle timeout for accumulator
+   * @private
+   */
+  handleTimeout(queueName, callback) {
+    const accumulator = this.activeAccumulators.get(queueName);
+    if (accumulator) {
+      const error = new Error(
+        `Timeout waiting for fork-join results. Received ${accumulator.collectedCount}/${accumulator.expectedCount}`
+      );
+      error.partialResults = accumulator.results;
+      callback(error, null);
+      this.cleanupAccumulator(queueName).catch(() => {});
+    }
+  }
+
+  /**
+   * Handle completion of result collection
+   * @private
+   */
+  async handleComplete(queueName, callback) {
+    const accumulator = this.activeAccumulators.get(queueName);
+    if (accumulator) {
+      callback(null, accumulator.results);
+      await this.cleanupAccumulator(queueName);
+    }
+  }
+
+  /**
+   * Cleanup accumulator queue and tracking
+   * @param {string} queueName - Accumulator queue name
+   */
+  async cleanupAccumulator(queueName) {
+    this.activeAccumulators.delete(queueName);
+
+    try {
+      await this.queueManager.deleteQueue(queueName, { ifEmpty: true });
+    } catch (error) {
+      // Queue might already be deleted or have messages
+      console.warn(`Failed to delete accumulator queue ${queueName}:`, error.message);
+    }
+  }
+
+  /**
+   * Built-in join strategies
+   */
+  static JoinStrategies = {
+    /**
+     * Merge all results into single object
+     */
+    merge: (results) => {
+      return results.reduce((acc, result) => ({ ...acc, ...result }), {});
+    },
+
+    /**
+     * Concatenate array results
+     */
+    concat: (results) => {
+      return results.reduce((acc, result) => {
+        if (Array.isArray(result)) {
+          return acc.concat(result);
+        }
+        acc.push(result);
+        return acc;
+      }, []);
+    },
+
+    /**
+     * Return first result
+     */
+    first: (results) => results[0],
+
+    /**
+     * Return last result
+     */
+    last: (results) => results[results.length - 1],
+
+    /**
+     * Return all results as array
+     */
+    all: (results) => results,
+
+    /**
+     * Sum numeric results
+     */
+    sum: (results) => {
+      return results.reduce((acc, result) => {
+        if (typeof result === 'number') {
+          return acc + result;
+        }
+        if (result && typeof result.value === 'number') {
+          return acc + result.value;
+        }
+        return acc;
+      }, 0);
+    }
+  };
+
+  /**
+   * Get active accumulators
+   */
+  getActiveAccumulators() {
+    return Array.from(this.activeAccumulators.entries()).map(([queue, data]) => ({
+      queue,
+      ...data
+    }));
+  }
+
+  /**
+   * Cleanup all active accumulators
+   */
+  async cleanupAll() {
+    const promises = [];
+    for (const queueName of this.activeAccumulators.keys()) {
+      promises.push(this.cleanupAccumulator(queueName));
+    }
+    await Promise.all(promises);
+  }
+}
+
+module.exports = ForkJoinHandler;

package/src/layers/QueueManager.js

@@ -0,0 +1,263 @@
+'use strict';
+
+/**
+ * QueueManager - Manages queue creation, configuration, and lifecycle
+ * Handles TTL, DLQ, auto-delete, and other queue properties
+ */
+class QueueManager {
+  constructor(mqClient, config = {}) {
+    this.client = mqClient;
+    this.config = {
+      defaultTTL: config.defaultTTL || 30000,
+      dlxExchange: config.dlxExchange || 'dlx',
+      autoDeleteTimeout: config.autoDeleteTimeout || 300000, // 5 minutes
+      maxRetries: config.maxRetries || 3,
+      ...config
+    };
+    this.managedQueues = new Set();
+  }
+
+  /**
+   * Create or ensure queue exists with specific configuration
+   * @param {string} queueName - Name of the queue
+   * @param {Object} options - Queue configuration options
+   */
+  async ensureQueue(queueName, options = {}) {
+    // Get channel from client's transport
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    const queueOptions = {
+      durable: options.durable !== false,
+      arguments: {
+        ...options.arguments
+      }
+    };
+
+    // Add TTL if specified
+    if (options.ttl !== null && options.ttl !== undefined) {
+      queueOptions.arguments['x-message-ttl'] = options.ttl || this.config.defaultTTL;
+    }
+
+    // Add DLQ configuration
+    if (options.dlq !== false) {
+      queueOptions.arguments['x-dead-letter-exchange'] = options.dlxExchange || this.config.dlxExchange;
+      queueOptions.arguments['x-dead-letter-routing-key'] = options.dlqKey || `${queueName}.dlq`;
+    }
+
+    // Add auto-delete expiry
+    if (options.autoDelete === true) {
+      queueOptions.arguments['x-expires'] = options.expires || this.config.autoDeleteTimeout;
+    }
+
+    // Add max length if specified
+    if (options.maxLength) {
+      queueOptions.arguments['x-max-length'] = options.maxLength;
+    }
+
+    // Add priority support
+    if (options.maxPriority) {
+      queueOptions.arguments['x-max-priority'] = options.maxPriority;
+    }
+
+    // Track managed queue
+    this.managedQueues.add(queueName);
+
+    // Assert the queue
+    return channel.assertQueue(queueName, queueOptions);
+  }
+
+  /**
+   * Setup service queues (main + DLQ + optional workflow queue)
+   * @param {string} serviceName - Name of the service
+   * @param {Object} options - Configuration options
+   */
+  async setupServiceQueues(serviceName, options = {}) {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    const queues = {};
+
+    // Create main processing queue
+    await this.ensureQueue(`${serviceName}.queue`, {
+      ttl: options.ttl || this.config.defaultTTL,
+      dlq: true,
+      durable: true,
+      maxRetries: this.config.maxRetries
+    });
+    queues.main = `${serviceName}.queue`;
+
+    // Create dead letter queue
+    await this.ensureQueue(`${serviceName}.dlq`, {
+      ttl: null, // No TTL for DLQ
+      dlq: false,
+      durable: true,
+      autoDelete: false
+    });
+    queues.dlq = `${serviceName}.dlq`;
+
+    // Create workflow queue if requested
+    if (options.includeWorkflow !== false) {
+      await this.ensureQueue(`${serviceName}.workflow`, {
+        ttl: options.workflowTTL || this.config.defaultTTL,
+        dlq: true,
+        durable: true
+      });
+      queues.workflow = `${serviceName}.workflow`;
+    }
+
+    // Setup DLQ exchange and bindings
+    await channel.assertExchange(this.config.dlxExchange, 'direct', {
+      durable: true
+    });
+
+    // Bind DLQ
+    await channel.bindQueue(
+      queues.dlq,
+      this.config.dlxExchange,
+      `${serviceName}.queue.dlq`
+    );
+
+    if (queues.workflow) {
+      await channel.bindQueue(
+        queues.dlq,
+        this.config.dlxExchange,
+        `${serviceName}.workflow.dlq`
+      );
+    }
+
+    return queues;
+  }
+
+  /**
+   * Create temporary queue for short-lived operations
+   * @param {string} prefix - Queue name prefix
+   * @param {Object} options - Queue options
+   */
+  async createTemporaryQueue(prefix = 'temp', options = {}) {
+    const queueName = `${prefix}.${Date.now()}.${Math.random().toString(36).substr(2, 9)}`;
+
+    await this.ensureQueue(queueName, {
+      durable: false,
+      autoDelete: true,
+      expires: options.expires || 60000, // 1 minute default
+      exclusive: options.exclusive || false,
+      ...options
+    });
+
+    return queueName;
+  }
+
+  /**
+   * Delete a queue
+   * @param {string} queueName - Name of the queue to delete
+   * @param {Object} options - Delete options
+   */
+  async deleteQueue(queueName, options = {}) {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    this.managedQueues.delete(queueName);
+    return channel.deleteQueue(queueName, options);
+  }
+
+  /**
+   * Purge all messages from a queue
+   * @param {string} queueName - Name of the queue to purge
+   */
+  async purgeQueue(queueName) {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    return channel.purgeQueue(queueName);
+  }
+
+  /**
+   * Get queue statistics
+   * @param {string} queueName - Name of the queue
+   */
+  async getQueueStats(queueName) {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    const result = await channel.checkQueue(queueName);
+    return {
+      name: result.queue,
+      messageCount: result.messageCount,
+      consumerCount: result.consumerCount
+    };
+  }
+
+  /**
+   * Setup exchange
+   * @param {string} exchangeName - Name of the exchange
+   * @param {string} type - Exchange type (direct, topic, fanout, headers)
+   * @param {Object} options - Exchange options
+   */
+  async setupExchange(exchangeName, type = 'direct', options = {}) {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    return channel.assertExchange(exchangeName, type, {
+      durable: options.durable !== false,
+      ...options
+    });
+  }
+
+  /**
+   * Bind queue to exchange
+   * @param {string} queue - Queue name
+   * @param {string} exchange - Exchange name
+   * @param {string} routingKey - Routing key for binding
+   */
+  async bindQueue(queue, exchange, routingKey = '') {
+    const transport = this.client._transport;
+    if (!transport || !transport.channel) {
+      throw new Error('MQ client not connected');
+    }
+    const channel = transport.channel;
+
+    return channel.bindQueue(queue, exchange, routingKey);
+  }
+
+  /**
+   * Cleanup all managed queues
+   */
+  async cleanup() {
+    const promises = [];
+    for (const queueName of this.managedQueues) {
+      if (queueName.startsWith('temp.')) {
+        promises.push(this.deleteQueue(queueName).catch(() => {}));
+      }
+    }
+    await Promise.all(promises);
+    this.managedQueues.clear();
+  }
+
+  /**
+   * Get list of managed queues
+   */
+  getManagedQueues() {
+    return Array.from(this.managedQueues);
+  }
+}
+
+module.exports = QueueManager;