@onlineapps/service-wrapper 2.0.37 → 2.0.38

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onlineapps/service-wrapper",
-  "version": "2.0.37",
+  "version": "2.0.38",
   "description": "Thin orchestration layer for microservices - delegates all infrastructure concerns to specialized connectors",
   "main": "src/index.js",
   "scripts": {
@@ -33,6 +33,7 @@
     "@onlineapps/conn-orch-orchestrator": "^1.0.1",
     "@onlineapps/conn-orch-registry": "^1.1.16",
     "@onlineapps/conn-orch-validator": "^2.0.0",
+    "@onlineapps/infrastructure-tools": "^1.0.3",
     "@onlineapps/monitoring-core": "^1.0.0"
   },
   "devDependencies": {

package/src/ServiceWrapper.js

@@ -330,19 +330,50 @@ class ServiceWrapper {
       // CRITICAL: Start workflow listeners ONLY after successful validation and registration with certificate
       // Both workflow.init and service-specific listeners start here
       if (registrationResult.success && registrationResult.certificate) {
-        this.logger?.info('✓ Certificate validated, starting workflow listeners...');
+        this.logger?.info('✓ Certificate validated, verifying infrastructure readiness...');
+        
+        // PHASE 8: Verify infrastructure services are ready before creating business queues
+        await this._verifyInfrastructureReady(serviceName);
+        
+        this.logger?.info('✓ Infrastructure verified, creating business queues...');
         try {
-          await this.mqClient.setupServiceQueues(serviceName, { includeWorkflow: true });
-          this.logger?.info('✓ Business queues initialized via QueueManager');
+          // CRITICAL: Create ALL business queues atomically (main, dlq, workflow)
+          // This MUST succeed completely - if one queue fails, all fail
+          const createdQueues = await this.mqClient.setupServiceQueues(serviceName, { includeWorkflow: true });
+          this.logger?.info('✓ Business queues initialized via QueueManager', { 
+            queues: Object.keys(createdQueues).map(k => createdQueues[k])
+          });
+          
+          // Verify all queues were created
+          const requiredQueues = ['main', 'dlq', 'workflow'];
+          const missingQueues = requiredQueues.filter(q => !createdQueues[q]);
+          if (missingQueues.length > 0) {
+            throw new Error(`Failed to create all required business queues. Missing: ${missingQueues.join(', ')}`);
+          }
         } catch (queueError) {
-          this.logger?.error('Failed to initialize business queues', { error: queueError.message });
+          this.logger?.error('Failed to initialize business queues', { 
+            error: queueError.message,
+            stack: queueError.stack
+          });
           throw new Error(`Failed to initialize business queues for ${serviceName}: ${queueError.message}`);
         }
 
-        // Start workflow.init listener (for all services)
-        await this._startWorkflowInitListener();
-        // Start service-specific workflow listener
-        await this._startServiceWorkflowListener();
+        // CRITICAL: Start workflow listeners ONLY after all queues are created
+        // Queues must exist before we try to consume from them
+        this.logger?.info('✓ Starting workflow listeners...');
+        try {
+          // Start workflow.init listener (for all services)
+          await this._startWorkflowInitListener();
+          // Start service-specific workflow listener
+          await this._startServiceWorkflowListener();
+          this.logger?.info('✓ All workflow listeners started successfully');
+        } catch (listenerError) {
+          this.logger?.error('Failed to start workflow listeners', {
+            error: listenerError.message,
+            stack: listenerError.stack
+          });
+          throw new Error(`Failed to start workflow listeners for ${serviceName}: ${listenerError.message}`);
+        }
       } else {
         this.logger?.warn('⚠ Registration succeeded but no certificate received - workflow listeners NOT started');
       }
@@ -368,6 +399,155 @@ class ServiceWrapper {
     }, heartbeatInterval);
   }
 
+  /**
+   * Verify infrastructure services are ready before creating business queues
+   * Phase 8: Business Services Infrastructure Verification
+   * 
+   * Checks:
+   * 1. All infrastructure services are UP (via Redis key infrastructure:health:all)
+   * 2. Required infrastructure queues exist (workflow.init, registry.register, etc.)
+   * 
+   * Uses exponential backoff retry mechanism if infrastructure not ready.
+   * @private
+   */
+  async _verifyInfrastructureReady(serviceName) {
+    const maxRetries = parseInt(process.env.INFRASTRUCTURE_VERIFY_MAX_RETRIES) || 12; // 12 * 5s = 60s max
+    const baseDelay = parseInt(process.env.INFRASTRUCTURE_VERIFY_BASE_DELAY) || 5000; // 5 seconds
+    const maxDelay = parseInt(process.env.INFRASTRUCTURE_VERIFY_MAX_DELAY) || 30000; // 30 seconds max
+    const redisUrl = this.config.wrapper?.cache?.url || process.env.REDIS_URL || 'redis://api_node_cache:6379';
+    
+    // Required infrastructure queues that must exist before business queues can be created
+    const requiredInfrastructureQueues = [
+      'workflow.init',        // Gateway responsibility
+      'registry.register',    // Registry responsibility
+      'registry.heartbeats'  // Registry responsibility
+    ];
+    
+    // Optional infrastructure queues (may not exist if services are not running)
+    const optionalInfrastructureQueues = [
+      'validation.requests',  // Validator responsibility
+      'workflow.completed',  // Delivery Dispatcher responsibility
+      'workflow.failed'      // Delivery Dispatcher responsibility
+    ];
+    
+    this.logger?.info('[InfrastructureVerify] Starting infrastructure readiness verification...', {
+      maxRetries,
+      baseDelay,
+      requiredQueues: requiredInfrastructureQueues
+    });
+    
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        // Step 1: Check infrastructure health via Redis
+        this.logger?.info(`[InfrastructureVerify] Attempt ${attempt}/${maxRetries}: Checking infrastructure health...`);
+        
+        const { waitForInfrastructureReady } = require('@onlineapps/infrastructure-tools');
+        const waitMaxTime = 10000; // 10 seconds per attempt
+        const waitCheckInterval = 2000; // Check every 2 seconds
+        
+        try {
+          await waitForInfrastructureReady({
+            redisUrl: redisUrl,
+            maxWait: waitMaxTime,
+            checkInterval: waitCheckInterval,
+            logger: {
+              log: (msg) => this.logger?.info(`[InfrastructureVerify] ${msg}`)
+            }
+          });
+          this.logger?.info('[InfrastructureVerify] ✓ All infrastructure services are UP');
+        } catch (healthError) {
+          // Infrastructure not ready yet - will retry with exponential backoff
+          const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+          this.logger?.warn(`[InfrastructureVerify] Infrastructure not ready (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`, {
+            error: healthError.message
+          });
+          
+          if (attempt < maxRetries) {
+            await new Promise(resolve => setTimeout(resolve, delay));
+            continue; // Retry
+          } else {
+            throw new Error(`Infrastructure services not ready after ${maxRetries} attempts: ${healthError.message}`);
+          }
+        }
+        
+        // Step 2: Verify required infrastructure queues exist
+        this.logger?.info('[InfrastructureVerify] Verifying required infrastructure queues exist...');
+        
+        const transport = this.mqClient._transport;
+        if (!transport || !transport._queueChannel) {
+          throw new Error('MQ transport not initialized - cannot verify infrastructure queues');
+        }
+        
+        const channel = transport._queueChannel;
+        if (channel.closed) {
+          throw new Error('MQ channel is closed - cannot verify infrastructure queues');
+        }
+        
+        const missingQueues = [];
+        for (const queueName of requiredInfrastructureQueues) {
+          try {
+            await channel.checkQueue(queueName);
+            this.logger?.info(`[InfrastructureVerify] ✓ Queue exists: ${queueName}`);
+          } catch (checkErr) {
+            if (checkErr.code === 404) {
+              missingQueues.push(queueName);
+              this.logger?.warn(`[InfrastructureVerify] ✗ Queue missing: ${queueName}`);
+            } else {
+              // Other error (e.g., channel closed) - treat as missing
+              missingQueues.push(queueName);
+              this.logger?.warn(`[InfrastructureVerify] ✗ Cannot check queue ${queueName}: ${checkErr.message}`);
+            }
+          }
+        }
+        
+        if (missingQueues.length > 0) {
+          const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+          this.logger?.warn(`[InfrastructureVerify] Missing required infrastructure queues (attempt ${attempt}/${maxRetries}): ${missingQueues.join(', ')}. Retrying in ${delay}ms...`);
+          
+          if (attempt < maxRetries) {
+            await new Promise(resolve => setTimeout(resolve, delay));
+            continue; // Retry
+          } else {
+            throw new Error(`Required infrastructure queues missing after ${maxRetries} attempts: ${missingQueues.join(', ')}`);
+          }
+        }
+        
+        // Step 3: Log optional queues status (for debugging)
+        for (const queueName of optionalInfrastructureQueues) {
+          try {
+            await channel.checkQueue(queueName);
+            this.logger?.info(`[InfrastructureVerify] ✓ Optional queue exists: ${queueName}`);
+          } catch (checkErr) {
+            if (checkErr.code === 404) {
+              this.logger?.debug(`[InfrastructureVerify] Optional queue missing (not critical): ${queueName}`);
+            }
+          }
+        }
+        
+        // All checks passed!
+        this.logger?.info('[InfrastructureVerify] ✓ Infrastructure verification complete - all required queues exist');
+        return;
+        
+      } catch (error) {
+        if (attempt === maxRetries) {
+          // Final attempt failed - throw error
+          this.logger?.error('[InfrastructureVerify] ✗ Infrastructure verification failed after all retries', {
+            error: error.message,
+            stack: error.stack
+          });
+          throw new Error(`Infrastructure verification failed for ${serviceName}: ${error.message}`);
+        }
+        
+        // Not final attempt - continue retry loop
+        const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay);
+        this.logger?.warn(`[InfrastructureVerify] Verification failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms...`, {
+          error: error.message
+        });
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
+    }
+  }
+
   /**
    * Initialize cache connector
    * @private
@@ -488,12 +668,25 @@ class ServiceWrapper {
       return;
     }
 
-    // Subscribe to workflow.init queue (for all services)
-    await this.mqClient.consume('workflow.init', async (message) => {
-      await this._processWorkflowMessage(message, 'workflow.init');
-    });
+    const queueName = 'workflow.init';
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Starting workflow.init listener for queue: ${queueName}`);
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Service: ${this.config.service?.name || 'unnamed-service'}`);
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Timestamp: ${new Date().toISOString()}`);
 
-    this.logger?.info('✓ workflow.init listener started (after successful registration)');
+    try {
+      // Subscribe to workflow.init queue (for all services)
+      await this.mqClient.consume(queueName, async (message) => {
+        await this._processWorkflowMessage(message, queueName);
+      });
+
+      this.logger?.info(`[ServiceWrapper] [CONSUMER] ✓ workflow.init listener started successfully for queue: ${queueName}`);
+    } catch (consumeErr) {
+      this.logger?.error(`[ServiceWrapper] [CONSUMER] ✗ Failed to start workflow.init listener for queue: ${queueName}`, {
+        error: consumeErr.message,
+        stack: consumeErr.stack
+      });
+      throw consumeErr;
+    }
   }
 
   /**
@@ -509,12 +702,24 @@ class ServiceWrapper {
     const serviceName = this.config.service?.name || 'unnamed-service';
     const serviceQueue = `${serviceName}.workflow`;
 
-    // Subscribe to service-specific queue
-    await this.mqClient.consume(serviceQueue, async (message) => {
-      await this._processWorkflowMessage(message, serviceQueue);
-    });
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Starting service-specific workflow listener for queue: ${serviceQueue}`);
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Service: ${serviceName}`);
+    this.logger?.info(`[ServiceWrapper] [CONSUMER] Timestamp: ${new Date().toISOString()}`);
 
-    this.logger?.info(`✓ Service-specific workflow listener started: ${serviceQueue}`);
+    try {
+      // Subscribe to service-specific queue
+      await this.mqClient.consume(serviceQueue, async (message) => {
+        await this._processWorkflowMessage(message, serviceQueue);
+      });
+
+      this.logger?.info(`[ServiceWrapper] [CONSUMER] ✓ Service-specific workflow listener started successfully for queue: ${serviceQueue}`);
+    } catch (consumeErr) {
+      this.logger?.error(`[ServiceWrapper] [CONSUMER] ✗ Failed to start service-specific workflow listener for queue: ${serviceQueue}`, {
+        error: consumeErr.message,
+        stack: consumeErr.stack
+      });
+      throw consumeErr;
+    }
   }
 
   /**