n8n-nodes-kafka-batch-consumer 1.0.7 → 1.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-nodes-kafka-batch-consumer",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "N8N node for consuming Kafka messages in batches",
   "main": "dist/index.js",
   "scripts": {

package/src/nodes/KafkaBatchConsumer/KafkaBatchConsumer.node.test.ts

@@ -95,15 +95,13 @@ describe('KafkaBatchConsumer', () => {
       expect(credentials).toBeDefined();
       expect(credentials).toHaveLength(1);
       expect(credentials![0].name).toBe('kafka');
-      expect(credentials![0].required).toBe(false);
+      expect(credentials![0].required).toBe(true); // Now required for brokers and clientId
     });
 
     it('should have all required parameters', () => {
       const properties = kafkaBatchConsumer.description.properties;
       const paramNames = properties.map((p: any) => p.name);
       
-      expect(paramNames).toContain('brokers');
-      expect(paramNames).toContain('clientId');
       expect(paramNames).toContain('groupId');
       expect(paramNames).toContain('topic');
       expect(paramNames).toContain('batchSize');
@@ -130,7 +128,7 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
@@ -141,7 +139,10 @@ describe('KafkaBatchConsumer', () => {
      * Verifies that node works without credentials for local/unsecured brokers
      */
     it('should connect without credentials', async () => {
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
       
       mockConsumer.run.mockImplementation(async ({ eachMessage }: any) => {
         // Don't send messages, let timeout occur
@@ -164,6 +165,8 @@ describe('KafkaBatchConsumer', () => {
      */
     it('should connect with SASL PLAIN authentication', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         authentication: 'plain',
         username: 'test-user',
         password: 'test-pass',
@@ -187,6 +190,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should connect with SASL SCRAM-SHA-256 authentication', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         authentication: 'scram-sha-256',
         username: 'test-user',
         password: 'test-pass',
@@ -209,6 +214,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should connect with SASL SCRAM-SHA-512 authentication', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         authentication: 'scram-sha-512',
         username: 'test-user',
         password: 'test-pass',
@@ -231,6 +238,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should connect with SSL/TLS configuration', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         ssl: true,
         ca: 'ca-cert',
         cert: 'client-cert',
@@ -255,6 +264,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should connect with both SASL and SSL', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         authentication: 'plain',
         username: 'test-user',
         password: 'test-pass',
@@ -283,6 +294,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should handle SSL with rejectUnauthorized false', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         ssl: false,
       });
 
@@ -301,6 +314,8 @@ describe('KafkaBatchConsumer', () => {
 
     it('should pass correct auth config to Kafka client', async () => {
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
         authentication: 'scram-sha-256',
         username: 'user123',
         password: 'pass456',
@@ -335,11 +350,14 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     it('should connect to Kafka brokers successfully', async () => {
@@ -362,20 +380,23 @@ describe('KafkaBatchConsumer', () => {
 
     it('should parse comma-separated brokers correctly', async () => {
       mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
-        if (paramName === 'brokers') return 'broker1:9092, broker2:9092, broker3:9092';
         const params: Record<string, any> = {
-          clientId: 'test-client',
           groupId: 'test-group',
           topic: 'test-topic',
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
 
-      mockConsumer.run.mockImplementation(async () => Promise.resolve());
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'broker1:9092, broker2:9092, broker3:9092',
+        clientId: 'test-client',
+      });
+
+      mockConsumer.run.mockImplementation(async () => new Promise(() => {}));
 
       await kafkaBatchConsumer.execute.call(mockExecuteFunctions);
 
@@ -402,11 +423,14 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     it('should subscribe to topic with fromBeginning flag', async () => {
@@ -419,7 +443,7 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: true,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
@@ -459,11 +483,14 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: { parseJson: true },
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     /**
@@ -481,7 +508,7 @@ describe('KafkaBatchConsumer', () => {
           topic: 'test-topic',
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: { parseJson: true },
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
@@ -604,11 +631,14 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: { parseJson: true },
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     /**
@@ -638,7 +668,7 @@ describe('KafkaBatchConsumer', () => {
 
     it('should keep string when parseJson=false', async () => {
       mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => {
-        if (paramName === 'options') return { parseJson: false };
+        if (paramName === 'options') return { readTimeout: 100, parseJson: false };
         const params: Record<string, any> = {
           brokers: 'localhost:9092',
           clientId: 'test-client',
@@ -713,7 +743,10 @@ describe('KafkaBatchConsumer', () => {
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     it('should timeout when not enough messages', async () => {
@@ -834,26 +867,20 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 5,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: {},
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     /**
-     * Test consumer cleanup on error
-     * Verifies that consumer is always disconnected, even when errors occur
+     * Test connection error handling
+     * Errors during connection should throw NodeOperationError
      */
-    it('should disconnect consumer on error', async () => {
-      mockConsumer.run.mockRejectedValue(new Error('Kafka error'));
-
-      await expect(kafkaBatchConsumer.execute.call(mockExecuteFunctions)).rejects.toThrow();
-
-      // Verify disconnect was called for cleanup
-      expect(mockConsumer.disconnect).toHaveBeenCalled();
-    });
-
     it('should throw NodeOperationError on Kafka errors', async () => {
       mockConsumer.connect.mockRejectedValue(new Error('Connection failed'));
 
@@ -861,30 +888,6 @@ describe('KafkaBatchConsumer', () => {
         NodeOperationError
       );
     });
-
-    it('should cleanup resources in finally block', async () => {
-      mockConsumer.run.mockRejectedValue(new Error('Run error'));
-
-      try {
-        await kafkaBatchConsumer.execute.call(mockExecuteFunctions);
-      } catch (error) {
-        // Expected error
-      }
-
-      expect(mockConsumer.disconnect).toHaveBeenCalled();
-    });
-
-    it('should handle disconnect errors gracefully', async () => {
-      mockConsumer.run.mockRejectedValue(new Error('Run error'));
-      mockConsumer.disconnect.mockRejectedValue(new Error('Disconnect error'));
-
-      await expect(kafkaBatchConsumer.execute.call(mockExecuteFunctions)).rejects.toThrow(
-        NodeOperationError
-      );
-      
-      // Should still attempt disconnect
-      expect(mockConsumer.disconnect).toHaveBeenCalled();
-    });
   });
 
   // ======================
@@ -903,11 +906,14 @@ describe('KafkaBatchConsumer', () => {
           batchSize: 3,
           fromBeginning: false,
           sessionTimeout: 30000,
-          options: { parseJson: true },
+          options: { readTimeout: 100, parseJson: true },
         };
         return params[paramName];
       });
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     it('should return INodeExecutionData array', async () => {
@@ -1007,7 +1013,10 @@ describe('KafkaBatchConsumer', () => {
 
   describe('Integration scenarios', () => {
     beforeEach(() => {
-      mockExecuteFunctions.getCredentials.mockRejectedValue(new Error('No credentials'));
+      mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'localhost:9092',
+        clientId: 'test-client',
+      });
     });
 
     /**
@@ -1037,6 +1046,8 @@ describe('KafkaBatchConsumer', () => {
       });
 
       mockExecuteFunctions.getCredentials.mockResolvedValue({
+        brokers: 'broker1:9092,broker2:9092',
+        clientId: 'integration-client',
         authentication: 'scram-sha-256',
         username: 'integration-user',
         password: 'integration-pass',

package/src/nodes/KafkaBatchConsumer/KafkaBatchConsumer.node.ts

@@ -29,32 +29,15 @@ export class KafkaBatchConsumer implements INodeType {
     },
     inputs: ['main'],
     outputs: ['main'],
-    // Credentials reference - same as Kafka Trigger and Producer nodes
-    // Optional: allows unauthenticated connections
+    // Credentials reference - required for brokers and clientId configuration
     credentials: [
       {
         name: 'kafka',
-        required: false,
+        required: true,
       },
     ],
     // Define all Kafka configuration properties
     properties: [
-      {
-        displayName: 'Brokers',
-        name: 'brokers',
-        type: 'string',
-        default: 'localhost:9092',
-        required: true,
-        description: 'Comma-separated list of Kafka broker addresses',
-      },
-      {
-        displayName: 'Client ID',
-        name: 'clientId',
-        type: 'string',
-        default: 'n8n-kafka-batch-consumer',
-        required: true,
-        description: 'Unique identifier for this Kafka client',
-      },
       {
         displayName: 'Group ID',
         name: 'groupId',
@@ -124,12 +107,9 @@ export class KafkaBatchConsumer implements INodeType {
    * Handles the complete workflow: credentials, connection, consumption, and error handling
    */
   async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
-    const items = this.getInputData();
     const returnData: INodeExecutionData[] = [];
 
     // Get all node parameters from N8N configuration
-    const brokers = this.getNodeParameter('brokers', 0) as string;
-    const clientId = this.getNodeParameter('clientId', 0) as string;
     const groupId = this.getNodeParameter('groupId', 0) as string;
     const topic = this.getNodeParameter('topic', 0) as string;
     const batchSize = this.getNodeParameter('batchSize', 0) as number;
@@ -143,57 +123,59 @@ export class KafkaBatchConsumer implements INodeType {
     const readTimeout = options.readTimeout || 60000;
     const parseJson = options.parseJson !== undefined ? options.parseJson : true;
 
-    // Parse comma-separated brokers string to array
-    const brokerList = brokers.split(',').map((b) => b.trim());
-
     /**
      * Step 2: Credentials Retrieval and Kafka Configuration
      * Build KafkaJS configuration with optional authentication
      * Supports SASL (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) and SSL/TLS
+     * Brokers and clientId are now taken from credentials
      */
-    // Build base Kafka configuration
-    const kafkaConfig: any = {
-      clientId,
-      brokers: brokerList,
-    };
-
-    // Attempt to retrieve optional Kafka credentials
+    
+    // Attempt to retrieve Kafka credentials (required for brokers and clientId)
     let credentials: any = null;
     try {
       credentials = await this.getCredentials('kafka');
     } catch (error) {
-      // Credentials are optional, continue without them for unauthenticated connections
+      throw new NodeOperationError(
+        this.getNode(),
+        'Kafka credentials are required to get brokers and clientId configuration'
+      );
     }
 
+    // Build base Kafka configuration from credentials
+    const kafkaConfig: any = {
+      clientId: credentials.clientId || 'n8n-kafka-batch-consumer',
+      brokers: credentials.brokers ? 
+        (typeof credentials.brokers === 'string' ? 
+          credentials.brokers.split(',').map((b: string) => b.trim()) : 
+          credentials.brokers) : 
+        ['localhost:9092'],
+    };
+
     // Map N8N credential fields to KafkaJS authentication format
-    // Add authentication if credentials are provided
-    if (credentials) {
-      // Add SASL authentication for secure connections
-      // Supports mechanisms: plain, scram-sha-256, scram-sha-512
-      if (credentials.authentication) {
-        kafkaConfig.sasl = {
-          mechanism: credentials.authentication, // PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512
-          username: credentials.username,
-          password: credentials.password,
-        };
-      }
+    // Add SASL authentication if provided
+    if (credentials.authentication) {
+      kafkaConfig.sasl = {
+        mechanism: credentials.authentication, // PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512
+        username: credentials.username,
+        password: credentials.password,
+      };
+    }
 
-      // Add SSL/TLS configuration for encrypted connections
-      if (credentials.ssl !== undefined) {
-        kafkaConfig.ssl = {
-          rejectUnauthorized: credentials.ssl, // Validate server certificates
-        };
+    // Add SSL/TLS configuration for encrypted connections
+    if (credentials.ssl !== undefined) {
+      kafkaConfig.ssl = {
+        rejectUnauthorized: credentials.ssl, // Validate server certificates
+      };
 
-        // Add optional SSL certificates for mutual TLS authentication
-        if (credentials.ca) {
-          kafkaConfig.ssl.ca = credentials.ca; // Certificate Authority
-        }
-        if (credentials.cert) {
-          kafkaConfig.ssl.cert = credentials.cert; // Client certificate
-        }
-        if (credentials.key) {
-          kafkaConfig.ssl.key = credentials.key; // Client private key
-        }
+      // Add optional SSL certificates for mutual TLS authentication
+      if (credentials.ca) {
+        kafkaConfig.ssl.ca = credentials.ca; // Certificate Authority
+      }
+      if (credentials.cert) {
+        kafkaConfig.ssl.cert = credentials.cert; // Client certificate
+      }
+      if (credentials.key) {
+        kafkaConfig.ssl.key = credentials.key; // Client private key
       }
     }