@platformatic/rdkafka 4.0.0

package/e2e/consumer.spec.js

@@ -0,0 +1,386 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var t = require('assert');
+var crypto = require('crypto');
+
+var eventListener = require('./listener');
+
+var KafkaConsumer = require('../').KafkaConsumer;
+
+var kafkaBrokerList = process.env.KAFKA_HOST || 'localhost:9092';
+var topic = 'test';
+
+describe('Consumer', function() {
+  var gcfg;
+
+  beforeEach(function() {
+    var grp = 'kafka-mocha-grp-' + crypto.randomBytes(20).toString('hex');
+     gcfg = {
+      'bootstrap.servers': kafkaBrokerList,
+      'group.id': grp,
+      'debug': 'all',
+      'rebalance_cb': true,
+      'enable.auto.commit': false
+    };
+  });
+
+  describe('commit', function() {
+    var consumer;
+    beforeEach(function(done) {
+      consumer = new KafkaConsumer(gcfg, {});
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(consumer);
+    });
+
+    it('should allow commit with an array', function(done) {
+      consumer.commit([{ topic: topic, partition: 0, offset: -1 }]);
+      done();
+    });
+
+    it('should allow commit without an array', function(done) {
+      consumer.commit({ topic: topic, partition: 0, offset: -1 });
+      done();
+    });
+
+    afterEach(function(done) {
+      consumer.disconnect(function() {
+        done();
+      });
+    });
+  });
+
+  describe('committed and position', function() {
+    var consumer;
+    beforeEach(function(done) {
+      consumer = new KafkaConsumer(gcfg, {});
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(consumer);
+    });
+
+    afterEach(function(done) {
+      consumer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('before assign, committed offsets are empty', function(done) {
+      consumer.committed(null, 1000, function(err, committed) {
+        t.ifError(err);
+        t.equal(Array.isArray(committed), true, 'Committed offsets should be an array');
+        t.equal(committed.length, 0);
+        done();
+      });
+    });
+
+    it('before assign, position returns an empty array', function() {
+      var position = consumer.position();
+      t.equal(Array.isArray(position), true, 'Position should be an array');
+      t.equal(position.length, 0);
+    });
+
+    it('after assign, should get committed array without offsets ', function(done) {
+      consumer.assign([{topic:topic, partition:0}]);
+      // Defer this for a second
+      setTimeout(function() {
+        consumer.committed(null, 1000, function(err, committed) {
+          t.ifError(err);
+          t.equal(committed.length, 1);
+          t.equal(typeof committed[0], 'object', 'TopicPartition should be an object');
+          t.deepStrictEqual(committed[0].partition, 0);
+          t.equal(committed[0].offset, undefined);
+          done();
+        });
+      }, 1000);
+    });
+
+    it('after assign and commit, should get committed offsets', function(done) {
+      consumer.assign([{topic:topic, partition:0}]);
+      consumer.commitSync({topic:topic, partition:0, offset:1000});
+      consumer.committed(null, 1000, function(err, committed) {
+        t.ifError(err);
+        t.equal(committed.length, 1);
+        t.equal(typeof committed[0], 'object', 'TopicPartition should be an object');
+        t.deepStrictEqual(committed[0].partition, 0);
+        t.deepStrictEqual(committed[0].offset, 1000);
+        done();
+      });
+    });
+
+    it('after assign, before consume, position should return an array without offsets', function(done) {
+      consumer.assign([{topic:topic, partition:0}]);
+      var position = consumer.position();
+      t.equal(Array.isArray(position), true, 'Position should be an array');
+      t.equal(position.length, 1);
+      t.equal(typeof position[0], 'object', 'TopicPartition should be an object');
+      t.deepStrictEqual(position[0].partition, 0);
+      t.equal(position[0].offset, undefined, 'before consuming, offset is undefined');
+      // see both.spec.js 'should be able to produce, consume messages, read position...'
+      // for checking of offset numeric value
+      done();
+    });
+
+    it('should obey the timeout', function(done) {
+      consumer.committed(null, 0, function(err, committed) {
+        if (!err) {
+          t.fail(err, 'not null', 'Error should be set for a timeout');
+        }
+        done();
+      });
+    });
+
+  });
+
+  describe('seek and positioning', function() {
+    var consumer;
+    beforeEach(function(done) {
+      consumer = new KafkaConsumer(gcfg, {});
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+        consumer.assign([{
+          topic: 'test',
+          partition: 0,
+          offset: 0
+        }]);
+        done();
+      });
+
+      eventListener(consumer);
+    });
+
+    afterEach(function(done) {
+      consumer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('should be able to seek', function(cb) {
+      consumer.seek({
+        topic: 'test',
+        partition: 0,
+        offset: 0
+      }, 1, function(err) {
+        t.ifError(err);
+        cb();
+      });
+    });
+
+    it('should be able to seek with a timeout of 0', function(cb) {
+      consumer.seek({
+        topic: 'test',
+        partition: 0,
+        offset: 0
+      }, 0, function(err) {
+        t.ifError(err);
+        cb();
+      });
+    });
+  });
+
+  describe('subscribe', function() {
+
+    var consumer;
+    beforeEach(function(done) {
+      consumer = new KafkaConsumer(gcfg, {});
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(consumer);
+    });
+
+    afterEach(function(done) {
+      consumer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('should be able to subscribe', function() {
+      t.equal(0, consumer.subscription().length);
+      consumer.subscribe([topic]);
+      t.equal(1, consumer.subscription().length);
+      t.equal('test', consumer.subscription()[0]);
+      t.equal(0, consumer.assignments().length);
+    });
+
+    it('should be able to unsubscribe', function() {
+      consumer.subscribe([topic]);
+      t.equal(1, consumer.subscription().length);
+      consumer.unsubscribe();
+      t.equal(0, consumer.subscription().length);
+      t.equal(0, consumer.assignments().length);
+    });
+  });
+
+  describe('assign', function() {
+
+    var consumer;
+    beforeEach(function(done) {
+      consumer = new KafkaConsumer(gcfg, {});
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(consumer);
+    });
+
+    afterEach(function(done) {
+      consumer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('should be able to take an assignment', function() {
+      t.equal(0, consumer.assignments().length);
+      consumer.assign([{ topic:topic, partition:0 }]);
+      t.equal(1, consumer.assignments().length);
+      t.equal(topic, consumer.assignments()[0].topic);
+      t.equal(0, consumer.subscription().length);
+    });
+
+    it('should be able to take an empty assignment', function() {
+      consumer.assign([{ topic:topic, partition:0 }]);
+      t.equal(1, consumer.assignments().length);
+      consumer.assign([]);
+      t.equal(0, consumer.assignments().length);
+    });
+  });
+
+  describe('disconnect', function() {
+    var tcfg = { 'auto.offset.reset': 'earliest' };
+
+    it('should happen gracefully', function(cb) {
+      var consumer = new KafkaConsumer(gcfg, tcfg);
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+
+        consumer.disconnect(function() {
+          cb();
+        });
+
+      });
+
+    });
+
+    it('should happen without issue after subscribing', function(cb) {
+      var consumer = new KafkaConsumer(gcfg, tcfg);
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+
+        consumer.subscribe([topic]);
+
+        consumer.disconnect(function() {
+          cb();
+        });
+
+      });
+
+    });
+
+    it('should happen without issue after consuming', function(cb) {
+      var consumer = new KafkaConsumer(gcfg, tcfg);
+      consumer.setDefaultConsumeTimeout(10000);
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+
+        consumer.subscribe([topic]);
+
+        consumer.consume(1, function(err, messages) {
+          t.ifError(err);
+
+          consumer.disconnect(function() {
+            cb();
+          });
+        });
+
+      });
+
+    });
+
+    it('should happen without issue after consuming an error', function(cb) {
+      var consumer = new KafkaConsumer(gcfg, tcfg);
+
+      consumer.setDefaultConsumeTimeout(1);
+
+      consumer.connect({ timeout: 2000 }, function(err, info) {
+        t.ifError(err);
+
+        consumer.subscribe([topic]);
+
+        consumer.consume(1, function(err, messages) {
+
+          // Timeouts do not classify as errors anymore
+          t.equal(messages[0], undefined, 'Message should not be set');
+
+          consumer.disconnect(function() {
+            cb();
+          });
+        });
+
+      });
+    });
+
+  });
+
+  describe('rebalance protocol', function () {
+    var strategies = {
+      'undefined': 'EAGER',
+      'range': 'EAGER',
+      'roundrobin': 'EAGER',
+      'cooperative-sticky': 'COOPERATIVE',
+    };
+
+    Object.keys(strategies).forEach(function (strategy) {
+      it('should return ' + strategies[strategy] + ' for ' + strategy, function(done) {
+        var consumer = new KafkaConsumer({
+          ...gcfg,
+          ...(strategy !== 'undefined' && { 'partition.assignment.strategy': strategy })
+        }, {});
+
+        t.equal(consumer.rebalanceProtocol(), 'NONE');
+  
+        consumer.connect({ timeout: 2000 }, function(err) {
+          t.ifError(err);
+  
+          consumer.subscribe([topic]);
+  
+          consumer.on('rebalance', function (err) {
+            if (err.code === -175) {
+              t.equal(consumer.rebalanceProtocol(), strategies[strategy]);
+              consumer.disconnect(done);
+            }
+          });
+  
+          consumer.consume(1, function(err) {
+            t.ifError(err);
+          });
+        });
+  
+        eventListener(consumer);
+      });
+    });
+  });
+});

package/e2e/groups.spec.js

@@ -0,0 +1,132 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var crypto = require('crypto');
+var t = require('assert');
+
+var Kafka = require('../');
+var kafkaBrokerList = process.env.KAFKA_HOST || 'localhost:9092';
+var eventListener = require('./listener');
+
+describe('Consumer group/Producer', function() {
+
+  var producer;
+  var consumer;
+  var grp = 'kafka-mocha-grp-' + crypto.randomBytes(20).toString('hex');
+
+  var config = {
+    'metadata.broker.list': kafkaBrokerList,
+    'group.id': grp,
+    'fetch.wait.max.ms': 1000,
+    'session.timeout.ms': 10000,
+    'enable.auto.commit': false,
+    'debug': 'all'
+  };
+
+  beforeEach(function(done) {
+    producer = new Kafka.Producer({
+      'client.id': 'kafka-mocha',
+      'metadata.broker.list': kafkaBrokerList,
+      'fetch.wait.max.ms': 1,
+      'debug': 'all',
+      'dr_cb': true
+    });
+
+    producer.connect({}, function(err, d) {
+      t.ifError(err);
+      t.equal(typeof d, 'object', 'metadata should be returned');
+      done();
+    });
+
+    eventListener(producer);
+  });
+
+  beforeEach(function(done) {
+    consumer = new Kafka.KafkaConsumer(config, {
+      'auto.offset.reset': 'largest'
+    });
+
+    consumer.connect({}, function(err, d) {
+      t.ifError(err);
+      t.equal(typeof d, 'object', 'metadata should be returned');
+      done();
+    });
+
+    eventListener(consumer);
+  });
+
+  afterEach(function(done) {
+    producer.disconnect(function() {
+      done();
+    });
+  });
+
+  it('should be able to commit, read committed and restart from the committed offset', function(done) {
+    var topic = 'test';
+    var key = 'key';
+    var payload = Buffer.from('value');
+    var count = 0;
+    var offsets = {
+      'first': true
+    };
+
+    var tt = setInterval(function() {
+      try {
+        producer.produce(topic, null, payload, key);
+      } catch (e) {
+        clearInterval(tt);
+      }
+    }, 100);
+
+    consumer.on('disconnected', function() {
+      var consumer2 = new Kafka.KafkaConsumer(config, {
+        'auto.offset.reset': 'largest'
+      });
+
+      consumer2.on('data', function(message) {
+        if (offsets.first) {
+          offsets.first = false;
+          t.deepStrictEqual(offsets.committed, message.offset, 'Offset read by consumer 2 incorrect');
+          clearInterval(tt);
+          consumer2.unsubscribe();
+          consumer2.disconnect(function() {
+            done();
+          });
+        }
+      });
+
+      consumer2.on('ready', function() {
+        consumer2.subscribe([topic]);
+        consumer2.consume();
+      });
+      consumer2.connect();
+    });
+
+    consumer.on('data', function(message) {
+      count++;
+      if (count === 3) {
+        consumer.commitMessageSync(message);
+        // test consumer.committed( ) API
+        consumer.committed(null, 5000, function(err, topicPartitions) {
+          t.ifError(err);
+          t.deepStrictEqual(topicPartitions.length, 1);
+          t.deepStrictEqual(topicPartitions[0].offset, message.offset + 1, 'Offset read by consumer 1 incorrect');
+          offsets.committed = message.offset + 1;
+          consumer.unsubscribe();
+          consumer.disconnect();
+        });
+
+      }
+    });
+
+    consumer.subscribe([topic]);
+    consumer.consume();
+  });
+
+});

package/e2e/listener.js

@@ -0,0 +1,51 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = eventListener;
+
+function eventListener(client) {
+  if (!process.env.DEBUG) {
+    return;
+  }
+
+  client
+    .on('event.error', function (err) {
+      console.error(err);
+    })
+    .on('event.log', function(event) {
+      var info = {
+        severity: event.severity,
+        fac: event.fac,
+      };
+      if (event.severity >= 7) {
+        console.error(info, event.message);
+      } else if (event.severity === 6 || event.severity === 5) {
+        console.error(info, event.message);
+      } else if (event.severity === 4) {
+        console.error(info, event.message);
+      } else if (event.severity > 0) {
+        console.error(info, event.message);
+      } else {
+        console.error(info, event.message);
+      }
+    })
+    .on('event.stats', function(event) {
+      console.log(event, event.message);
+    })
+    .on('event.throttle', function(event) {
+      console.log(event, '%s#%d throttled.', event.brokerName, event.brokerId);
+      // event.throttleTime;
+    })
+    .on('event.event', function(event) {
+      console.log(event, event.message);
+    })
+    .on('ready', function(info) {
+      console.log('%s connected to kafka server', info.name);
+    });
+
+}