@platformatic/rdkafka 4.0.0

package/e2e/producer-transaction.spec.js

@@ -0,0 +1,333 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+
+var kafkaBrokerList = process.env.KAFKA_HOST || 'localhost:9092';
+
+describe('Transactional Producer', function () {
+  var TRANSACTIONS_TIMEOUT_MS = 30000;
+  var r = Date.now() + '_' + Math.round(Math.random() * 1000);
+  var topicIn = 'transaction_input_' + r;
+  var topicOut = 'transaction_output_' + r;
+
+  var producerTras;
+  var consumerTrans;
+
+  before(function (done) {
+    /*
+    prepare: 
+    transactional consumer (read from input topic)
+    transactional producer (write to output topic)
+    write 3 messages to input topic: A, B, C
+        A will be skipped, B will be committed, C will be aborted
+    */
+    var connecting = 3;
+    var producerInput;
+    function connectedCb(err) {
+      if (err) {
+        done(err);
+        return;
+      }
+      connecting--;
+      if (connecting === 0) {
+        producerInput.produce(topicIn, -1, Buffer.from('A'));
+        producerInput.produce(topicIn, -1, Buffer.from('B'));
+        producerInput.produce(topicIn, -1, Buffer.from('C'));
+        producerInput.disconnect(function (err) {
+          consumerTrans.subscribe([topicIn]);
+          done(err);
+        })
+      }
+    }
+    producerInput = Kafka.Producer({
+      'client.id': 'kafka-test',
+      'metadata.broker.list': kafkaBrokerList,
+      'enable.idempotence': true
+    });
+    producerInput.setPollInterval(100);
+    producerInput.connect({}, connectedCb);
+
+    producerTras = new Kafka.Producer({
+      'client.id': 'kafka-test',
+      'metadata.broker.list': kafkaBrokerList,
+      'dr_cb': true,
+      'debug': 'all',
+      'transactional.id': 'noderdkafka_transactions_send_offset',
+      'enable.idempotence': true
+    });
+    producerTras.setPollInterval(100);
+    producerTras.connect({}, connectedCb);
+
+    consumerTrans = new Kafka.KafkaConsumer({
+      'metadata.broker.list': kafkaBrokerList,
+      'group.id': 'gropu_transaction_consumer',
+      'enable.auto.commit': false
+    }, {
+      'auto.offset.reset': 'earliest',
+    });
+    consumerTrans.connect({}, connectedCb);
+  });
+
+  after(function (done) {
+    let connected = 2;
+    function execDisconnect(client) {
+      if (!client.isConnected) {
+        connected--;
+        if (connected === 0) {
+          done();
+        }
+      } else {
+        client.disconnect(function() {
+          connected--;
+          if (connected === 0) {
+            done();
+          }
+        });
+      }
+    }
+    execDisconnect(producerTras);
+    execDisconnect(consumerTrans);
+  });
+
+  it('should init transactions', function(done) {
+    producerTras.initTransactions(TRANSACTIONS_TIMEOUT_MS, function (err) {
+      done(err);
+    });
+  });
+
+  it('should complete transaction', function(done) {
+    function readMessage() {
+      consumerTrans.consume(1, function(err, m) {
+        if (err) {
+          done(err);
+          return;
+        }
+        if (m.length === 0) {
+          readMessage();
+        } else {
+          var v = m[0].value.toString();
+          if (v === 'A') { // skip first message
+            readMessage();
+            return;
+          }
+          if (v !== 'B') {
+            done('Expected B');
+            return;
+          }
+          producerTras.beginTransaction(function (err) {
+            if (err) {
+              done(err);
+              return;
+            }
+            producerTras.produce(topicOut, -1, Buffer.from(v));
+            var position = consumerTrans.position();
+            producerTras.sendOffsetsToTransaction(position, consumerTrans, function(err) {
+              if (err) {
+                done(err);
+                return;
+              }
+              producerTras.commitTransaction(function(err) {
+                if (err) {
+                  done(err);
+                  return;
+                }
+                consumerTrans.committed(5000, function(err, tpo) {
+                  if (err) {
+                    done(err);
+                    return;
+                  }
+                  if (JSON.stringify(position) !== JSON.stringify(tpo)) {
+                    done('Committed mismatch');
+                    return;
+                  }
+                  done();
+                });
+              });
+            });
+          });
+        }
+      });
+    }
+    readMessage();
+  });
+
+  describe('abort transaction', function() {
+    var lastConsumerTransPosition;
+    before(function(done) {
+      function readMessage() {
+        consumerTrans.consume(1, function(err, m) {
+          if (err) {
+            done(err);
+            return;
+          }
+          if (m.length === 0) {
+            readMessage();
+          } else {
+            var v = m[0].value.toString();
+            if (v !== 'C') {
+              done('Expected C');
+              return;
+            }
+            producerTras.beginTransaction(function (err) {
+              if (err) {
+                done(err);
+                return;
+              }
+              producerTras.produce(topicOut, -1, Buffer.from(v));
+              lastConsumerTransPosition = consumerTrans.position();
+              producerTras.sendOffsetsToTransaction(lastConsumerTransPosition, consumerTrans, function(err) {
+                if (err) {
+                  done(err);
+                  return;
+                }
+                done();
+              });
+            });
+          }
+        });
+      }
+      readMessage();
+    });
+
+    it ('should consume committed and uncommitted for read_uncommitted', function(done) {
+      var allMsgs = [];
+      var consumer = new Kafka.KafkaConsumer({
+        'metadata.broker.list': kafkaBrokerList,
+        'group.id': 'group_read_uncommitted',
+        'enable.auto.commit': false,
+        'isolation.level': 'read_uncommitted'
+      }, {
+        'auto.offset.reset': 'earliest',
+      });
+      consumer.connect({}, function(err) {
+        if (err) {
+          done(err);
+          return;
+        }
+        consumer.subscribe([topicOut]);
+        consumer.consume();
+      });
+      consumer.on('data', function(msg) {
+        var v = msg.value.toString();
+        allMsgs.push(v);
+        // both B and C must be consumed
+        if (allMsgs.length === 2 && allMsgs[0] === 'B' && allMsgs[1] === 'C') {
+          consumer.disconnect(function(err) {
+            if (err) {
+              done(err);
+              return;
+            }
+            done();
+          })
+        }
+      });
+    });
+
+    it ('should consume only committed for read_committed', function(done) {
+      var allMsgs = [];
+      var consumer = new Kafka.KafkaConsumer({
+        'metadata.broker.list': kafkaBrokerList,
+        'group.id': 'group_read_committed',
+        'enable.partition.eof': true,
+        'enable.auto.commit': false,
+        'isolation.level': 'read_committed'
+      }, {
+        'auto.offset.reset': 'earliest',
+      });
+      consumer.connect({}, function(err) {
+        if (err) {
+          done(err);
+          return;
+        }
+        consumer.subscribe([topicOut]);
+        consumer.consume();
+      });
+      consumer.on('data', function(msg) {
+        var v = msg.value.toString();
+        allMsgs.push(v);
+      });
+      consumer.on('partition.eof', function(eof) {
+        if (allMsgs.length === 1 && allMsgs[0] === 'B') {
+          consumer.disconnect(function(err) {
+            if (err) {
+              done(err);
+              return;
+            }
+            done();
+          })
+        } else {
+          done('Expected only B');
+          return;
+        }
+      });
+    });
+
+    it('should abort transaction', function(done) {
+      producerTras.abortTransaction(function(err) {
+        if (err) {
+          done(err);
+          return;
+        }
+        consumerTrans.committed(5000, function(err, tpo) {
+          if (err) {
+            done(err);
+            return;
+          }
+          if (lastConsumerTransPosition[0].offset <= tpo[0].offset) {
+            done('Committed mismatch');
+            return;
+          }
+          done();
+        });
+      });
+    });
+
+    it('should consume only committed', function(done) {
+      var gotB = false;
+      var consumer = new Kafka.KafkaConsumer({
+        'metadata.broker.list': kafkaBrokerList,
+        'group.id': 'group_default',
+        'enable.partition.eof': true,
+        'enable.auto.commit': false,
+      }, {
+        'auto.offset.reset': 'earliest',
+      });
+      consumer.connect({}, function(err) {
+        if (err) {
+          done(err);
+          return;
+        }
+        consumer.subscribe([topicOut]);
+        consumer.consume();
+      });
+      consumer.on('data', function(msg) {
+        var v = msg.value.toString();
+        if (v !== 'B') {
+          done('Expected B');
+          return;
+        }
+        gotB = true;
+      });
+      consumer.on('partition.eof', function(eof) {
+        if (!gotB) {
+          done('Expected B');
+          return;
+        }
+        consumer.disconnect(function(err) {
+          if (err) {
+            done(err);
+            return;
+          }
+          done();
+        });
+      });
+    });
+  });
+});

package/e2e/producer.spec.js

@@ -0,0 +1,283 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+var t = require('assert');
+var crypto = require('crypto');
+
+var eventListener = require('./listener');
+
+var kafkaBrokerList = process.env.KAFKA_HOST || 'localhost:9092';
+
+var serviceStopped = false;
+
+describe('Producer', function() {
+
+  var producer;
+
+  describe('with dr_cb', function() {
+    beforeEach(function(done) {
+      producer = new Kafka.Producer({
+        'client.id': 'kafka-test',
+        'metadata.broker.list': kafkaBrokerList,
+        'dr_cb': true,
+        'debug': 'all'
+      });
+      producer.connect({}, function(err) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(producer);
+    });
+
+    afterEach(function(done) {
+      producer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('should connect to Kafka', function(done) {
+      producer.getMetadata({}, function(err, metadata) {
+        t.ifError(err);
+        t.ok(metadata);
+
+        // Ensure it is in the correct format
+        t.ok(metadata.orig_broker_name, 'Broker name is not set');
+        t.notStrictEqual(metadata.orig_broker_id, undefined, 'Broker id is not set');
+        t.equal(Array.isArray(metadata.brokers), true);
+        t.equal(Array.isArray(metadata.topics), true);
+
+        done();
+      });
+    });
+
+    it('should produce a message with a null payload and null key', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.strictEqual( report.key, null);
+        done();
+      });
+
+      producer.produce('test', null, null, null);
+    });
+
+    it('should produce a message with a payload and key', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+        t.strictEqual(report.value, undefined);
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.equal(report.key, 'key');
+        done();
+      });
+
+      producer.produce('test', null, Buffer.from('value'), 'key');
+    });
+
+    it('should produce a message with a payload and key buffer', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+        t.strictEqual(report.value, undefined);
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.equal(report.key.length > 3, true);
+        done();
+      });
+
+      producer.produce('test', null, Buffer.from('value'), Buffer.from('key\0s'));
+    });
+
+    it('should produce a message with an opaque', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.equal(report.opaque, 'opaque');
+        done();
+      });
+
+      producer.produce('test', null, Buffer.from('value'), null, null, 'opaque');
+    });
+
+
+    it('should get 100% deliverability', function(done) {
+      var total = 0;
+      var max = 10000;
+      var verified_received = 0;
+
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer
+        .on('delivery-report', function(err, report) {
+          t.ifError(err);
+          t.notStrictEqual(report, undefined);
+          t.strictEqual(typeof report.topic, 'string');
+          t.strictEqual(typeof report.partition, 'number');
+          t.strictEqual(typeof report.offset, 'number');
+          verified_received++;
+          if (verified_received === max) {
+            clearInterval(tt);
+            done();
+          }
+        });
+
+      // Produce
+      for (total = 0; total <= max; total++) {
+        producer.produce('test', null, Buffer.from('message ' + total), null);
+      }
+
+    });
+
+  });
+
+  describe('with_dr_msg_cb', function() {
+    beforeEach(function(done) {
+      producer = new Kafka.Producer({
+        'client.id': 'kafka-test',
+        'metadata.broker.list': kafkaBrokerList,
+        'dr_msg_cb': true,
+        'debug': 'all'
+      });
+      producer.connect({}, function(err) {
+        t.ifError(err);
+        done();
+      });
+
+      eventListener(producer);
+    });
+
+    afterEach(function(done) {
+      producer.disconnect(function() {
+        done();
+      });
+    });
+
+    it('should produce a message with a payload and key', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.ok(report.key.toString(), 'key');
+        t.equal(report.value.toString(), 'hai');
+        done();
+      });
+
+      producer.produce('test', null, Buffer.from('hai'), 'key');
+    });
+
+    it('should produce a message with an empty payload and empty key (https://github.com/Blizzard/node-rdkafka/issues/117)', function(done) {
+      var tt = setInterval(function() {
+        producer.poll();
+      }, 200);
+
+      producer.once('delivery-report', function(err, report) {
+        clearInterval(tt);
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.equal(report.key.toString(), '', 'key should be an empty string');
+        t.strictEqual(report.value.toString(), '', 'payload should be an empty string');
+        done();
+      });
+
+      producer.produce('test', null, Buffer.from(''), '');
+    });
+
+    it('should produce a message with a null payload and null key  (https://github.com/Blizzard/node-rdkafka/issues/117)', function(done) {
+      producer.setPollInterval(10);
+
+      producer.once('delivery-report', function(err, report) {
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+
+        t.strictEqual(typeof report.topic, 'string');
+        t.strictEqual(typeof report.partition, 'number');
+        t.strictEqual(typeof report.offset, 'number');
+        t.strictEqual(report.key, null, 'key should be null');
+        t.strictEqual(report.value, null, 'payload should be null');
+        done();
+      });
+
+      producer.produce('test', null, null, null);
+    });
+
+    it('should produce an int64 key (https://github.com/Blizzard/node-rdkafka/issues/208)', function(done) {
+
+      var v1 = 0x0000000000000084;
+      var arr = new Uint8Array(8);
+      arr[0] = 0x00;
+      arr[1] = 0x00;
+      arr[2] = 0x00;
+      arr[3] = 0x00;
+      arr[4] = 0x00;
+      arr[5] = 0x00;
+      arr[6] = 0x00;
+      arr[7] = 84;
+      var buf = Buffer.from(arr.buffer);
+
+      producer.setPollInterval(10);
+
+      producer.once('delivery-report', function(err, report) {
+        t.ifError(err);
+        t.notStrictEqual(report, undefined);
+
+        t.deepEqual(buf, report.key);
+        done();
+      });
+
+      producer.produce('test', null, null, Buffer.from(arr.buffer));
+
+    });
+
+  });
+
+});