@platformatic/rdkafka 4.0.0

package/bench/consumer-raw-rdkafka.js

@@ -0,0 +1,76 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+var count = 0;
+var total = 0;
+var store = [];
+var host = process.argv[2] || 'localhost:9092';
+var topic = process.argv[3] || 'test';
+
+var consumer = new Kafka.KafkaConsumer({
+  'metadata.broker.list': host,
+  'group.id': 'node-rdkafka-bench-s',
+  'fetch.wait.max.ms': 100,
+  'fetch.message.max.bytes': 1024 * 1024,
+  'enable.auto.commit': false
+  // paused: true,
+}, {
+  'auto.offset.reset': 'earliest'
+});
+
+var interval;
+
+consumer.connect()
+  .once('ready', function() {
+    consumer.subscribe([topic]);
+    consumer.consume();
+  })
+  .on('rebalance', function() {
+    console.log('rebalance');
+  })
+  .once('data', function() {
+    interval = setInterval(function() {
+      console.log('%d messages per second', count);
+      if (count > 0) {
+        store.push(count);
+      }
+      count = 0;
+    }, 1000);
+  })
+  .on('data', function(message) {
+    count += 1;
+    total += 1;
+  });
+
+function shutdown() {
+  clearInterval(interval);
+
+  if (store.length > 0) {
+    var calc = 0;
+    for (var x in store) {
+      calc += store[x];
+    }
+
+    var mps = parseFloat(calc * 1.0/store.length);
+
+    console.log('%d messages per second on average', mps);
+
+  }
+
+  var killTimer = setTimeout(function() {
+    process.exit();
+  }, 5000);
+
+  consumer.disconnect(function() {
+    clearTimeout(killTimer);
+    process.exit();
+  });
+
+}

package/bench/consumer-subscribe.js

@@ -0,0 +1,77 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+var count = 0;
+var total = 0;
+var store = [];
+var host = process.argv[2] || 'localhost:9092';
+var topic = process.argv[3] || 'test';
+
+var consumer = new Kafka.KafkaConsumer({
+  'metadata.broker.list': host,
+  'group.id': 'node-rdkafka-bench',
+  'fetch.wait.max.ms': 100,
+  'fetch.message.max.bytes': 1024 * 1024,
+  'enable.auto.commit': false
+  // paused: true,
+}, {
+  'auto.offset.reset': 'earliest'
+});
+
+var interval;
+
+consumer.connect()
+  .once('ready', function() {
+    consumer.subscribe([topic]);
+    consumer.consume();
+  })
+  .once('data', function() {
+    interval = setInterval(function() {
+      console.log('%d messages per second', count);
+      if (count > 0) {
+        store.push(count);
+      }
+      count = 0;
+    }, 1000);
+  })
+  .on('data', function(message) {
+    count += 1;
+    total += 1;
+  });
+
+process.once('SIGTERM', shutdown);
+process.once('SIGINT', shutdown);
+process.once('SIGHUP', shutdown);
+
+function shutdown() {
+  clearInterval(interval);
+
+  if (store.length > 0) {
+    var calc = 0;
+    for (var x in store) {
+      calc += store[x];
+    }
+
+    var mps = parseFloat(calc * 1.0/store.length);
+
+    console.log('%d messages per second on average', mps);
+
+  }
+
+  var killTimer = setTimeout(function() {
+    process.exit();
+  }, 5000);
+
+  consumer.disconnect(function() {
+    clearTimeout(killTimer);
+    process.exit();
+  });
+
+}

package/bench/kafka-consumer-stream.js

@@ -0,0 +1,100 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Writable = require('stream').Writable;
+
+var Kafka = require('../');
+var count = 0;
+var total = 0;
+var store = [];
+var host = process.argv[2] || 'localhost:9092';
+var topic = process.argv[3] || 'test';
+
+var stream = Kafka.createReadStream({
+  'metadata.broker.list': host,
+  'group.id': 'node-rdkafka-benchs',
+  'fetch.wait.max.ms': 100,
+  'fetch.message.max.bytes': 1024 * 1024,
+  'enable.auto.commit': false
+  // paused: true,
+}, {
+  'auto.offset.reset': 'earliest'
+}, {
+  fetchSize: 16,
+  topics: [topic]
+});
+
+// Track how many messages we see per second
+var interval;
+
+var isShuttingDown = false;
+
+stream
+  .on('error', function(err) {
+    console.log('Shutting down due to error');
+    console.log(err.stack);
+    shutdown();
+  })
+  .once('data', function(d) {
+    interval = setInterval(function() {
+      if (isShuttingDown) {
+        clearInterval(interval);
+      }
+      console.log('%d messages per second', count);
+      if (count > 0) {
+        // Don't store ones when we didn't get data i guess?
+        store.push(count);
+        // setTimeout(shutdown, 500);
+      }
+     count = 0;
+   }, 1000).unref();
+  })
+  .on('end', function() {
+    // Can be called more than once without issue because of guard var
+    console.log('Shutting down due to stream end');
+    shutdown();
+  })
+  .pipe(new Writable({
+    objectMode: true,
+    write: function(message, encoding, cb) {
+      count += 1;
+      total += 1;
+      setImmediate(cb);
+    }
+  }));
+
+process.once('SIGTERM', shutdown);
+process.once('SIGINT', shutdown);
+process.once('SIGHUP', shutdown);
+
+function shutdown() {
+  if (isShuttingDown) {
+    return;
+  }
+  clearInterval(interval);
+  isShuttingDown = true;
+  if (store.length > 0) {
+    var calc = 0;
+    for (var x in store) {
+      calc += store[x];
+    }
+
+    var mps = parseFloat(calc * 1.0/store.length);
+
+    console.log('%d messages per second on average', mps);
+  }
+
+  // Destroy the stream
+  stream.destroy();
+
+  stream.once('end', function() {
+    console.log('total: %d', total);
+  });
+
+}

package/bench/producer-raw-rdkafka.js

@@ -0,0 +1,125 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+var crypto = require('crypto');
+var count = 0;
+var total = 0;
+var totalComplete = 0;
+var verifiedComplete = 0;
+var errors = 0;
+var store = [];
+var started;
+var done = false;
+var host = process.argv[2] || '127.0.0.1:9092';
+var topicName = process.argv[3] || 'test';
+var compression = process.argv[4] || 'gzip';
+var MAX = process.argv[5] || 10000000;
+
+var producer = new Kafka.Producer({
+  'metadata.broker.list': host,
+  'group.id': 'node-rdkafka-bench',
+  'compression.codec': compression,
+  'retry.backoff.ms': 200,
+  'message.send.max.retries': 10,
+  'socket.keepalive.enable': true,
+  'queue.buffering.max.messages': 100000,
+  'queue.buffering.max.ms': 1000,
+  'batch.num.messages': 1000
+});
+
+// Track how many messages we see per second
+var interval;
+var ok = true;
+
+function getTimer() {
+  if (!interval) {
+    interval = setTimeout(function() {
+      interval = false;
+      if (!done) {
+        console.log('%d messages per sent second', count);
+        store.push(count);
+        count = 0;
+        getTimer();
+
+      } else {
+        console.log('%d messages remaining sent in last batch <1000ms', count);
+      }
+    }, 1000);
+  }
+
+  return interval;
+}
+
+var t;
+
+crypto.randomBytes(4096, function(ex, buffer) {
+
+  producer.connect()
+    .on('ready', function() {
+      getTimer();
+
+      started = new Date().getTime();
+
+      var sendMessage = function() {
+        try {
+          var errorCode = producer.produce(topicName, null, buffer, null);
+          verifiedComplete += 1;
+        } catch (e) {
+          console.error(e);
+          errors++;
+        }
+
+        count += 1;
+        totalComplete += 1;
+        if (totalComplete === MAX) {
+          shutdown();
+        }
+        if (total < MAX) {
+          total += 1;
+
+          // This is 100% sync so we need to setImmediate to give it time
+          // to breathe.
+          setImmediate(sendMessage);
+        }
+      };
+
+      sendMessage();
+
+    })
+    .on('event.error', function(err) {
+      console.error(err);
+      process.exit(1);
+    })
+    .on('disconnected', shutdown);
+
+});
+
+function shutdown(e) {
+  done = true;
+
+  clearInterval(interval);
+
+  var killTimer = setTimeout(function() {
+    process.exit();
+  }, 5000);
+
+  producer.disconnect(function() {
+    clearTimeout(killTimer);
+    var ended = new Date().getTime();
+    var elapsed = ended - started;
+
+    // console.log('Ended %s', ended);
+    console.log('total: %d messages over %d ms', total, elapsed);
+
+    console.log('%d messages / second', parseInt(total / (elapsed / 1000)));
+    process.exit();
+  });
+
+}

package/bench/producer-rdkafka.js

@@ -0,0 +1,117 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+var crypto = require('crypto');
+var count = 0;
+var total = 0;
+var totalComplete = 0;
+var store = [];
+var host = process.argv[2] || '127.0.0.1:9092';
+var topicName = process.argv[3] || 'test';
+var compression = process.argv[4] || 'gzip';
+var MAX = process.argv[5] || 1000000;
+
+var stream = Kafka.Producer.createWriteStream({
+  'metadata.broker.list': host,
+  'group.id': 'node-rdkafka-bench',
+  'compression.codec': compression,
+  'retry.backoff.ms': 200,
+  'message.send.max.retries': 10,
+  'socket.keepalive.enable': true,
+  'queue.buffering.max.messages': 100000,
+  'queue.buffering.max.ms': 1000,
+  'batch.num.messages': 1000,
+}, {}, {
+  topic: topicName,
+  pollInterval: 20
+});
+
+stream.on('error', function(e) {
+  console.log(e);
+  process.exit(1);
+});
+
+// Track how many messages we see per second
+var interval;
+var done = false;
+
+function log() {
+  console.log('%d messages per sent second', count);
+  store.push(count);
+  count = 0;
+}
+
+crypto.randomBytes(4096, function(ex, buffer) {
+
+  var x = function(e) {
+    if (e) {
+      console.error(e);
+    }
+    count += 1;
+    totalComplete += 1;
+    if (totalComplete >= MAX && !done) {
+      done = true;
+      clearInterval(interval);
+      setTimeout(shutdown, 5000);
+    }
+  };
+
+  function write() {
+    if (!stream.write(buffer, 'base64', x)) {
+      return stream.once('drain', write);
+    } else {
+      total++;
+    }
+
+    if (total < MAX) {
+      // we are not done
+      setImmediate(write);
+    }
+
+  }
+
+  write();
+  interval = setInterval(log, 1000);
+  stream.on('error', function(err) {
+    console.log(err);
+  });
+  // stream.on('end', shutdown);
+
+});
+
+
+process.once('SIGTERM', shutdown);
+process.once('SIGINT', shutdown);
+process.once('SIGHUP', shutdown);
+
+function shutdown() {
+
+  if (store.length > 0) {
+    var calc = 0;
+    for (var x in store) {
+      calc += store[x];
+    }
+
+    var mps = parseFloat(calc * 1.0/store.length);
+
+    console.log('%d messages per second on average', mps);
+    console.log('%d messages total', total);
+
+  }
+
+  clearInterval(interval);
+
+  stream.end();
+
+  stream.on('close', function() {
+    console.log('total: %d', total);
+  });
+
+}

package/bench/seed.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+kafka_root=${KAFKA_ROOT:-/opt/kafka}
+# Generate and insert some messages
+
+OS=$(uname -s)
+
+function initializeTopic {
+  topic=$1
+  host=$2
+  msg_size=$3
+  batch_size=$4
+  batch_count=$5
+
+  if [ $host == "localhost:9092" ]; then
+    ${kafka_root}/bin/kafka-topics.sh --create --zookeeper localhost:2181 \
+      --replication-factor 1 --partitions 1 --topic ${topic}
+  fi
+
+  echo "Generating messages (size: ${msg_size})"
+  : > /tmp/msgs  # Truncate /tmp/msgs
+  for i in $(seq 1 ${batch_size}); do
+    if [ $OS == 'Darwin' ]; then
+      printf %s\\n "$(head -c${msg_size} /dev/urandom | base64)" >> /tmp/msgs
+    else
+      printf %s\\n "$(head --bytes=${msg_size} /dev/urandom | base64 --wrap=0)" >> /tmp/msgs
+    fi
+  done
+
+  echo "Done generating messages"
+
+  for i in $(seq 1 ${batch_count}); do
+    echo "Adding $(wc -l /tmp/msgs) messages to topic ${topic}"
+    "${kafka_root}/bin/kafka-console-producer.sh" \
+      --broker-list ${host} --topic ${topic} < /tmp/msgs
+  done
+}
+
+initializeTopic "librdtesting-01" "localhost:9092" "4096" "5000" "2000"

package/binding.gyp

@@ -0,0 +1,155 @@
+{
+  "variables": {
+    # may be redefined in command line on configuration stage
+    # "BUILD_LIBRDKAFKA%": "<!(echo ${BUILD_LIBRDKAFKA:-1})"
+    "BUILD_LIBRDKAFKA%": "<!(node ./util/get-env.js BUILD_LIBRDKAFKA 1)",
+  },
+  "targets": [
+    {
+      "target_name": "node-librdkafka",
+      'sources': [
+        'src/binding.cc',
+        'src/callbacks.cc',
+        'src/common.cc',
+        'src/config.cc',
+        'src/connection.cc',
+        'src/errors.cc',
+        'src/kafka-consumer.cc',
+        'src/per-isolate-data.cc',
+        'src/producer.cc',
+        'src/topic.cc',
+        'src/workers.cc',
+        'src/admin.cc'
+      ],
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")",
+        "<(module_root_dir)/"
+      ],
+      'conditions': [
+        [
+          'OS=="win"',
+          {
+            'actions': [
+              {
+                'action_name': 'nuget_librdkafka_download',
+                'inputs': [
+                  'deps/windows-install.py'
+                ],
+                'outputs': [
+                  'deps/precompiled/librdkafka.lib',
+                  'deps/precompiled/librdkafkacpp.lib'
+                ],
+                'message': 'Getting librdkafka from nuget',
+                'action': ['python', '<@(_inputs)']
+              }
+            ],
+            'cflags_cc' : [
+              '-std=c++17'
+            ],
+            'msvs_settings': {
+              'VCLinkerTool': {
+                'AdditionalDependencies': [
+                  'librdkafka.lib',
+                  'librdkafkacpp.lib'
+                ],
+                'AdditionalLibraryDirectories': [
+                  '../deps/precompiled/'
+                ]
+              },
+              'VCCLCompilerTool': {
+                'AdditionalOptions': [
+                  '/GR'
+                ],
+                'AdditionalUsingDirectories': [
+                  'deps/precompiled/'
+                ],
+                'AdditionalIncludeDirectories': [
+                  'deps/librdkafka/src',
+                  'deps/librdkafka/src-cpp'
+                ]
+              }
+            },
+            'include_dirs': [
+              'deps/include'
+            ]
+          },
+          {
+            'conditions': [
+              [ "<(BUILD_LIBRDKAFKA)==1",
+                {
+                  "dependencies": [
+                    "deps/librdkafka.gyp:librdkafka"
+                  ],
+                  "include_dirs": [
+                    "deps/librdkafka/src",
+                    "deps/librdkafka/src-cpp"
+                  ],
+                  'conditions': [
+                    [
+                      'OS=="linux"',
+                      {
+                        "libraries": [
+                          "../build/deps/librdkafka.so",
+                          "../build/deps/librdkafka++.so",
+                          "-Wl,-rpath='$$ORIGIN/../deps'",
+                        ],
+                      }
+                    ],
+                    [
+                      'OS=="mac"',
+                      {
+                        "libraries": [
+                          "../build/deps/librdkafka.dylib",
+                          "../build/deps/librdkafka++.dylib",
+                        ],
+                      }
+                    ]
+                  ],
+                },
+                # Else link against globally installed rdkafka and use
+                # globally installed headers.  On Debian, you should
+                # install the librdkafka1, librdkafka++1, and librdkafka-dev
+                # .deb packages.
+                {
+                  "libraries": ["-lrdkafka", "-lrdkafka++"],
+                  "include_dirs": [
+                    "/usr/include/librdkafka",
+                    "/usr/local/include/librdkafka",
+                    "/opt/include/librdkafka",
+                  ],
+                },
+              ],
+              [
+                'OS=="linux"',
+                {
+                  'cflags_cc' : [
+                    '-std=c++17'
+                  ],
+                  'cflags_cc!': [
+                    '-fno-rtti'
+                  ]
+                }
+              ],
+              [
+                'OS=="mac"',
+                {
+                  'xcode_settings': {
+                    'MACOSX_DEPLOYMENT_TARGET': '10.11',
+                    'GCC_ENABLE_CPP_RTTI': 'YES',
+                    'OTHER_LDFLAGS': [
+                      '-L/usr/local/opt/openssl/lib'
+                    ],
+                    'OTHER_CPLUSPLUSFLAGS': [
+                      '-I/usr/local/opt/openssl/include',
+                      '-std=c++17'
+                    ],
+                  },
+                }
+              ]
+            ]
+          }
+        ]
+      ]
+    }
+  ]
+}