@point3/node-rdkafka 3.6.0-1

package/lib/producer.js

@@ -0,0 +1,375 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = Producer;
+
+var Client = require('./client');
+
+var util = require('util');
+var Kafka = require('../librdkafka.js');
+var ProducerStream = require('./producer-stream');
+var LibrdKafkaError = require('./error');
+var shallowCopy = require('./util').shallowCopy;
+
+util.inherits(Producer, Client);
+
+/**
+ * Producer class for sending messages to Kafka
+ *
+ * This is the main entry point for writing data to Kafka. You
+ * configure this like you do any other client, with a global
+ * configuration and default topic configuration.
+ *
+ * Once you instantiate this object, you need to connect to it first.
+ * This allows you to get the metadata and make sure the connection
+ * can be made before you depend on it. After that, problems with
+ * the connection will by brought down by using poll, which automatically
+ * runs when a transaction is made on the object.
+ *
+ * @param {object} conf - Key value pairs to configure the producer
+ * @param {object} topicConf - Key value pairs to create a default
+ * topic configuration
+ * @extends Client
+ * @constructor
+ */
+function Producer(conf, topicConf) {
+  if (!(this instanceof Producer)) {
+    return new Producer(conf, topicConf);
+  }
+
+  conf = shallowCopy(conf);
+  topicConf = shallowCopy(topicConf);
+
+  /**
+   * Producer message. This is sent to the wrapper, not received from it
+   *
+   * @typedef {object} Producer~Message
+   * @property {string|buffer} message - The buffer to send to Kafka.
+   * @property {Topic} topic - The Kafka topic to produce to.
+   * @property {number} partition - The partition to produce to. Defaults to
+   * the partitioner
+   * @property {string} key - The key string to use for the message.
+   * @see Consumer~Message
+   */
+
+  var gTopic = conf.topic || false;
+  var gPart = conf.partition || null;
+  var dr_cb = conf.dr_cb || null;
+  var dr_msg_cb = conf.dr_msg_cb || null;
+
+  // delete keys we don't want to pass on
+  delete conf.topic;
+  delete conf.partition;
+
+  delete conf.dr_cb;
+  delete conf.dr_msg_cb;
+
+  // client is an initialized consumer object
+  // @see NodeKafka::Producer::Init
+  Client.call(this, conf, Kafka.Producer, topicConf);
+
+  // Delete these keys after saving them in vars
+  this.globalConfig = conf;
+  this.topicConfig = topicConf;
+  this.defaultTopic = gTopic || null;
+  this.defaultPartition = gPart == null ? -1 : gPart;
+
+  this.sentMessages = 0;
+
+  this.pollInterval = undefined;
+
+  if (dr_msg_cb || dr_cb) {
+    this._cb_configs.event.delivery_cb =  function(err, report) {
+      if (err) {
+        err = LibrdKafkaError.create(err);
+      }
+      this.emit('delivery-report', err, report);
+    }.bind(this);
+    this._cb_configs.event.delivery_cb.dr_msg_cb = !!dr_msg_cb;
+
+    if (typeof dr_cb === 'function') {
+      this.on('delivery-report', dr_cb);
+    }
+
+  }
+}
+
+/**
+ * Produce a message to Kafka synchronously.
+ *
+ * This is the method mainly used in this class. Use it to produce
+ * a message to Kafka.
+ *
+ * When this is sent off, there is no guarantee it is delivered. If you need
+ * guaranteed delivery, change your *acks* settings, or use delivery reports.
+ *
+ * @param {string} topic - The topic name to produce to.
+ * @param {number|null} partition - The partition number to produce to.
+ * @param {Buffer|null} message - The message to produce.
+ * @param {string} key - The key associated with the message.
+ * @param {number|null} timestamp - Timestamp to send with the message.
+ * @param {object} opaque - An object you want passed along with this message, if provided.
+ * @param {object} headers - A list of custom key value pairs that provide message metadata.
+ * @throws {LibrdKafkaError} - Throws a librdkafka error if it failed.
+ * @return {boolean} - returns an error if it failed, or true if not
+ * @see Producer#produce
+ */
+Producer.prototype.produce = function(topic, partition, message, key, timestamp, opaque, headers) {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+
+  // I have removed support for using a topic object. It is going to be removed
+  // from librdkafka soon, and it causes issues with shutting down
+  if (!topic || typeof topic !== 'string') {
+    throw new TypeError('"topic" must be a string');
+  }
+
+  this.sentMessages++;
+
+  partition = partition == null ? this.defaultPartition : partition;
+
+  return this._errorWrap(
+    this._client.produce(topic, partition, message, key, timestamp, opaque, headers));
+
+};
+
+/**
+ * Create a write stream interface for a producer.
+ *
+ * This stream does not run in object mode. It only takes buffers of data.
+ *
+ * @param {object} conf - Key value pairs to configure the producer
+ * @param {object} topicConf - Key value pairs to create a default
+ * topic configuration
+ * @param {object} streamOptions - Stream options
+ * @return {ProducerStream} - returns the write stream for writing to Kafka.
+ */
+Producer.createWriteStream = function(conf, topicConf, streamOptions) {
+  var producer = new Producer(conf, topicConf);
+  return new ProducerStream(producer, streamOptions);
+};
+
+/**
+ * Poll for events
+ *
+ * We need to run poll in order to learn about new events that have occurred.
+ * This is no longer done automatically when we produce, so we need to run
+ * it manually, or set the producer to automatically poll.
+ *
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.poll = function() {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+  this._client.poll();
+  return this;
+};
+
+/**
+ * Set automatic polling for events.
+ *
+ * We need to run poll in order to learn about new events that have occurred.
+ * If you would like this done on an interval with disconnects and reconnections
+ * managed, you can do it here
+ *
+ * @param {number} interval - Interval, in milliseconds, to poll
+ *
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.setPollInterval = function(interval) {
+  // If we already have a poll interval we need to stop it
+  if (this.pollInterval) {
+    clearInterval(this.pollInterval);
+    this.pollInterval = undefined;
+  }
+
+  if (interval === 0) {
+    // If the interval was set to 0, bail out. We don't want to process this.
+    // If there was an interval previously set, it has been removed.
+    return;
+  }
+
+  var self = this;
+
+  // Now we want to make sure we are connected.
+  if (!this._isConnected) {
+    // If we are not, execute this once the connection goes through.
+    this.once('ready', function() {
+      self.setPollInterval(interval);
+    });
+    return;
+  }
+
+  // We know we are connected at this point.
+  // Unref this interval
+  this.pollInterval = setInterval(function() {
+    try {
+      self.poll();
+    } catch (e) {
+      // We can probably ignore errors here as far as broadcasting.
+      // Disconnection issues will get handled below
+    }
+  }, interval).unref();
+
+  // Handle disconnections
+  this.once('disconnected', function() {
+    // Just rerun this function with interval 0. If any
+    // poll interval is set, this will remove it
+    self.setPollInterval(0);
+  });
+
+  return this;
+};
+
+/**
+ * Flush the producer
+ *
+ * Flush everything on the internal librdkafka producer buffer. Do this before
+ * disconnects usually
+ *
+ * @param {number} timeout - Number of milliseconds to try to flush before giving up.
+ * @param {function} callback - Callback to fire when the flush is done.
+ *
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.flush = function(timeout, callback) {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+
+  if (timeout === undefined || timeout === null) {
+    timeout = 500;
+  }
+
+  this._client.flush(timeout, function(err) {
+    if (err) {
+      err = LibrdKafkaError.create(err);
+    }
+
+    if (callback) {
+      callback(err);
+    }
+  });
+  return this;
+};
+
+/**
+ * Save the base disconnect method here so we can overwrite it and add a flush
+ */
+Producer.prototype._disconnect = Producer.prototype.disconnect;
+
+/**
+ * Disconnect the producer
+ *
+ * Flush everything on the internal librdkafka producer buffer. Then disconnect
+ *
+ * @param {number} timeout - Number of milliseconds to try to flush before giving up, defaults to 5 seconds.
+ * @param {function} cb - The callback to fire when
+ */
+Producer.prototype.disconnect = function(timeout, cb) {
+  var self = this;
+  var timeoutInterval = 5000;
+
+  if (typeof timeout === 'function') {
+    cb = timeout;
+  } else {
+    timeoutInterval = timeout;
+  }
+
+  this.flush(timeoutInterval, function() {
+    self._disconnect(cb);
+  });
+};
+
+/**
+ * Init a transaction.
+ *
+ * Initialize transactions, this is only performed once per transactional producer.
+ *
+ * @param {number} timeout - Number of milliseconds to try to initialize before giving up, defaults to 5 seconds.
+ * @param {function} cb - Callback to return when operation is completed
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.initTransactions = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.initTransactions(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Begin a transaction.
+ *
+ * 'initTransaction' must have been called successfully (once) before this function is called.
+ *
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.beginTransaction = function(cb) {
+  this._client.beginTransaction(function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Commit the current transaction (as started with 'beginTransaction').
+ *
+ * @param {number} timeout - Number of milliseconds to try to commit before giving up, defaults to 5 seconds
+ * @param {function} cb - Callback to return when operation is completed
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.commitTransaction = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.commitTransaction(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Aborts the ongoing transaction.
+ *
+ * @param {number} timeout - Number of milliseconds to try to abort, defaults to 5 seconds
+ * @param {function} cb - Callback to return when operation is completed
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.abortTransaction = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.abortTransaction(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Send the current offsets of the consumer to the ongoing transaction.
+ *
+ * @param {number} offsets - Offsets to send as part of the next commit
+ * @param {Consumer} consumer - An instance of the consumer
+ * @param {number} timeout - Number of milliseconds to try to send offsets, defaults to 5 seconds
+ * @param {function} cb - Callback to return when operation is completed
+ * @return {Producer} - returns itself.
+ */
+Producer.prototype.sendOffsetsToTransaction = function(offsets, consumer, timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.sendOffsetsToTransaction(offsets, consumer.getClient(), timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};

package/lib/tools/ref-counter.js

@@ -0,0 +1,52 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = RefCounter;
+
+/**
+ * Ref counter class.
+ *
+ * Is used to basically determine active/inactive and allow callbacks that
+ * hook into each.
+ *
+ * For the producer, it is used to begin rapid polling after a produce until
+ * the delivery report is dispatched.
+ */
+function RefCounter(onActive, onPassive) {
+  this.context = {};
+  this.onActive = onActive;
+  this.onPassive = onPassive;
+  this.currentValue = 0;
+  this.isRunning = false;
+}
+
+/**
+ * Increment the ref counter
+ */
+RefCounter.prototype.increment = function() {
+  this.currentValue += 1;
+
+  // If current value exceeds 0, activate the start
+  if (this.currentValue > 0 && !this.isRunning) {
+    this.isRunning = true;
+    this.onActive(this.context);
+  }
+};
+
+/**
+ * Decrement the ref counter
+ */
+RefCounter.prototype.decrement = function() {
+  this.currentValue -= 1;
+
+  if (this.currentValue <= 0 && this.isRunning) {
+    this.isRunning = false;
+    this.onPassive(this.context);
+  }
+};

package/lib/topic-partition.js

@@ -0,0 +1,88 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Topic = require('./topic');
+
+module.exports = TopicPartition;
+
+/**
+ * Map an array of topic partition js objects to real topic partition objects.
+ *
+ * @param array The array of topic partition raw objects to map to topic
+ *              partition objects
+ */
+TopicPartition.map = function(array) {
+  return array.map(function(element) {
+    return TopicPartition.create(element);
+  });
+};
+
+/**
+ * Take a topic partition javascript object and convert it to the class.
+ * The class will automatically convert offset identifiers to special constants
+ *
+ * @param element The topic partition raw javascript object
+ */
+TopicPartition.create = function(element) {
+  // Just ensure we take something that can have properties. The topic partition
+  // class will
+  element = element || {};
+  return new TopicPartition(element.topic, element.partition, element.offset);
+};
+
+/**
+ * Create a topic partition. Just does some validation and decoration
+ * on topic partitions provided.
+ *
+ * Goal is still to behave like a plain javascript object but with validation
+ * and potentially some extra methods
+ */
+function TopicPartition(topic, partition, offset) {
+  if (!(this instanceof TopicPartition)) {
+    return new TopicPartition(topic, partition, offset);
+  }
+
+  // Validate that the elements we are iterating over are actual topic partition
+  // js objects. They do not need an offset, but they do need partition
+  if (!topic) {
+    throw new TypeError('"topic" must be a string and must be set');
+  }
+
+  if (partition === null || partition === undefined) {
+    throw new TypeError('"partition" must be a number and must set');
+  }
+
+  // We can just set topic and partition as they stand.
+  this.topic = topic;
+  this.partition = partition;
+
+  if (offset === undefined || offset === null) {
+    this.offset = Topic.OFFSET_STORED;
+  } else if (typeof offset === 'string') {
+    switch (offset.toLowerCase()) {
+      case 'earliest':
+      case 'beginning':
+        this.offset = Topic.OFFSET_BEGINNING;
+        break;
+      case 'latest':
+      case 'end':
+        this.offset = Topic.OFFSET_END;
+        break;
+      case 'stored':
+        this.offset = Topic.OFFSET_STORED;
+        break;
+      default:
+        throw new TypeError('"offset", if provided as a string, must be beginning, end, or stored.');
+    }
+  } else if (typeof offset === 'number') {
+    this.offset = offset;
+  } else {
+    throw new TypeError('"offset" must be a special string or number if it is set');
+  }
+}

package/lib/topic.js

@@ -0,0 +1,42 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var librdkafka = require('../librdkafka');
+
+module.exports = Topic;
+
+var topicKey = 'RdKafka::Topic::';
+var topicKeyLength = topicKey.length;
+
+// Take all of the topic special codes from librdkafka and add them
+// to the object
+// You can find this list in the C++ code at
+// https://github.com/edenhill/librdkafka/blob/master/src-cpp/rdkafkacpp.h#L1250
+for (var key in librdkafka.topic) {
+  // Skip it if it doesn't start with ErrorCode
+  if (key.indexOf('RdKafka::Topic::') !== 0) {
+    continue;
+  }
+
+  // Replace/add it if there are any discrepancies
+  var newKey = key.substring(topicKeyLength);
+  Topic[newKey] = librdkafka.topic[key];
+}
+
+/**
+ * Create a topic. Just returns the string you gave it right now.
+ *
+ * Looks like a class, but all it does is return the topic name.
+ * This is so that one day if there are interface changes that allow
+ * different use of topic parameters, we can just add to this constructor and
+ * have it return something richer
+ */
+function Topic(topicName) {
+  return topicName;
+}

package/lib/util.js

@@ -0,0 +1,29 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var util = module.exports = {};
+
+util.shallowCopy = function (obj) {
+
+  if (!util.isObject(obj)) { return obj; }
+
+  var copy = {};
+
+  for (var k in obj) {
+    if (obj.hasOwnProperty(k)) {
+      copy[k] = obj[k];
+    }
+  }
+
+  return copy;
+};
+
+util.isObject = function (obj) {
+  return obj && typeof obj === 'object';
+};

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@point3/node-rdkafka",
+  "version": "v3.6.0-1",
+  "description": "Node.js bindings for librdkafka with Prebuilt binary",
+  "librdkafka": "2.12.0",
+  "main": "lib/index.js",
+  "scripts": {
+    "configure": "node-gyp configure",
+    "build": "node-gyp build",
+    "test": "make test",
+    "install": "node-gyp-build",
+    "prebuild": "prebuildify --strip --target=19.0.0 --target=20.0.0 --target=22.0.0 --target=23.0.0",
+    "prebuild-all": "prebuildify --strip --target=19.0.0 --target=20.0.0 --target=22.0.0 --target=23.0.0 --arch x64 --arch arm64"
+  },
+  "keywords": [
+    "kafka",
+    "librdkafka"
+  ],
+  "files": [
+    "lib/",
+    "util/",
+    "prebuilds/",
+    "binding.gyp",
+    "deps/librdkafka.gyp",
+    "deps/librdkafka/",
+    "deps/windows-install.cmd",
+    "deps/windows-install.ps1"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:Blizzard/node-rdkafka.git"
+  },
+  "contributors": [
+    {
+      "name": "Stephen Parente",
+      "email": "webmakersteve@gmail.com"
+    },
+    {
+      "name": "Matt Gollob",
+      "email": "mattness@users.noreply.github.com"
+    }
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "bluebird": "^3.5.3",
+    "jsdoc": "^4.0.2",
+    "jshint": "^2.10.1",
+    "mocha": "^10.2.0",
+    "node-gyp": "^11.1.0",
+    "prebuildify": "^6.0.1",
+    "toolkit-jsdoc": "^1.0.0",
+    "bindings": "^1.3.1"
+  },
+  "dependencies": {
+    "nan": "^2.22.0",
+    "node-gyp-build": "^4.8.4"
+  },
+  "engines": {
+    "node": ">=16 <25"
+  }
+}

package/util/configure.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var query = process.argv[2];
+
+var fs = require('fs');
+var path = require('path');
+
+var baseDir = path.resolve(__dirname, '../');
+var releaseDir = path.join(baseDir, 'build', 'deps');
+
+var isWin = /^win/.test(process.platform);
+
+// Skip running this if we are running on a windows system
+if (isWin) {
+  process.stderr.write('Skipping run because we are on windows\n');
+  process.exit(0);
+}
+
+var childProcess = require('child_process');
+
+try {
+  childProcess.execSync('./configure --prefix=' + releaseDir + ' --libdir=' + releaseDir, {
+    cwd: baseDir,
+    stdio: [0,1,2]
+  });
+  process.exit(0);
+} catch (e) {
+  process.stderr.write(e.message + '\n');
+  process.exit(1);
+}

package/util/get-env.js

@@ -0,0 +1,6 @@
+'use strict';
+
+var env = process.argv[2];
+var def = process.argv[3] || '';
+
+process.stdout.write(process.env[env] || def);

package/util/test-compile.js

@@ -0,0 +1,11 @@
+var kafka = require('../lib');
+
+var p = new kafka.Producer({ 'bootstrap.servers': 'localhost:9092' }, {});
+
+p.connect({ timeout: 1000 }, function(err) {
+  if (!err) {
+    p.disconnect();
+  } else {
+    process.exit(0);
+  }
+});