@point3/node-rdkafka 3.6.0-1

package/lib/producer/high-level-producer.js

@@ -0,0 +1,323 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = HighLevelProducer;
+
+var util = require('util');
+var Producer = require('../producer');
+var LibrdKafkaError = require('../error');
+var EventEmitter = require('events').EventEmitter;
+var RefCounter = require('../tools/ref-counter');
+var shallowCopy = require('../util').shallowCopy;
+var isObject = require('../util').isObject;
+
+util.inherits(HighLevelProducer, Producer);
+
+var noopSerializer = createSerializer(function (v) { return v; });
+
+/**
+ * Create a serializer
+ *
+ * Method simply wraps a serializer provided by a user
+ * so it adds context to the error
+ *
+ * @returns {function} Serialization function
+ */
+function createSerializer(serializer) {
+  var applyFn = function serializationWrapper(v, cb) {
+    try {
+      return cb ? serializer(v, cb) : serializer(v);
+    } catch (e) {
+      var modifiedError = new Error('Could not serialize value: ' + e.message);
+      modifiedError.value = v;
+      modifiedError.serializer = serializer;
+      throw modifiedError;
+    }
+  };
+
+  // We can check how many parameters the function has and activate the asynchronous
+  // operation if the number of parameters the function accepts is > 1
+  return {
+    apply: applyFn,
+    async: serializer.length > 1
+  };
+}
+
+/**
+ * Producer class for sending messages to Kafka in a higher level fashion
+ *
+ * This is the main entry point for writing data to Kafka if you want more
+ * functionality than librdkafka supports out of the box. You
+ * configure this like you do any other client, with a global
+ * configuration and default topic configuration.
+ *
+ * Once you instantiate this object, you need to connect to it first.
+ * This allows you to get the metadata and make sure the connection
+ * can be made before you depend on it. After that, problems with
+ * the connection will by brought down by using poll, which automatically
+ * runs when a transaction is made on the object.
+ *
+ * This has a few restrictions, so it is not for free!
+ *
+ * 1. You may not define opaque tokens
+ *    The higher level producer is powered by opaque tokens.
+ * 2. Every message ack will dispatch an event on the node thread.
+ * 3. Will use a ref counter to determine if there are outgoing produces.
+ *
+ * This will return the new object you should use instead when doing your
+ * produce calls
+ *
+ * @param {object} conf - Key value pairs to configure the producer
+ * @param {object} topicConf - Key value pairs to create a default
+ * topic configuration
+ * @extends Producer
+ * @constructor
+ */
+function HighLevelProducer(conf, topicConf) {
+  if (!(this instanceof HighLevelProducer)) {
+    return new HighLevelProducer(conf, topicConf);
+  }
+
+  // Force this to be true for the high level producer
+  conf = shallowCopy(conf);
+  conf.dr_cb = true;
+
+  // producer is an initialized consumer object
+  // @see NodeKafka::Producer::Init
+  Producer.call(this, conf, topicConf);
+  var self = this;
+
+  // Add a delivery emitter to the producer
+  this._hl = {
+    deliveryEmitter: new EventEmitter(),
+    messageId: 0,
+    // Special logic for polling. We use a reference counter to know when we need
+    // to be doing it and when we can stop. This means when we go into fast polling
+    // mode we don't need to do multiple calls to poll since they all will yield
+    // the same result
+    pollingRefTimeout: null,
+  };
+
+  // Add the polling ref counter to the class which ensures we poll when we go active
+  this._hl.pollingRef = new RefCounter(function() {
+    self._hl.pollingRefTimeout = setInterval(function() {
+      try {
+        self.poll();
+      } catch (e) {
+        if (!self._isConnected) {
+          // If we got disconnected for some reason there is no point
+          // in polling anymore
+          clearInterval(self._hl.pollingRefTimeout);
+        }
+      }
+    }, 1);
+  }, function() {
+    clearInterval(self._hl.pollingRefTimeout);
+  });
+
+  // Default poll interval. More sophisticated polling is also done in create rule method
+  this.setPollInterval(1000);
+
+  // Listen to all delivery reports to propagate elements with a _message_id to the emitter
+  this.on('delivery-report', function(err, report) {
+    if (report.opaque && report.opaque.__message_id !== undefined) {
+      self._hl.deliveryEmitter.emit(report.opaque.__message_id, err, report.offset);
+    }
+  });
+
+  // Save old produce here since we are making some modifications for it
+  this._oldProduce = this.produce;
+  this.produce = this._modifiedProduce;
+
+  // Serializer information
+  this.keySerializer = noopSerializer;
+  this.valueSerializer = noopSerializer;
+}
+
+/**
+ * Produce a message to Kafka asynchronously.
+ *
+ * This is the method mainly used in this class. Use it to produce
+ * a message to Kafka.
+ *
+ * When this is sent off, and you recieve your callback, the assurances afforded
+ * to you will be equal to those provided by your ack level.
+ *
+ * @param {string} topic - The topic name to produce to.
+ * @param {number|null} partition - The partition number to produce to.
+ * @param {Buffer|null} message - The message to produce.
+ * @param {string} key - The key associated with the message.
+ * @param {number|null} timestamp - Timestamp to send with the message.
+ * @param {object} headers - A list of custom key value pairs that provide message metadata.
+ * @param {function} callback - Callback to call when the delivery report is recieved.
+ * @throws {LibrdKafkaError} - Throws a librdkafka error if it failed.
+ * @return {boolean} - returns an error if it failed, or true if not
+ * @see Producer#produce
+ */
+HighLevelProducer.prototype._modifiedProduce = function(topic, partition, message, key, timestamp, headers, callback) {
+  // headers are optional
+  if (arguments.length === 6) {
+    callback = headers;
+    headers = undefined;
+  }
+
+  // Add the message id
+  var opaque = {
+    __message_id: this._hl.messageId++,
+  };
+
+  this._hl.pollingRef.increment();
+
+  var self = this;
+
+  var resolvedSerializedValue;
+  var resolvedSerializedKey;
+  var calledBack = false;
+
+  // Actually do the produce with new key and value based on deserialized
+  // results
+  function doProduce(v, k) {
+    try {
+      var r = self._oldProduce(topic, partition,
+        v, k,
+        timestamp, opaque, headers);
+
+      self._hl.deliveryEmitter.once(opaque.__message_id, function(err, offset) {
+        self._hl.pollingRef.decrement();
+        setImmediate(function() {
+          // Offset must be greater than or equal to 0 otherwise it is a null offset
+          // Possibly because we have acks off
+          callback(err, offset >= 0 ? offset : null);
+        });
+      });
+
+      return r;
+    } catch (e) {
+      callback(e);
+    }
+  }
+
+  function produceIfComplete() {
+    if (resolvedSerializedKey !== undefined && resolvedSerializedValue !== undefined) {
+      doProduce(resolvedSerializedValue, resolvedSerializedKey);
+    }
+  }
+
+  // To run on a promise if returned by the serializer
+  function finishSerializedValue(v) {
+    if (!calledBack) {
+      resolvedSerializedValue = v;
+      produceIfComplete();
+    }
+  }
+
+  // To run on a promise of returned by the serializer
+  function finishSerializedKey(k) {
+    resolvedSerializedKey = k;
+
+    if (!calledBack) {
+      produceIfComplete();
+    }
+  }
+
+  function failSerializedValue(err) {
+    if (!calledBack) {
+      calledBack = true;
+      callback(err);
+    }
+  }
+
+  function failSerializedKey(err) {
+    if (!calledBack) {
+      calledBack = true;
+      callback(err);
+    }
+  }
+
+  function valueSerializerCallback(err, v) {
+    if (err) {
+      failSerializedValue(err);
+    } else {
+      finishSerializedValue(v);
+    }
+  }
+
+  function keySerializerCallback(err, v) {
+    if (err) {
+      failSerializedKey(err);
+    } else {
+      finishSerializedKey(v);
+    }
+  }
+
+  try {
+    if (this.valueSerializer.async) {
+      // If this is async we need to give it a callback
+      this.valueSerializer.apply(message, valueSerializerCallback);
+    } else {
+      var serializedValue = this.valueSerializer.apply(message);
+      // Check if we were returned a promise in order to support promise behavior
+      if (serializedValue &&
+          typeof serializedValue.then === 'function' &&
+          typeof serializedValue.catch === 'function') {
+        // This is a promise. We need to hook into its then and catch
+        serializedValue.then(finishSerializedValue).catch(failSerializedValue);
+      } else {
+        resolvedSerializedValue = serializedValue;
+      }
+    }
+
+    if (this.keySerializer.async) {
+      // If this is async we need to give it a callback
+      this.keySerializer.apply(key, keySerializerCallback);
+    } else {
+      var serializedKey = this.keySerializer.apply(key);
+      // Check if we were returned a promise in order to support promise behavior
+      if (serializedKey &&
+          typeof serializedKey.then === 'function' &&
+          typeof serializedKey.catch === 'function') {
+        // This is a promise. We need to hook into its then and catch
+        serializedKey.then(finishSerializedKey).catch(failSerializedKey);
+      } else {
+        resolvedSerializedKey = serializedKey;
+      }
+    }
+
+    // Only do the produce here if we are complete. That is, if the key
+    // and value have been serialized.
+    produceIfComplete();
+  } catch (e) {
+    setImmediate(function() {
+      calledBack = true;
+      callback(e);
+    });
+  }
+};
+
+/**
+ * Set the key serializer
+ *
+ * This allows the value inside the produce call to differ from the value of the
+ * value actually produced to kafka. Good if, for example, you want to serialize
+ * it to a particular format.
+ */
+HighLevelProducer.prototype.setKeySerializer = function(serializer) {
+  this.keySerializer = createSerializer(serializer);
+};
+
+/**
+ * Set the value serializer
+ *
+ * This allows the value inside the produce call to differ from the value of the
+ * value actually produced to kafka. Good if, for example, you want to serialize
+ * it to a particular format.
+ */
+HighLevelProducer.prototype.setValueSerializer = function(serializer) {
+  this.valueSerializer = createSerializer(serializer);
+};

package/lib/producer-stream.js

@@ -0,0 +1,307 @@
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+'use strict';
+
+module.exports = ProducerStream;
+
+var Writable = require('stream').Writable;
+var util = require('util');
+var ErrorCode = require('./error').codes;
+
+util.inherits(ProducerStream, Writable);
+
+/**
+ * Writable stream integrating with the Kafka Producer.
+ *
+ * This class is used to write data to Kafka in a streaming way. It takes
+ * buffers of data and puts them into the appropriate Kafka topic. If you need
+ * finer control over partitions or keys, this is probably not the class for
+ * you. In that situation just use the Producer itself.
+ *
+ * The stream detects if Kafka is already connected. You can safely begin
+ * writing right away.
+ *
+ * This stream does not operate in Object mode and can only be given buffers.
+ *
+ * @param {Producer} producer - The Kafka Producer object.
+ * @param {array} topics - Array of topics
+ * @param {object} options - Topic configuration.
+ * @constructor
+ * @extends stream.Writable
+ */
+function ProducerStream(producer, options) {
+  if (!(this instanceof ProducerStream)) {
+    return new ProducerStream(producer, options);
+  }
+
+  if (options === undefined) {
+    options = {};
+  } else if (typeof options === 'string') {
+    options = { encoding: options };
+  } else if (options === null || typeof options !== 'object') {
+    throw new TypeError('"streamOptions" argument must be a string or an object');
+  }
+
+  if (!options.objectMode && !options.topic) {
+    throw new TypeError('ProducerStreams not using objectMode must provide a topic to produce to.');
+  }
+
+  if (options.objectMode !== true) {
+    this._write = this._write_buffer;
+  } else {
+    this._write = this._write_message;
+  }
+
+  Writable.call(this, options);
+
+  this.producer = producer;
+  this.topicName = options.topic;
+
+  this.autoClose = options.autoClose === undefined ? true : !!options.autoClose;
+  this.connectOptions = options.connectOptions || {};
+
+  this.producer.setPollInterval(options.pollInterval || 1000);
+
+  if (options.encoding) {
+    this.setDefaultEncoding(options.encoding);
+  }
+
+  // Connect to the producer. Unless we are already connected
+  if (!this.producer.isConnected()) {
+    this.connect(this.connectOptions);
+  }
+
+  var self = this;
+
+  this.once('finish', function() {
+    if (this.autoClose) {
+      this.close();
+    }
+  });
+
+}
+
+ProducerStream.prototype.connect = function(options) {
+  this.producer.connect(options, function(err, data) {
+    if (err) {
+      this.emit('error', err);
+      return;
+    }
+
+  }.bind(this));
+};
+
+/**
+ * Internal stream write method for ProducerStream when writing buffers.
+ *
+ * This method should never be called externally. It has some recursion to
+ * handle cases where the producer is not yet connected.
+ *
+ * @param  {buffer} chunk - Chunk to write.
+ * @param  {string} encoding - Encoding for the buffer
+ * @param  {Function} cb - Callback to call when the stream is done processing
+ * the data.
+ * @private
+ * @see https://github.com/nodejs/node/blob/master/lib/fs.js#L1901
+ */
+ProducerStream.prototype._write_buffer = function(data, encoding, cb) {
+  if (!(data instanceof Buffer)) {
+    this.emit('error', new Error('Invalid data. Can only produce buffers'));
+    return;
+  }
+
+  var self = this;
+
+  if (!this.producer.isConnected()) {
+    this.producer.once('ready', function() {
+      self._write(data, encoding, cb);
+    });
+    return;
+  }
+
+  try {
+    this.producer.produce(self.topicName, null, data, null);
+    setImmediate(cb);
+  } catch (e) {
+    if (ErrorCode.ERR__QUEUE_FULL === e.code) {
+      // Poll for good measure
+      self.producer.poll();
+
+      // Just delay this thing a bit and pass the params
+      // backpressure will get exerted this way.
+      setTimeout(function() {
+        self._write(data, encoding, cb);
+      }, 500);
+    } else {
+      if (self.autoClose) {
+        self.close();
+      }
+      setImmediate(function() {
+        cb(e);
+      });
+    }
+  }
+};
+
+/**
+ * Internal stream write method for ProducerStream when writing objects.
+ *
+ * This method should never be called externally. It has some recursion to
+ * handle cases where the producer is not yet connected.
+ *
+ * @param  {object} message - Message to write.
+ * @param  {string} encoding - Encoding for the buffer
+ * @param  {Function} cb - Callback to call when the stream is done processing
+ * the data.
+ * @private
+ * @see https://github.com/nodejs/node/blob/master/lib/fs.js#L1901
+ */
+ProducerStream.prototype._write_message = function(message, encoding, cb) {
+  var self = this;
+
+  if (!this.producer.isConnected()) {
+    this.producer.once('ready', function() {
+      self._write(message, encoding, cb);
+    });
+    return;
+  }
+
+  try {
+    this.producer.produce(message.topic, message.partition, message.value, message.key, message.timestamp, message.opaque, message.headers);
+    setImmediate(cb);
+  } catch (e) {
+    if (ErrorCode.ERR__QUEUE_FULL === e.code) {
+      // Poll for good measure
+      self.producer.poll();
+
+      // Just delay this thing a bit and pass the params
+      // backpressure will get exerted this way.
+      setTimeout(function() {
+        self._write(message, encoding, cb);
+      }, 500);
+    } else {
+      if (self.autoClose) {
+        self.close();
+      }
+      setImmediate(function() {
+        cb(e);
+      });
+    }
+  }
+};
+
+function writev(producer, topic, chunks, cb) {
+
+  // @todo maybe a produce batch method?
+  var doneCount = 0;
+  var err = null;
+  var chunk = null;
+
+  function maybeDone(e) {
+    if (e) {
+      err = e;
+    }
+    doneCount ++;
+    if (doneCount === chunks.length) {
+      cb(err);
+    }
+  }
+
+  function retry(restChunks) {
+    // Poll for good measure
+    producer.poll();
+
+    // Just delay this thing a bit and pass the params
+    // backpressure will get exerted this way.
+    setTimeout(function() {
+      writev(producer, topic, restChunks, cb);
+    }, 500);
+  }
+
+  for (var i = 0; i < chunks.length; i++) {
+    chunk = chunks[i];
+
+    try {
+      if (Buffer.isBuffer(chunk)) {
+        producer.produce(topic, null, chunk, null);
+      } else {
+        producer.produce(chunk.topic, chunk.partition, chunk.value, chunk.key, chunk.timestamp, chunk.opaque, chunk.headers);
+      }
+      maybeDone();
+    } catch (e) {
+      if (ErrorCode.ERR__QUEUE_FULL === e.code) {
+        retry(chunks.slice(i));
+      } else {
+        cb(e);
+      }
+      break;
+    }
+  }
+
+}
+
+ProducerStream.prototype._writev = function(data, cb) {
+  if (!this.producer.isConnected()) {
+    this.once('ready', function() {
+      this._writev(data, cb);
+    });
+    return;
+  }
+
+  var self = this;
+  var len = data.length;
+  var chunks = new Array(len);
+  var size = 0;
+
+  for (var i = 0; i < len; i++) {
+    var chunk = data[i].chunk;
+
+    chunks[i] = chunk;
+    size += chunk.length;
+  }
+
+  writev(this.producer, this.topicName, chunks, function(err) {
+    if (err) {
+      self.close();
+      cb(err);
+      return;
+    }
+    cb();
+  });
+
+};
+
+ProducerStream.prototype.close = function(cb) {
+  var self = this;
+  if (cb) {
+    this.once('close', cb);
+  }
+
+  // Use interval variables in here
+  if (self.producer._isConnected) {
+    self.producer.disconnect(function() {
+      // Previously this set the producer to null. I'm not sure there is any benefit
+      // to that other than I guess helping flag it for GC?
+      // https://github.com/Blizzard/node-rdkafka/issues/344
+      close();
+    });
+  } else if (self.producer._isConnecting){
+    self.producer.once('ready', function() {
+      // Don't pass CB this time because it has already been passed
+      self.close();
+    });
+  } else {
+    setImmediate(close);
+  }
+
+  function close() {
+    self.emit('close');
+  }
+};