@ohos-ports/confluentinc-kafka-javascript 1.10.0-beta.1

package/lib/producer.js

@@ -0,0 +1,405 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *           (c) 2024 Confluent, Inc.
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = Producer;
+
+var Client = require('./client');
+
+var util = require('util');
+var Kafka = require('../librdkafka.js');
+var ProducerStream = require('./producer-stream');
+var LibrdKafkaError = require('./error');
+var shallowCopy = require('./util').shallowCopy;
+
+util.inherits(Producer, Client);
+
+/**
+ * Producer class for sending messages to Kafka
+ *
+ * This is the main entry point for writing data to Kafka. You
+ * configure this like you do any other client, with a global
+ * configuration and default topic configuration.
+ *
+ * Once you instantiate this object, you need to connect to it first.
+ * This allows you to get the metadata and make sure the connection
+ * can be made before you depend on it. After that, problems with
+ * the connection will by brought down by using poll, which automatically
+ * runs when a transaction is made on the object.
+ *
+ * Methods on a Producer will throw a [LibrdKafkaError]{@link RdKafka.LibrdKafkaError}
+ * on failure.
+ *
+ * @param {object} conf - Key value pairs to configure the producer
+ * @param {object?} topicConf - Key value pairs to create a default
+ *                              topic configuration
+ * @extends RdKafka.Client
+ * @constructor
+ * @memberof RdKafka
+ * @see [Producer example]{@link https://github.com/confluentinc/confluent-kafka-javascript/blob/master/examples/node-rdkafka/producer.md}
+ */
+function Producer(conf, topicConf) {
+  if (!(this instanceof Producer)) {
+    return new Producer(conf, topicConf);
+  }
+
+  conf = shallowCopy(conf);
+  topicConf = shallowCopy(topicConf);
+
+  /**
+   * Producer message. This is sent to the wrapper, not received from it
+   *
+   * @typedef {object} Producer~Message
+   * @property {string|buffer} message - The buffer to send to Kafka.
+   * @property {Topic} topic - The Kafka topic to produce to.
+   * @property {number} partition - The partition to produce to. Defaults to
+   * the partitioner
+   * @property {string} key - The key string to use for the message.
+   * @see Consumer~Message
+   */
+
+  var gTopic = conf.topic || false;
+  var gPart = conf.partition || null;
+  var dr_cb = conf.dr_cb || null;
+  var dr_msg_cb = conf.dr_msg_cb || null;
+
+  // delete keys we don't want to pass on
+  delete conf.topic;
+  delete conf.partition;
+
+  delete conf.dr_cb;
+  delete conf.dr_msg_cb;
+
+  // client is an initialized producer object
+  // @see NodeKafka::Producer::Init
+  Client.call(this, conf, Kafka.Producer, topicConf);
+
+  // Delete these keys after saving them in vars
+  this.globalConfig = conf;
+  this.topicConfig = topicConf;
+  this.defaultTopic = gTopic || null;
+  this.defaultPartition = gPart === null ? -1 : gPart;
+
+  this.sentMessages = 0;
+
+  this.pollInterval = undefined;
+
+  if (dr_msg_cb || dr_cb) {
+    this._cb_configs.event.delivery_cb =  function(err, report) {
+      if (err) {
+        err = LibrdKafkaError.create(err);
+      }
+      this.emit('delivery-report', err, report);
+    }.bind(this);
+    this._cb_configs.event.delivery_cb.dr_msg_cb = !!dr_msg_cb;
+
+    if (typeof dr_cb === 'function') {
+      this.on('delivery-report', dr_cb);
+    }
+
+  }
+}
+
+/**
+ * Produce a message to Kafka asynchronously.
+ *
+ * This is the method mainly used in this class. Use it to produce
+ * a message to Kafka.
+ *
+ * When this is sent off, there is no guarantee it is delivered, as the method
+ * returns immediately after queuing the message in the library.
+ *
+ * If you need guaranteed delivery, change your `acks` settings and use delivery reports.
+ *
+ * @param {string} topic - The topic name to produce to.
+ * @param {number|null} partition - The partition number to produce to.
+ * @param {Buffer|null} message - The message to produce.
+ * @param {string} key - The key associated with the message.
+ * @param {number|null} timestamp - Timestamp to send with the message.
+ * @param {object} opaque - An object you want passed along with this message, if provided.
+ *                          This will be available in the delivery report
+ * @param {object} headers - A list of custom key value pairs that provide message metadata.
+ * @return {boolean} - throws an error if it failed, or returns true if not.
+ */
+Producer.prototype.produce = function(topic, partition, message, key, timestamp, opaque, headers) {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+
+  // I have removed support for using a topic object. It is going to be removed
+  // from librdkafka soon, and it causes issues with shutting down
+  if (!topic || typeof topic !== 'string') {
+    throw new TypeError('"topic" must be a string');
+  }
+
+  this.sentMessages++;
+
+  partition = partition === null ? this.defaultPartition : partition;
+
+  return this._errorWrap(
+    this._client.produce(topic, partition, message, key, timestamp, opaque, headers));
+
+};
+
+/**
+ * Create a write stream interface for a producer.
+ *
+ * This stream does not run in object mode. It only takes buffers of data.
+ *
+ * @param {object} conf - Key value pairs to configure the producer.
+ * @param {object} topicConf - Key value pairs to create a default topic configuration.
+ * @param {object} streamOptions - Stream options.
+ * @return {RdKafka.ProducerStream} - returns the write stream for writing to Kafka.
+ */
+Producer.createWriteStream = function(conf, topicConf, streamOptions) {
+  var producer = new Producer(conf, topicConf);
+  return new ProducerStream(producer, streamOptions);
+};
+
+/**
+ * Poll for events.
+ *
+ * We need to run poll in order to learn about new events that have occurred.
+ * This is not done automatically when we produce, so we need to run
+ * it manually, or set the producer to automatically poll using
+ * {@link RdKafka.Producer#setPollInterval} or {@link RdKafka.Producer#setPollInBackground}.
+ *
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.poll = function() {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+  this._client.poll();
+  return this;
+};
+
+/**
+ * Set automatic polling for events.
+ *
+ * We need to run poll in order to learn about new events that have occurred.
+ * If you would like this done on an interval with disconnects and reconnections
+ * managed, you can do it here.
+ *
+ * @param {number} interval - Interval, in milliseconds, to poll for.
+ *
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.setPollInterval = function(interval) {
+  // If we already have a poll interval we need to stop it
+  if (this.pollInterval) {
+    clearInterval(this.pollInterval);
+    this.pollInterval = undefined;
+  }
+
+  if (interval === 0) {
+    // If the interval was set to 0, bail out. We don't want to process this.
+    // If there was an interval previously set, it has been removed.
+    return;
+  }
+
+  var self = this;
+
+  // Now we want to make sure we are connected.
+  if (!this._isConnected) {
+    // If we are not, execute this once the connection goes through.
+    this.once('ready', function() {
+      self.setPollInterval(interval);
+    });
+    return;
+  }
+
+  // We know we are connected at this point.
+  // Unref this interval
+  this.pollInterval = setInterval(function() {
+    try {
+      self.poll();
+    } catch {
+      // We can probably ignore errors here as far as broadcasting.
+      // Disconnection issues will get handled below
+    }
+  }, interval).unref();
+
+  // Handle disconnections
+  this.once('disconnected', function() {
+    // Just rerun this function with interval 0. If any
+    // poll interval is set, this will remove it
+    self.setPollInterval(0);
+  });
+
+  return this;
+};
+
+/**
+ * Set automatic polling for events on the librdkafka background thread.
+ *
+ * This provides several advantages over `setPollInterval`, as the polling
+ * does not happen on the event loop, but on the C thread spawned by librdkafka,
+ * and can be more efficient for high-throughput producers.
+ *
+ * If set = true, this will disable any polling interval set by `setPollInterval`.
+ *
+ * @param {boolean} set Whether to poll in the background or not.
+ * @returns {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.setPollInBackground = function(set) {
+  if (set) {
+    this.setPollInterval(0); // Clear poll interval from JS.
+  }
+  this._client.setPollInBackground(set);
+  return this;
+};
+
+/**
+ * Flush the producer,
+ *
+ * The producer accumulates messages upto `linger.ms` on an internal buffer.
+ * This method flushes on the buffer immediately. Do this before disconnects, usually.
+ *
+ * @param {number?} timeout - Number of milliseconds to try to flush before giving up.
+ * @param {function?} callback - Callback to fire when the flush is done.
+ *
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.flush = function(timeout, callback) {
+  if (!this._isConnected) {
+    throw new Error('Producer not connected');
+  }
+
+  if (timeout === undefined || timeout === null) {
+    timeout = 500;
+  }
+
+  this._client.flush(timeout, function(err) {
+    if (err) {
+      err = LibrdKafkaError.create(err);
+    }
+
+    if (callback) {
+      callback(err);
+    }
+  });
+  return this;
+};
+
+/**
+ * Save the base disconnect method here so we can overwrite it and add a flush.
+ * @private
+ */
+Producer.prototype._disconnect = Producer.prototype.disconnect;
+
+/**
+ * Disconnect the producer.
+ *
+ * Flush everything on the internal librdkafka producer buffer. Then disconnect.
+ *
+ * @param {number?} timeout - Number of milliseconds to try to flush before giving up, defaults to 5 seconds.
+ * @param {function?} cb - The callback to fire when disconnected.
+ */
+Producer.prototype.disconnect = function(timeout, cb) {
+  var self = this;
+  var timeoutInterval = 5000;
+
+  if (typeof timeout === 'function') {
+    cb = timeout;
+  } else {
+    timeoutInterval = timeout;
+  }
+
+  this.flush(timeoutInterval, function() {
+    self._disconnect(cb);
+  });
+};
+
+/**
+ * Initialize transactions for this producer instance.
+ *
+ * Initialize transactions, this must only be performed once per transactional producer,
+ * before it can be used.
+ *
+ * @param {number} timeout - Number of milliseconds to try to initialize before giving up, defaults to 5 seconds.
+ * @param {function} cb - Callback to fire when operation is completed.
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.initTransactions = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.initTransactions(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Begin a transaction.
+ *
+ * `initTransaction` must have been called successfully (once) before this function is called.
+ *
+ * There can only be one ongoing transaction at a time.
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.beginTransaction = function(cb) {
+  this._client.beginTransaction(function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Commit the current transaction (as started with `beginTransaction`).
+ *
+ * @param {number} timeout - Number of milliseconds to try to commit before giving up, defaults to 5 seconds.
+ * @param {function} cb - Callback to fire when operation is completed.
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.commitTransaction = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.commitTransaction(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Aborts the ongoing transaction (as started with `beginTransaction`).
+ *
+ * @param {number} timeout - Number of milliseconds to try to abort, defaults to 5 seconds.
+ * @param {function} cb - Callback to fire when operation is completed.
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.abortTransaction = function(timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.abortTransaction(timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};
+
+/**
+ * Send the current offsets of the consumer to the ongoing transaction.
+ *
+ * @param {RdKafka.TopicPartition[]} offsets - An array of topic-partitions with offsets filled in.
+ * @param {RdKafka.KafkaConsumer} consumer - An instance of the consumer to send offsets for.
+ * @param {number} timeout - Number of milliseconds to try to send offsets, defaults to 5 seconds.
+ * @param {function} cb - Callback to return when operation is completed.
+ * @return {RdKafka.Producer} - returns itself.
+ */
+Producer.prototype.sendOffsetsToTransaction = function(offsets, consumer, timeout, cb) {
+  if (typeof timeout === 'function') {
+    cb = timeout;
+    timeout = 5000;
+  }
+  this._client.sendOffsetsToTransaction(offsets, consumer.getClient(), timeout, function(err) {
+    cb(err ? LibrdKafkaError.create(err) : err);
+  });
+};

package/lib/rdkafka.js

@@ -0,0 +1,44 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *           (c) 2024 Confluent, Inc.
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var KafkaConsumer = require('./kafka-consumer');
+var Producer = require('./producer');
+var HighLevelProducer = require('./producer/high-level-producer');
+var error = require('./error');
+var util = require('util');
+var lib = require('../librdkafka');
+var Topic = require('./topic');
+var Admin = require('./admin');
+var features = lib.features().split(',');
+
+/**
+ * Namespace for non-promifisied, callback-based Kafka client API.
+ * @namespace RdKafka
+ */
+module.exports = {
+  Consumer: util.deprecate(KafkaConsumer, 'Use KafkaConsumer instead. This may be changed in a later version'),
+  Producer: Producer,
+  HighLevelProducer: HighLevelProducer,
+  AdminClient: Admin,
+  KafkaConsumer: KafkaConsumer,
+  createReadStream: KafkaConsumer.createReadStream,
+  createWriteStream: Producer.createWriteStream,
+  CODES: {
+    ERRORS: error.codes,
+  },
+  Topic: Topic,
+  features: features,
+  librdkafkaVersion: lib.librdkafkaVersion,
+  IsolationLevel: Admin.IsolationLevel,
+  OffsetSpec: Admin.OffsetSpec,
+  ConsumerGroupStates: Admin.ConsumerGroupStates,
+  ConsumerGroupTypes: Admin.ConsumerGroupTypes,
+  AclOperationTypes: Admin.AclOperationTypes,
+};

package/lib/tools/ref-counter.js

@@ -0,0 +1,53 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+module.exports = RefCounter;
+
+/**
+ * Ref counter class.
+ *
+ * Is used to basically determine active/inactive and allow callbacks that
+ * hook into each.
+ *
+ * For the producer, it is used to begin rapid polling after a produce until
+ * the delivery report is dispatched.
+ * @private
+ */
+function RefCounter(onActive, onPassive) {
+  this.context = {};
+  this.onActive = onActive;
+  this.onPassive = onPassive;
+  this.currentValue = 0;
+  this.isRunning = false;
+}
+
+/**
+ * Increment the ref counter
+ */
+RefCounter.prototype.increment = function() {
+  this.currentValue += 1;
+
+  // If current value exceeds 0, activate the start
+  if (this.currentValue > 0 && !this.isRunning) {
+    this.isRunning = true;
+    this.onActive(this.context);
+  }
+};
+
+/**
+ * Decrement the ref counter
+ */
+RefCounter.prototype.decrement = function() {
+  this.currentValue -= 1;
+
+  if (this.currentValue <= 0 && this.isRunning) {
+    this.isRunning = false;
+    this.onPassive(this.context);
+  }
+};

package/lib/topic-partition.js

@@ -0,0 +1,91 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Topic = require('./topic');
+
+module.exports = TopicPartition;
+
+/**
+ * Map an array of topic partition js objects to real topic partition objects.
+ *
+ * @param array The array of topic partition raw objects to map to topic
+ *              partition objects
+ * @private
+ */
+TopicPartition.map = function(array) {
+  return array.map(function(element) {
+    return TopicPartition.create(element);
+  });
+};
+
+/**
+ * Take a topic partition javascript object and convert it to the class.
+ * The class will automatically convert offset identifiers to special constants
+ *
+ * @param element The topic partition raw javascript object
+ * @private
+ */
+TopicPartition.create = function(element) {
+  // Just ensure we take something that can have properties. The topic partition
+  // class will
+  element = element || {};
+  return new TopicPartition(element.topic, element.partition, element.offset);
+};
+
+/**
+ * Create a topic partition. Just does some validation and decoration
+ * on topic partitions provided.
+ *
+ * Goal is still to behave like a plain javascript object but with validation
+ * and potentially some extra methods
+ * @private
+ */
+function TopicPartition(topic, partition, offset) {
+  if (!(this instanceof TopicPartition)) {
+    return new TopicPartition(topic, partition, offset);
+  }
+
+  // Validate that the elements we are iterating over are actual topic partition
+  // js objects. They do not need an offset, but they do need partition
+  if (!topic) {
+    throw new TypeError('"topic" must be a string and must be set');
+  }
+
+  if (partition === null || partition === undefined) {
+    throw new TypeError('"partition" must be a number and must set');
+  }
+
+  // We can just set topic and partition as they stand.
+  this.topic = topic;
+  this.partition = partition;
+
+  if (offset === undefined || offset === null) {
+    this.offset = Topic.OFFSET_STORED;
+  } else if (typeof offset === 'string') {
+    switch (offset.toLowerCase()) {
+      case 'earliest':
+      case 'beginning':
+        this.offset = Topic.OFFSET_BEGINNING;
+        break;
+      case 'latest':
+      case 'end':
+        this.offset = Topic.OFFSET_END;
+        break;
+      case 'stored':
+        this.offset = Topic.OFFSET_STORED;
+        break;
+      default:
+        throw new TypeError('"offset", if provided as a string, must be beginning, end, or stored.');
+    }
+  } else if (typeof offset === 'number') {
+    this.offset = offset;
+  } else {
+    throw new TypeError('"offset" must be a special string or number if it is set');
+  }
+}

package/lib/topic.js

@@ -0,0 +1,43 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var librdkafka = require('../librdkafka');
+
+module.exports = Topic;
+
+var topicKey = 'RdKafka::Topic::';
+var topicKeyLength = topicKey.length;
+
+// Take all of the topic special codes from librdkafka and add them
+// to the object
+// You can find this list in the C++ code at
+// https://github.com/confluentinc/librdkafka/blob/master/src-cpp/rdkafkacpp.h#L1250
+for (var key in librdkafka.topic) {
+  // Skip it if it doesn't start with ErrorCode
+  if (key.indexOf('RdKafka::Topic::') !== 0) {
+    continue;
+  }
+
+  // Replace/add it if there are any discrepancies
+  var newKey = key.substring(topicKeyLength);
+  Topic[newKey] = librdkafka.topic[key];
+}
+
+/**
+ * Create a topic. Just returns the string you gave it right now.
+ *
+ * Looks like a class, but all it does is return the topic name.
+ * This is so that one day if there are interface changes that allow
+ * different use of topic parameters, we can just add to this constructor and
+ * have it return something richer
+ * @private
+ */
+function Topic(topicName) {
+  return topicName;
+}

package/lib/util.js

@@ -0,0 +1,55 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *               2024 Confluent, Inc.
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var util = module.exports = {};
+
+util.shallowCopy = function (obj) {
+
+  if (!util.isObject(obj)) { return obj; }
+
+  var copy = {};
+
+  for (var k in obj) {
+    if (Object.hasOwn(obj, k)) {
+      copy[k] = obj[k];
+    }
+  }
+
+  return copy;
+};
+
+util.isObject = function (obj) {
+  return obj && typeof obj === 'object';
+};
+
+// Convert Map or object to a list of [key, value, key, value...].
+util.dictToStringList = function (mapOrObject) {
+  let list = null;
+  if (mapOrObject && (mapOrObject instanceof Map)) {
+    list =
+      Array
+        .from(mapOrObject).reduce((acc, [key, value]) => {
+          acc.push(key, value);
+          return acc;
+        }, [])
+        .map(v => String(v));
+  } else if (util.isObject(mapOrObject)) {
+    list =
+      Object
+        .entries(mapOrObject).reduce((acc, [key, value]) => {
+          acc.push(key, value);
+          return acc;
+        }, [])
+        .map(v => String(v));
+  }
+  return list;
+};
+
+util.bindingVersion = '1.10.0';

package/librdkafka.js

@@ -0,0 +1,12 @@
+/*
+ * confluent-kafka-javascript - Node.js wrapper  for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016-2023 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var kafka = require('bindings')('confluent-kafka-javascript');
+
+module.exports = kafka;