@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/0114-sticky_partitioning.cpp

@@ -0,0 +1,176 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2020-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Test sticky.partitioning.linger.ms producer property.
+ *
+ */
+
+#include <iostream>
+#include <fstream>
+#include <iterator>
+#include <string>
+#include "testcpp.h"
+#include "test.h"
+
+/**
+ * @brief Specify sticky.partitioning.linger.ms and check consumed
+ * messages to verify it takes effect.
+ */
+static void do_test_sticky_partitioning(int sticky_delay) {
+  std::string topic = Test::mk_topic_name(__FILE__, 1);
+  Test::create_topic_wait_exists(NULL, topic.c_str(), 3, 1, 5000);
+
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 0);
+
+  Test::conf_set(conf, "sticky.partitioning.linger.ms",
+                 tostr() << sticky_delay);
+
+  std::string errstr;
+  RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
+  if (!p)
+    Test::Fail("Failed to create Producer: " + errstr);
+
+  RdKafka::Consumer *c = RdKafka::Consumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create Consumer: " + errstr);
+  delete conf;
+
+  RdKafka::Topic *t = RdKafka::Topic::create(c, topic, NULL, errstr);
+  if (!t)
+    Test::Fail("Failed to create Topic: " + errstr);
+
+  c->start(t, 0, RdKafka::Topic::OFFSET_BEGINNING);
+  c->start(t, 1, RdKafka::Topic::OFFSET_BEGINNING);
+  c->start(t, 2, RdKafka::Topic::OFFSET_BEGINNING);
+
+  const int msgrate = 100;
+  const int msgsize = 10;
+
+  /* Produce messages */
+  char val[msgsize];
+  memset(val, 'a', msgsize);
+
+  /* produce for for seconds at 100 msgs/sec */
+  for (int s = 0; s < 4; s++) {
+    int64_t end_wait = test_clock() + (1 * 1000000);
+
+    for (int i = 0; i < msgrate; i++) {
+      RdKafka::ErrorCode err = p->produce(topic, RdKafka::Topic::PARTITION_UA,
+                                          RdKafka::Producer::RK_MSG_COPY, val,
+                                          msgsize, NULL, 0, -1, NULL);
+      if (err)
+        Test::Fail("Produce failed: " + RdKafka::err2str(err));
+    }
+
+    while (test_clock() < end_wait)
+      p->poll(100);
+  }
+
+  Test::Say(tostr() << "Produced " << 4 * msgrate << " messages\n");
+  p->flush(5 * 1000);
+
+  /* Consume messages */
+  int partition_msgcnt[3]   = {0, 0, 0};
+  int num_partitions_active = 0;
+  int i                     = 0;
+
+  int64_t end_wait = test_clock() + (5 * 1000000);
+  while (test_clock() < end_wait) {
+    RdKafka::Message *msg = c->consume(t, i, 5);
+
+    switch (msg->err()) {
+    case RdKafka::ERR__TIMED_OUT:
+      i++;
+      if (i > 2)
+        i = 0;
+      break;
+
+    case RdKafka::ERR_NO_ERROR:
+      partition_msgcnt[msg->partition()]++;
+      break;
+
+    default:
+      Test::Fail("Consume error: " + msg->errstr());
+      break;
+    }
+
+    delete msg;
+  }
+
+  c->stop(t, 0);
+  c->stop(t, 1);
+  c->stop(t, 2);
+
+  for (int i = 0; i < 3; i++) {
+    /* Partitions must receive 100+ messages to be deemed 'active'. This
+     * is because while topics are being updated, it is possible for some
+     * number of messages to be partitioned to joining partitions before
+     * they become available. This can cause some initial turnover in
+     * selecting a sticky partition. This behavior is acceptable, and is
+     * not important for the purpose of this segment of the test. */
+
+    if (partition_msgcnt[i] > (msgrate - 1))
+      num_partitions_active++;
+  }
+
+  Test::Say("Partition Message Count: \n");
+  for (int i = 0; i < 3; i++) {
+    Test::Say(tostr() << " " << i << ": " << partition_msgcnt[i] << "\n");
+  }
+
+  /* When sticky.partitioning.linger.ms is long (greater than expected
+   * length of run), one partition should be sticky and receive messages. */
+  if (sticky_delay == 5000 && num_partitions_active > 1)
+    Test::Fail(tostr() << "Expected only 1 partition to receive msgs" << " but "
+                       << num_partitions_active
+                       << " partitions received msgs.");
+
+  /* When sticky.partitioning.linger.ms is short (sufficiently smaller than
+   * length of run), it is extremely likely that all partitions are sticky
+   * at least once and receive messages. */
+  if (sticky_delay == 1000 && num_partitions_active <= 1)
+    Test::Fail(tostr() << "Expected more than one partition to receive msgs"
+                       << " but only " << num_partitions_active
+                       << " partition received msgs.");
+
+  delete t;
+  delete p;
+  delete c;
+}
+
+extern "C" {
+int main_0114_sticky_partitioning(int argc, char **argv) {
+  /* long delay (5 secs) */
+  do_test_sticky_partitioning(5000);
+  /* short delay (0.001 secs) */
+  do_test_sticky_partitioning(1);
+  return 0;
+}
+}

package/deps/librdkafka/tests/0115-producer_auth.cpp

@@ -0,0 +1,182 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2020-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include <map>
+#include <cstring>
+#include <cstdlib>
+#include "testcpp.h"
+
+
+namespace {
+class DrCb : public RdKafka::DeliveryReportCb {
+ public:
+  DrCb(RdKafka::ErrorCode exp_err) : cnt(0), exp_err(exp_err) {
+  }
+
+  void dr_cb(RdKafka::Message &msg) {
+    Test::Say("Delivery report: " + RdKafka::err2str(msg.err()) + "\n");
+    if (msg.err() != exp_err)
+      Test::Fail("Delivery report: Expected " + RdKafka::err2str(exp_err) +
+                 " but got " + RdKafka::err2str(msg.err()));
+    cnt++;
+  }
+
+  int cnt;
+  RdKafka::ErrorCode exp_err;
+};
+};  // namespace
+
+/**
+ * @brief Test producer auth failures.
+ *
+ * @param topic_known If true we make sure the producer knows about the topic
+ *                    before restricting access to it and producing,
+ *                    this should result in the ProduceRequest failing,
+ *                    if false we restrict access prior to this which should
+ *                    result in MetadataRequest failing.
+ */
+
+
+static void do_test_producer(bool topic_known) {
+  Test::Say(tostr() << _C_MAG << "[ Test producer auth with topic "
+                    << (topic_known ? "" : "not ") << "known ]\n");
+
+  /* Create producer */
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 20);
+
+  std::string errstr;
+  DrCb dr(RdKafka::ERR_NO_ERROR);
+  conf->set("dr_cb", &dr, errstr);
+
+  std::string bootstraps;
+  if (conf->get("bootstrap.servers", bootstraps) != RdKafka::Conf::CONF_OK)
+    Test::Fail("Failed to retrieve bootstrap.servers");
+
+  RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
+  if (!p)
+    Test::Fail("Failed to create Producer: " + errstr);
+  delete conf;
+
+  /* Create topic */
+  std::string topic_unauth = Test::mk_topic_name("0115-unauthorized", 1);
+  Test::create_topic_wait_exists(NULL, topic_unauth.c_str(), 3, 1, 5000);
+
+  int exp_dr_cnt = 0;
+
+  RdKafka::ErrorCode err;
+
+  if (topic_known) {
+    /* Produce a single message to make sure metadata is known. */
+    Test::Say("Producing seeding message 0\n");
+    err = p->produce(topic_unauth, RdKafka::Topic::PARTITION_UA,
+                     RdKafka::Producer::RK_MSG_COPY, (void *)"0", 1, NULL, 0, 0,
+                     NULL);
+    TEST_ASSERT(!err, "produce() failed: %s", RdKafka::err2str(err).c_str());
+
+    p->flush(-1);
+    exp_dr_cnt++;
+  }
+
+  /* Add denying ACL for unauth topic */
+  test_kafka_cmd(
+      "kafka-acls.sh --bootstrap-server %s "
+      "--add --deny-principal 'User:*' "
+      "--operation All --deny-host '*' "
+      "--topic '%s'",
+      bootstraps.c_str(), topic_unauth.c_str());
+
+  /* Allow ACLs to propagate */
+  rd_sleep(1);
+
+  /* Produce message to any partition. */
+  Test::Say("Producing message 1 to any partition\n");
+  err = p->produce(topic_unauth, RdKafka::Topic::PARTITION_UA,
+                   RdKafka::Producer::RK_MSG_COPY, (void *)"1", 1, NULL, 0, 0,
+                   NULL);
+  TEST_ASSERT(!err, "produce() failed: %s", RdKafka::err2str(err).c_str());
+  exp_dr_cnt++;
+
+  /* Produce message to specific partition. */
+  Test::Say("Producing message 2 to partition 0\n");
+  err = p->produce(topic_unauth, 0, RdKafka::Producer::RK_MSG_COPY, (void *)"3",
+                   1, NULL, 0, 0, NULL);
+  TEST_ASSERT(!err, "produce() failed: %s", RdKafka::err2str(err).c_str());
+  exp_dr_cnt++;
+
+  /* Wait for DRs */
+  dr.exp_err = RdKafka::ERR_TOPIC_AUTHORIZATION_FAILED;
+  p->flush(-1);
+
+
+  /* Produce message to any and specific partition, should fail immediately. */
+  Test::Say("Producing message 3 to any partition\n");
+  err = p->produce(topic_unauth, RdKafka::Topic::PARTITION_UA,
+                   RdKafka::Producer::RK_MSG_COPY, (void *)"3", 1, NULL, 0, 0,
+                   NULL);
+  TEST_ASSERT(err == dr.exp_err,
+              "Expected produce() to fail with ERR_TOPIC_AUTHORIZATION_FAILED, "
+              "not %s",
+              RdKafka::err2str(err).c_str());
+
+  /* Specific partition */
+  Test::Say("Producing message 4 to partition 0\n");
+  err = p->produce(topic_unauth, 0, RdKafka::Producer::RK_MSG_COPY, (void *)"4",
+                   1, NULL, 0, 0, NULL);
+  TEST_ASSERT(err == dr.exp_err,
+              "Expected produce() to fail with ERR_TOPIC_AUTHORIZATION_FAILED, "
+              "not %s",
+              RdKafka::err2str(err).c_str());
+
+  /* Final flush just to make sure */
+  p->flush(-1);
+
+  TEST_ASSERT(exp_dr_cnt == dr.cnt, "Expected %d deliveries, not %d",
+              exp_dr_cnt, dr.cnt);
+
+  Test::Say(tostr() << _C_GRN << "[ Test producer auth with topic "
+                    << (topic_known ? "" : "not ") << "known: PASS ]\n");
+
+  delete p;
+}
+
+extern "C" {
+int main_0115_producer_auth(int argc, char **argv) {
+  /* We can't bother passing Java security config to kafka-acls.sh */
+  if (test_needs_auth()) {
+    Test::Skip("Cluster authentication required\n");
+    return 0;
+  }
+
+  do_test_producer(true);
+  do_test_producer(false);
+
+  return 0;
+}
+}

package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp

@@ -0,0 +1,216 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2020-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <iostream>
+#include <map>
+#include <cstring>
+#include <cstdlib>
+#include "testcpp.h"
+extern "C" {
+#include "test.h"
+#include "tinycthread.h"
+#include "rdatomic.h"
+}
+
+/**
+ * Test KafkaConsumer close and destructor behaviour.
+ */
+
+
+struct args {
+  RdKafka::Queue *queue;
+  RdKafka::KafkaConsumer *c;
+};
+
+static int run_polling_thread(void *p) {
+  struct args *args = (struct args *)p;
+
+  while (!args->c->closed()) {
+    RdKafka::Message *msg;
+
+    /* We use a long timeout to also verify that the
+     * consume() call is yielded/woken by librdkafka
+     * when consumer_close_queue() finishes. */
+    msg = args->queue->consume(60 * 1000 /*60s*/);
+    if (msg)
+      delete msg;
+  }
+
+  return 0;
+}
+
+
+static void start_polling_thread(thrd_t *thrd, struct args *args) {
+  if (thrd_create(thrd, run_polling_thread, (void *)args) != thrd_success)
+    Test::Fail("Failed to create thread");
+}
+
+static void stop_polling_thread(thrd_t thrd, struct args *args) {
+  int ret;
+  if (thrd_join(thrd, &ret) != thrd_success)
+    Test::Fail("Thread join failed");
+}
+
+
+static void do_test_consumer_close(bool do_subscribe,
+                                   bool do_unsubscribe,
+                                   bool do_close,
+                                   bool with_queue) {
+  std::string testname =
+      tostr() << "Test C++ KafkaConsumer close " << "subscribe=" << do_subscribe
+              << ", unsubscribe=" << do_unsubscribe << ", close=" << do_close
+              << ", queue=" << with_queue;
+  SUB_TEST("%s", testname.c_str());
+
+  rd_kafka_mock_cluster_t *mcluster;
+  const char *bootstraps;
+
+  mcluster = test_mock_cluster_new(3, &bootstraps);
+
+  std::string errstr;
+
+  /*
+   * Produce messages to topics
+   */
+  const int msgs_per_partition = 10;
+  RdKafka::Conf *pconf;
+  Test::conf_init(&pconf, NULL, 10);
+  Test::conf_set(pconf, "bootstrap.servers", bootstraps);
+  RdKafka::Producer *p = RdKafka::Producer::create(pconf, errstr);
+  if (!p)
+    Test::Fail(tostr() << __FUNCTION__
+                       << ": Failed to create producer: " << errstr);
+  delete pconf;
+  Test::produce_msgs(p, "some_topic", 0, msgs_per_partition, 10,
+                     true /*flush*/);
+  delete p;
+
+  /* Create consumer */
+  RdKafka::Conf *conf;
+  Test::conf_init(&conf, NULL, 0);
+  Test::conf_set(conf, "bootstrap.servers", bootstraps);
+  Test::conf_set(conf, "group.id", "mygroup");
+  Test::conf_set(conf, "auto.offset.reset", "beginning");
+
+  RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+  if (!c)
+    Test::Fail("Failed to create KafkaConsumer: " + errstr);
+  delete conf;
+
+  if (do_subscribe) {
+    std::vector<std::string> topics;
+    topics.push_back("some_topic");
+    RdKafka::ErrorCode err;
+    if ((err = c->subscribe(topics)))
+      Test::Fail("subscribe failed: " + RdKafka::err2str(err));
+  }
+
+  int received = 0;
+  while (received < msgs_per_partition) {
+    RdKafka::Message *msg = c->consume(500);
+    if (msg) {
+      ++received;
+      delete msg;
+    }
+  }
+
+  RdKafka::ErrorCode err;
+  if (do_unsubscribe)
+    if ((err = c->unsubscribe()))
+      Test::Fail("unsubscribe failed: " + RdKafka::err2str(err));
+
+  if (do_close) {
+    if (with_queue) {
+      RdKafka::Queue *queue = RdKafka::Queue::create(c);
+      struct args args      = {queue, c};
+      thrd_t thrd;
+
+      /* Serve queue in background thread until close() is done */
+      start_polling_thread(&thrd, &args);
+
+      RdKafka::Error *error;
+
+      Test::Say("Closing with queue\n");
+      if ((error = c->close(queue)))
+        Test::Fail("close(queue) failed: " + error->str());
+
+      stop_polling_thread(thrd, &args);
+
+      Test::Say("Attempting second close\n");
+      /* A second call should fail */
+      if (!(error = c->close(queue)))
+        Test::Fail("Expected second close(queue) to fail");
+      if (error->code() != RdKafka::ERR__DESTROY)
+        Test::Fail("Expected second close(queue) to fail with DESTROY, not " +
+                   error->str());
+      delete error;
+
+      delete queue;
+
+    } else {
+      if ((err = c->close()))
+        Test::Fail("close failed: " + RdKafka::err2str(err));
+
+      /* A second call should fail */
+      if ((err = c->close()) != RdKafka::ERR__DESTROY)
+        Test::Fail("Expected second close to fail with DESTROY, not " +
+                   RdKafka::err2str(err));
+    }
+  }
+
+  /* Call an async method that will do nothing but verify that we're not
+   * crashing due to use-after-free. */
+  if ((err = c->commitAsync()))
+    Test::Fail("Expected commitAsync close to succeed, got " +
+               RdKafka::err2str(err));
+
+  delete c;
+
+  test_mock_cluster_destroy(mcluster);
+
+  SUB_TEST_PASS();
+}
+
+extern "C" {
+int main_0116_kafkaconsumer_close(int argc, char **argv) {
+  TEST_SKIP_MOCK_CLUSTER(0);
+
+  /* Parameters:
+   *  subscribe, unsubscribe, close, with_queue */
+  for (int i = 0; i < 1 << 4; i++) {
+    bool subscribe   = i & (1 << 0);
+    bool unsubscribe = i & (1 << 1);
+    bool do_close    = i & (1 << 2);
+    bool with_queue  = i & (1 << 3);
+    do_test_consumer_close(subscribe, unsubscribe, do_close, with_queue);
+  }
+
+  return 0;
+}
+}