@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2020-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -55,7 +56,7 @@ static void rebalance_cb(rd_kafka_t *rk,
             rd_kafka_err2name(rebalance_exp_event), rd_kafka_err2name(err));
 
         if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
-                test_consumer_assign("assign", rk, parts);
+                test_consumer_assign_by_rebalance_protocol("assign", rk, parts);
         } else {
                 rd_kafka_resp_err_t commit_err;
 
@@ -92,8 +93,8 @@ static void rebalance_cb(rd_kafka_t *rk,
                                     rd_kafka_err2name(commit_exp_err),
                                     rd_kafka_err2name(commit_err));
                 }
-
-                test_consumer_unassign("unassign", rk);
+                test_consumer_unassign_by_rebalance_protocol("unassign", rk,
+                                                             parts);
         }
 
         /* Make sure only one rebalance callback is served per poll()
@@ -169,21 +170,26 @@ static void do_test_session_timeout(const char *use_commit_type) {
         test_conf_set(conf, "auto.offset.reset", "earliest");
         test_conf_set(conf, "enable.auto.commit",
                       !strcmp(commit_type, "auto") ? "true" : "false");
+        rd_kafka_mock_set_group_consumer_session_timeout_ms(mcluster, 1000);
+        rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 100);
 
         c = test_create_consumer(groupid, rebalance_cb, conf, NULL);
 
         test_consumer_subscribe(c, topic);
 
-        /* Let Heartbeats fail after a couple of successful ones */
-        rd_kafka_mock_push_request_errors(
-            mcluster, RD_KAFKAP_Heartbeat, 9, RD_KAFKA_RESP_ERR_NO_ERROR,
-            RD_KAFKA_RESP_ERR_NO_ERROR, RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
-            RD_KAFKA_RESP_ERR_NOT_COORDINATOR);
+        if (test_consumer_group_protocol_classic()) {
+                /* Let Heartbeats fail after a couple of successful ones */
+                rd_kafka_mock_push_request_errors(
+                    mcluster, RD_KAFKAP_Heartbeat, 9,
+                    RD_KAFKA_RESP_ERR_NO_ERROR, RD_KAFKA_RESP_ERR_NO_ERROR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR);
+        }
 
         expect_rebalance("initial assignment", c,
                          RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, 5 + 2);
@@ -191,6 +197,12 @@ static void do_test_session_timeout(const char *use_commit_type) {
         /* Consume a couple of messages so that we have something to commit */
         test_consumer_poll("consume", c, 0, -1, 0, 10, NULL);
 
+        if (!test_consumer_group_protocol_classic()) {
+                /* Increase HB interval so member is fenced from the group */
+                rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster,
+                                                                       2000);
+        }
+
         /* The commit in the rebalance callback should fail when the
          * member has timed out from the group. */
         commit_exp_err = RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID;
@@ -291,7 +303,16 @@ int main_0106_cgrp_sess_timeout(int argc, char **argv) {
         do_test_session_timeout("async");
         do_test_session_timeout("auto");
 
-        do_test_commit_on_lost();
+        /* With KIP-848 session timeout is remote only.
+         * Partitions will stay assigned and fetchable
+         * when there's no communication with the
+         * coordinator, but the messages won't be
+         * committed.
+         * TODO: see if session timeout can be received by
+         * the broker and be enforced locally too. */
+        if (test_consumer_group_protocol_classic()) {
+                do_test_commit_on_lost();
+        }
 
         return 0;
 }

package/deps/librdkafka/tests/0107-topic_recreate.c

@@ -189,7 +189,7 @@ static void do_test_create_delete_create(int part_cnt_1, int part_cnt_2) {
         consumer = test_create_consumer(topic, NULL, NULL, NULL);
 
         /* Create topic */
-        test_create_topic(consumer, topic, part_cnt_1, 3);
+        test_create_topic_wait_exists(consumer, topic, part_cnt_1, 3, 5000);
 
         /* Start consumer */
         test_consumer_subscribe(consumer, topic);
@@ -216,7 +216,7 @@ static void do_test_create_delete_create(int part_cnt_1, int part_cnt_2) {
         rd_sleep(5);
 
         /* Re-create topic */
-        test_create_topic(consumer, topic, part_cnt_2, 3);
+        test_create_topic_wait_exists(consumer, topic, part_cnt_2, 3, 5000);
 
         mtx_lock(&value_mtx);
         value = "after";

package/deps/librdkafka/tests/0109-auto_create_topics.cpp

@@ -39,21 +39,39 @@
  * The same test is run with and without allow.auto.create.topics
  * and with and without wildcard subscribes.
  *
+ * With the KIP 848 consumer group protocol, topics aren't automatically
+ * created when subscribing and the topic authorization error
+ * applies to the whole subscription, not just the unauthorized topics.
  */
 
 
 static void do_test_consumer(bool allow_auto_create_topics,
-                             bool with_wildcards) {
+                             bool with_wildcards,
+                             bool test_unauthorized_topic) {
   Test::Say(tostr() << _C_MAG << "[ Test allow.auto.create.topics="
                     << (allow_auto_create_topics ? "true" : "false")
                     << " with_wildcards=" << (with_wildcards ? "true" : "false")
-                    << " ]\n");
+                    << " test_unauthorized_topic="
+                    << (test_unauthorized_topic ? "true" : "false") << " ]\n");
 
   bool has_acl_cli = test_broker_version >= TEST_BRKVER(2, 1, 0, 0) &&
                      !test_needs_auth(); /* We can't bother passing Java
                                           * security config to kafka-acls.sh */
+  if (test_unauthorized_topic && !has_acl_cli) {
+    Test::Say(
+        "Skipping unauthorized topic test since kafka-acls.sh is not "
+        "available\n");
+    return;
+  }
+  if (!test_consumer_group_protocol_classic() && allow_auto_create_topics) {
+    Test::Say(
+        "Skipping test as it would be duplicate "
+        "with KIP 848 consumer protocol\n");
+    return;
+  }
 
   bool supports_allow = test_broker_version >= TEST_BRKVER(0, 11, 0, 0);
+  const int cgrp_consumer_expected_consecutive_error_cnt = 3;
 
   std::string topic_exists    = Test::mk_topic_name("0109-exists", 1);
   std::string topic_notexists = Test::mk_topic_name("0109-notexists", 1);
@@ -83,7 +101,7 @@ static void do_test_consumer(bool allow_auto_create_topics,
   /* Create topics */
   Test::create_topic(c, topic_exists.c_str(), 1, 1);
 
-  if (has_acl_cli) {
+  if (test_unauthorized_topic) {
     Test::create_topic(c, topic_unauth.c_str(), 1, 1);
 
     /* Add denying ACL for unauth topic */
@@ -107,34 +125,50 @@ static void do_test_consumer(bool allow_auto_create_topics,
   std::map<std::string, RdKafka::ErrorCode> exp_errors;
 
   topics.push_back(topic_notexists);
-  if (has_acl_cli)
+
+  if (test_unauthorized_topic)
     topics.push_back(topic_unauth);
 
   if (with_wildcards) {
     topics.push_back("^" + topic_exists);
     topics.push_back("^" + topic_notexists);
-    /* If the subscription contains at least one wildcard/regex
-     * then no auto topic creation will take place (since the consumer
-     * requests all topics in metadata, and not specific ones, thus
-     * not triggering topic auto creation).
-     * We need to handle the expected error cases accordingly. */
-    exp_errors["^" + topic_notexists] = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
-    exp_errors[topic_notexists]       = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
-
-    if (has_acl_cli) {
-      /* Unauthorized topics are not included in list-all-topics Metadata,
-       * which we use for wildcards, so in this case the error code for
-       * unauthorixed topics show up as unknown topic. */
-      exp_errors[topic_unauth] = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
-    }
   } else {
     topics.push_back(topic_exists);
+  }
 
-    if (has_acl_cli)
+  /* `classic` protocol case: if the subscription contains at least one
+   * wildcard/regex then no auto topic creation will take place (since the
+   * consumer requests all topics in metadata, and not specific ones, thus not
+   * triggering topic auto creation). We need to handle the expected error cases
+   * accordingly.
+   *
+   * `consumer` protocol case: there's no automatic topic creation. */
+  if (test_consumer_group_protocol_classic()) {
+    if (with_wildcards) {
+      exp_errors["^" + topic_notexists] = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
+      exp_errors[topic_notexists]       = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
+      if (test_unauthorized_topic) {
+        /* Unauthorized topics are not included in list-all-topics Metadata,
+         * which we use for wildcards, so in this case the error code for
+         * unauthorixed topics show up as unknown topic. */
+        exp_errors[topic_unauth] = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
+      }
+    } else if (test_unauthorized_topic) {
       exp_errors[topic_unauth] = RdKafka::ERR_TOPIC_AUTHORIZATION_FAILED;
+    }
+  } else if (test_unauthorized_topic) {
+    /* Authorization errors happen if even a single topic
+     * is unauthorized and an error is returned for the whole subscription
+     * without reference to the topic. */
+    exp_errors[""] = RdKafka::ERR_TOPIC_AUTHORIZATION_FAILED;
   }
 
-  if (supports_allow && !allow_auto_create_topics)
+  /* `classic` protocol case: expect an error only if the broker supports the
+   * property and the test disallowed it.
+   *
+   * `consumer` protocol case: there's no automatic topic creation. */
+  if (supports_allow && !allow_auto_create_topics &&
+      test_consumer_group_protocol_classic())
     exp_errors[topic_notexists] = RdKafka::ERR_UNKNOWN_TOPIC_OR_PART;
 
   RdKafka::ErrorCode err;
@@ -143,7 +177,8 @@ static void do_test_consumer(bool allow_auto_create_topics,
 
   /* Start consuming until EOF is reached, which indicates that we have an
    * assignment and any errors should have been reported. */
-  bool run = true;
+  bool run                  = true;
+  int consecutive_error_cnt = 0;
   while (run) {
     RdKafka::Message *msg = c->consume(tmout_multip(1000));
     switch (msg->err()) {
@@ -155,6 +190,17 @@ static void do_test_consumer(bool allow_auto_create_topics,
       run = false;
       break;
 
+    case RdKafka::ERR_TOPIC_AUTHORIZATION_FAILED:
+      if (test_consumer_group_protocol_classic()) {
+        run = true;
+      } else {
+        /* `consumer` rebalance protocol:
+         * wait for `unauthorized_error_cnt` consecutive errors. */
+        run = (++consecutive_error_cnt) <
+              cgrp_consumer_expected_consecutive_error_cnt;
+      }
+      /* FALLTHRU */
+
     default:
       Test::Say("Consume error on " + msg->topic_name() + ": " + msg->errstr() +
                 "\n");
@@ -186,6 +232,13 @@ static void do_test_consumer(bool allow_auto_create_topics,
                      RdKafka::err2str(msg->err()));
       } else {
         exp_errors.erase(msg->topic_name());
+        if (!test_consumer_group_protocol_classic() &&
+            test_unauthorized_topic &&
+            consecutive_error_cnt <
+                cgrp_consumer_expected_consecutive_error_cnt) {
+          /* Expect same error on next HB */
+          exp_errors[""] = RdKafka::ERR_TOPIC_AUTHORIZATION_FAILED;
+        }
       }
 
       break;
@@ -207,11 +260,18 @@ static void do_test_consumer(bool allow_auto_create_topics,
 extern "C" {
 int main_0109_auto_create_topics(int argc, char **argv) {
   /* Parameters:
-   *  allow auto create, with wildcards */
-  do_test_consumer(true, true);
-  do_test_consumer(true, false);
-  do_test_consumer(false, true);
-  do_test_consumer(false, false);
+   *  allow auto create, with wildcards, test unauthorized topic */
+  do_test_consumer(true, false, false);
+  do_test_consumer(false, false, false);
+
+  do_test_consumer(true, true, false);
+  do_test_consumer(false, true, false);
+
+  do_test_consumer(true, false, true);
+  do_test_consumer(false, false, true);
+
+  do_test_consumer(true, true, true);
+  do_test_consumer(false, true, true);
 
   return 0;
 }

package/deps/librdkafka/tests/0110-batch_size.cpp

@@ -154,8 +154,7 @@ static void do_test_batch_size() {
   while (event_cb.avg_batchsize == 0)
     p->poll(1000);
 
-  Test::Say(tostr() << "Batchsize: "
-                    << "configured " << batchsize << ", min "
+  Test::Say(tostr() << "Batchsize: " << "configured " << batchsize << ", min "
                     << event_cb.min_batchsize << ", max "
                     << event_cb.max_batchsize << ", avg "
                     << event_cb.avg_batchsize << "\n");

package/deps/librdkafka/tests/0112-assign_unknown_part.c

@@ -43,7 +43,6 @@ int main_0112_assign_unknown_part(int argc, char **argv) {
         uint64_t testid   = test_id_generate();
         rd_kafka_t *c;
         rd_kafka_topic_partition_list_t *tpl;
-        int r;
 
         test_conf_init(NULL, NULL, 60);
 
@@ -51,8 +50,7 @@ int main_0112_assign_unknown_part(int argc, char **argv) {
         c = test_create_consumer(topic, NULL, NULL, NULL);
 
         TEST_SAY("Creating topic %s with 1 partition\n", topic);
-        test_create_topic(c, topic, 1, 1);
-        test_wait_topic_exists(c, topic, 10 * 1000);
+        test_create_topic_wait_exists(c, topic, 1, 1, 10 * 1000);
 
         TEST_SAY("Producing message to partition 0\n");
         test_produce_msgs_easy(topic, testid, 0, 1);
@@ -68,17 +66,8 @@ int main_0112_assign_unknown_part(int argc, char **argv) {
         TEST_SAY("Changing partition count for topic %s\n", topic);
         test_create_partitions(NULL, topic, 2);
 
-        /* FIXME: The new partition might not have propagated through the
-         *        cluster by the time the producer tries to produce to it
-         *        which causes the produce to fail.
-         *        Loop until the partition count is correct. */
-        while ((r = test_get_partition_count(c, topic, 5000)) != 2) {
-                TEST_SAY(
-                    "Waiting for %s partition count to reach 2, "
-                    "currently %d\n",
-                    topic, r);
-                rd_sleep(1);
-        }
+        /* Allow the partition to propagate */
+        rd_sleep(1);
 
         TEST_SAY("Producing message to partition 1\n");
         test_produce_msgs_easy(topic, testid, 1, 1);