@platformatic/rdkafka 4.0.0 → 4.1.0-alpha.1

package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2016-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -87,6 +88,9 @@ static void do_test_fetch_max_bytes(void) {
     Test::Fail("Failed to create KafkaConsumer: " + errstr);
   delete conf;
 
+  /* For next consumer */
+  test_wait_topic_exists(c->c_ptr(), topic.c_str(), 5000);
+
   /* Subscribe */
   std::vector<std::string> topics;
   topics.push_back(topic);

package/deps/librdkafka/tests/0083-cb_event.c

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2018-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,6 +102,7 @@ int main_0083_cb_event(int argc, char **argv) {
         err   = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000));
         TEST_ASSERT(!err, "Topic auto creation failed: %s",
                     rd_kafka_err2str(err));
+        test_wait_topic_exists(rk_p, topic, 5000);
 
         test_conf_init(&conf, &tconf, 0);
         rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
@@ -152,13 +154,16 @@ int main_0083_cb_event(int argc, char **argv) {
                                                     "expecting message\n");
                                         if (rd_kafka_event_error(rkev) ==
                                             RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
-                                                rd_kafka_assign(
-                                                    rk_c,
+                                                test_consumer_assign_by_rebalance_protocol(
+                                                    "rebalance event", rk_c,
                                                     rd_kafka_event_topic_partition_list(
                                                         rkev));
                                                 expecting_io = _NOPE;
                                         } else
-                                                rd_kafka_assign(rk_c, NULL);
+                                                test_consumer_unassign_by_rebalance_protocol(
+                                                    "rebalance event", rk_c,
+                                                    rd_kafka_event_topic_partition_list(
+                                                        rkev));
                                         break;
 
                                 case RD_KAFKA_EVENT_FETCH:

package/deps/librdkafka/tests/0084-destroy_flags.c

@@ -42,22 +42,7 @@ static void destroy_flags_rebalance_cb(rd_kafka_t *rk,
                                        rd_kafka_topic_partition_list_t *parts,
                                        void *opaque) {
         rebalance_cnt++;
-
-        TEST_SAY("rebalance_cb: %s with %d partition(s)\n",
-                 rd_kafka_err2str(err), parts->cnt);
-
-        switch (err) {
-        case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
-                test_consumer_assign("rebalance", rk, parts);
-                break;
-
-        case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
-                test_consumer_unassign("rebalance", rk);
-                break;
-
-        default:
-                TEST_FAIL("rebalance_cb: error: %s", rd_kafka_err2str(err));
-        }
+        test_rebalance_cb(rk, err, parts, opaque);
 }
 
 struct df_args {
@@ -162,6 +147,8 @@ static void do_test_destroy_flags(const char *topic,
                  destroy_flags, args->client_type, args->produce_cnt,
                  args->consumer_subscribe, args->consumer_unsubscribe,
                  local_mode ? "local" : "broker");
+
+        TEST_LATER_CHECK();
 }
 
 
@@ -184,8 +171,7 @@ static void destroy_flags(int local_mode) {
         /* Create the topic to avoid not-yet-auto-created-topics being
          * subscribed to (and thus raising an error). */
         if (!local_mode) {
-                test_create_topic(NULL, topic, 3, 1);
-                test_wait_topic_exists(NULL, topic, 5000);
+                test_create_topic_wait_exists(NULL, topic, 3, 1, 5000);
         }
 
         for (i = 0; i < (int)RD_ARRAYSIZE(args); i++) {
@@ -202,11 +188,21 @@ static void destroy_flags(int local_mode) {
 
 
 int main_0084_destroy_flags_local(int argc, char **argv) {
+        /* FIXME: fix the test with subscribe/unsubscribe PR. */
+        if (!test_consumer_group_protocol_classic()) {
+                TEST_SKIP("FIXME: fix the test with subscribe/unsubscribe PR");
+                return 0;
+        }
         destroy_flags(1 /*no brokers*/);
         return 0;
 }
 
 int main_0084_destroy_flags(int argc, char **argv) {
+        /* FIXME: fix the test with subscribe/unsubscribe PR. */
+        if (!test_consumer_group_protocol_classic()) {
+                TEST_SKIP("FIXME: fix the test with subscribe/unsubscribe PR");
+                return 0;
+        }
         destroy_flags(0 /*with brokers*/);
         return 0;
 }

package/deps/librdkafka/tests/0085-headers.cpp

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -357,6 +358,8 @@ int main_0085_headers(int argc, char **argv) {
 
   delete conf;
 
+  Test::create_topic_wait_exists(p, topic.c_str(), 1, -1, 5000);
+
   std::vector<RdKafka::TopicPartition *> parts;
   parts.push_back(RdKafka::TopicPartition::create(
       topic, 0, RdKafka::Topic::OFFSET_BEGINNING));
@@ -379,6 +382,7 @@ int main_0085_headers(int argc, char **argv) {
   test_failed_produce();
   test_assignment_op();
 
+  Test::delete_topic(p, topic.c_str());
   c->close();
   delete c;
   delete p;

package/deps/librdkafka/tests/0086-purge.c

@@ -57,8 +57,13 @@ struct waitmsgs {
 };
 
 static mtx_t produce_req_lock;
+static mtx_t produce_sent_req_lock;
 static cnd_t produce_req_cnd;
+static cnd_t produce_sent_req_cnd;
+/* Number of completed produce requests, increased on response */
 static int produce_req_cnt = 0;
+/* Number of sent produce requests, increased on request */
+static int produce_sent_req_cnt = 0;
 
 
 #if WITH_SOCKEM
@@ -75,10 +80,13 @@ static rd_kafka_resp_err_t on_request_sent(rd_kafka_t *rk,
                                            size_t size,
                                            void *ic_opaque) {
 
-        /* Save socket fd to limit ProduceRequest */
-        if (ApiKey == RD_KAFKAP_ApiVersion) {
+        if (ApiKey == RD_KAFKAP_Produce) {
+                mtx_lock(&produce_sent_req_lock);
+                produce_sent_req_cnt++;
                 test_sockfd = sockfd;
-                return RD_KAFKA_RESP_ERR_NO_ERROR;
+                if (produce_sent_req_cnt > 1)
+                        cnd_broadcast(&produce_sent_req_cnd);
+                mtx_unlock(&produce_sent_req_lock);
         }
 
         return RD_KAFKA_RESP_ERR_NO_ERROR;
@@ -226,6 +234,8 @@ do_test_purge(const char *what, int remote, int idempotence, int gapless) {
         test_conf_set(conf, "enable.gapless.guarantee",
                       gapless ? "true" : "false");
         rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
+        produce_req_cnt      = 0;
+        produce_sent_req_cnt = 0;
 
         if (remote) {
 #if WITH_SOCKEM
@@ -238,12 +248,18 @@ do_test_purge(const char *what, int remote, int idempotence, int gapless) {
                         test_curr->is_fatal_cb = gapless_is_not_fatal_cb;
 
                 mtx_init(&produce_req_lock, mtx_plain);
+                mtx_init(&produce_sent_req_lock, mtx_plain);
                 cnd_init(&produce_req_cnd);
+                cnd_init(&produce_sent_req_cnd);
         } else {
                 test_conf_set(conf, "bootstrap.servers", NULL);
         }
 
         rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+        if (remote)
+                /* Pre-create the topic to avoid unknown topic errors
+                 * because of metadata propagation */
+                test_create_topic_wait_exists(NULL, topic, 3, -1, 5000);
 
         TEST_SAY("Producing %d messages to topic %s\n", msgcnt, topic);
 
@@ -269,16 +285,16 @@ do_test_purge(const char *what, int remote, int idempotence, int gapless) {
                             rd_kafka_err2str(err));
 
                 waitmsgs.exp_err[i] =
-                    (remote && i < 10
-                         ? RD_KAFKA_RESP_ERR_NO_ERROR
-                         : remote && i < 20 ? RD_KAFKA_RESP_ERR__PURGE_INFLIGHT
-                                            : RD_KAFKA_RESP_ERR__PURGE_QUEUE);
+                    (remote && i < 10   ? RD_KAFKA_RESP_ERR_NO_ERROR
+                     : remote && i < 20 ? RD_KAFKA_RESP_ERR__PURGE_INFLIGHT
+                                        : RD_KAFKA_RESP_ERR__PURGE_QUEUE);
 
                 waitmsgs.cnt++;
         }
 
 
         if (remote) {
+                test_timing_t t_produce;
                 /* Wait for ProduceRequest to be sent */
                 mtx_lock(&produce_req_lock);
                 cnd_timedwait_ms(&produce_req_cnd, &produce_req_lock,
@@ -287,6 +303,23 @@ do_test_purge(const char *what, int remote, int idempotence, int gapless) {
                             "First Produce request should've been sent by now");
                 mtx_unlock(&produce_req_lock);
 
+                /* Wait second ProduceRequest is in-flight */
+                TIMING_START(&t_produce, "PRODUCE");
+                mtx_lock(&produce_sent_req_lock);
+                if (produce_sent_req_cnt < 2) {
+                        cnd_timedwait_ms(&produce_sent_req_cnd,
+                                         &produce_sent_req_lock, 15 * 1000);
+                }
+                TEST_ASSERT(produce_sent_req_cnt == 2,
+                            "No more than 2 requests should've "
+                            "been sent by now, "
+                            "%d sent produce requests",
+                            produce_sent_req_cnt);
+                mtx_unlock(&produce_sent_req_lock);
+                /* Even if still not in-flight, it should be sent
+                 * almost immediately given there are 19 messages in queue. */
+                TIMING_ASSERT_LATER(&t_produce, 0, 100);
+
                 purge_and_expect(what, __LINE__, rk, RD_KAFKA_PURGE_F_QUEUE,
                                  &waitmsgs, 10,
                                  "in-flight messages should not be purged");

package/deps/librdkafka/tests/0088-produce_metadata_timeout.c

@@ -114,7 +114,7 @@ int main_0088_produce_metadata_timeout(int argc, char **argv) {
         rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
 
         /* Create topic with single partition, for simplicity. */
-        test_create_topic(rk, topic, 1, 1);
+        test_create_topic_wait_exists(rk, topic, 1, 1, 5000);
 
         rkt = rd_kafka_topic_new(rk, topic, NULL);
 

package/deps/librdkafka/tests/0089-max_poll_interval.c

@@ -61,7 +61,7 @@ static void do_test(void) {
 
         testid = test_id_generate();
 
-        test_create_topic(NULL, topic, 1, 1);
+        test_create_topic_wait_exists(NULL, topic, 1, 1, 5000);
 
         test_produce_msgs_easy(topic, testid, -1, msgcnt);
 
@@ -212,7 +212,7 @@ static void do_test_with_log_queue(void) {
 
         testid = test_id_generate();
 
-        test_create_topic(NULL, topic, 1, 1);
+        test_create_topic_wait_exists(NULL, topic, 1, 1, 5000);
 
         test_produce_msgs_easy(topic, testid, -1, msgcnt);
 
@@ -380,7 +380,7 @@ do_test_rejoin_after_interval_expire(rd_bool_t forward_to_another_q,
             "%d",
             forward_to_another_q, forward_to_consumer_q);
 
-        test_create_topic(NULL, topic, 1, 1);
+        test_create_topic_wait_exists(NULL, topic, 1, 1, 5000);
 
         test_str_id_generate(groupid, sizeof(groupid));
         test_conf_init(&conf, NULL, 60);
@@ -413,7 +413,9 @@ do_test_rejoin_after_interval_expire(rd_bool_t forward_to_another_q,
         TEST_ASSERT(rd_kafka_event_error(event) ==
                         RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS,
                     "Group join should assign partitions");
-        rd_kafka_assign(rk, rd_kafka_event_topic_partition_list(event));
+        test_consumer_assign_by_rebalance_protocol(
+            "initial group join", rk,
+            rd_kafka_event_topic_partition_list(event));
         rd_kafka_event_destroy(event);
 
         rd_sleep(10 + 1); /* Exceed max.poll.interval.ms. */
@@ -426,7 +428,8 @@ do_test_rejoin_after_interval_expire(rd_bool_t forward_to_another_q,
         TEST_ASSERT(rd_kafka_event_error(event) ==
                         RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS,
                     "Group leave should revoke partitions");
-        rd_kafka_assign(rk, NULL);
+        test_consumer_unassign_by_rebalance_protocol(
+            "group leave", rk, rd_kafka_event_topic_partition_list(event));
         rd_kafka_event_destroy(event);
 
         event = test_wait_event(polling_queue, RD_KAFKA_EVENT_REBALANCE,
@@ -435,7 +438,9 @@ do_test_rejoin_after_interval_expire(rd_bool_t forward_to_another_q,
         TEST_ASSERT(rd_kafka_event_error(event) ==
                         RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS,
                     "Group rejoin should assign partitions");
-        rd_kafka_assign(rk, rd_kafka_event_topic_partition_list(event));
+
+        test_consumer_assign_by_rebalance_protocol(
+            "group rejoin", rk, rd_kafka_event_topic_partition_list(event));
         rd_kafka_event_destroy(event);
 
         if (forward_to_another_q)
@@ -466,7 +471,7 @@ static void do_test_max_poll_reset_with_consumer_cb(void) {
 
         SUB_TEST();
 
-        test_create_topic(NULL, topic, 1, 1);
+        test_create_topic_wait_exists(NULL, topic, 1, 1, 5000);
         uint64_t testid = test_id_generate();
 
         test_produce_msgs_easy(topic, testid, -1, 100);

package/deps/librdkafka/tests/0090-idempotence.c

@@ -130,11 +130,10 @@ static void do_test_implicit_ack(const char *what,
 
         rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
 
-        test_create_topic(rk, topic, 1, 1);
+        test_create_topic_wait_exists(rk, topic, 1, 1, 5000);
 
         rkt = test_create_producer_topic(rk, topic, NULL);
 
-
         TEST_SAY("Producing %d messages\n", msgcnt);
         test_produce_msgs(rk, rkt, testid, -1, 0, msgcnt, NULL, 0);
 

package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,10 +102,7 @@ static void rebalance_cb(rd_kafka_t *rk,
                     rd_kafka_name(cons->rk), cons->rebalance_cnt,
                     cons->max_rebalance_cnt);
 
-        if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS)
-                rd_kafka_assign(rk, parts);
-        else
-                rd_kafka_assign(rk, NULL);
+        test_rebalance_cb(rk, err, parts, opaque);
 }
 
 
@@ -206,7 +204,7 @@ static void do_test_with_assign(const char *topic) {
 
         test_conf_init(&conf, NULL, 60);
 
-        test_create_topic(NULL, topic, 2, 1);
+        test_create_topic_wait_exists(NULL, topic, 2, 1, 5000);
 
         test_conf_set(conf, "session.timeout.ms", "6000");
         test_conf_set(conf, "max.poll.interval.ms", "7000" /*7s*/);
@@ -251,7 +249,7 @@ static void do_test_no_poll(const char *topic) {
 
         test_conf_init(&conf, NULL, 60);
 
-        test_create_topic(NULL, topic, 2, 1);
+        test_create_topic_wait_exists(NULL, topic, 2, 1, 5000);
 
         test_conf_set(conf, "session.timeout.ms", "6000");
         test_conf_set(conf, "max.poll.interval.ms", "7000" /*7s*/);
@@ -285,7 +283,7 @@ int main_0091_max_poll_interval_timeout(int argc, char **argv) {
         const char *topic =
             test_mk_topic_name("0091_max_poll_interval_tmout", 1);
 
-        test_create_topic(NULL, topic, 2, 1);
+        test_create_topic_wait_exists(NULL, topic, 2, 1, 5000);
 
         do_test_with_subscribe(topic);
 

package/deps/librdkafka/tests/0092-mixed_msgver.c

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2018-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,6 +55,11 @@ int main_0092_mixed_msgver(int argc, char **argv) {
                 return 0;
         }
 
+        if (test_broker_version >= TEST_BRKVER(4, 0, 0, 0)) {
+                TEST_SKIP("MsgVersion 1 was removed in Apache Kafka 4.0\n");
+                return 0;
+        }
+
         testid = test_id_generate();
 
         rk = test_create_producer();

package/deps/librdkafka/tests/0093-holb.c

@@ -2,6 +2,7 @@
  * librdkafka - Apache Kafka C library
  *
  * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -91,10 +92,7 @@ static void rebalance_cb(rd_kafka_t *rk,
                     rd_kafka_name(cons->rk), cons->rebalance_cnt,
                     cons->max_rebalance_cnt);
 
-        if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS)
-                rd_kafka_assign(rk, parts);
-        else
-                rd_kafka_assign(rk, NULL);
+        test_rebalance_cb(rk, err, parts, opaque);
 }
 
 
@@ -110,12 +108,17 @@ int main_0093_holb_consumer(int argc, char **argv) {
 
         test_conf_init(&conf, NULL, 60);
 
-        test_create_topic(NULL, topic, 1, 1);
+        test_create_topic_wait_exists(NULL, topic, 1, 1, 5000);
 
         test_produce_msgs_easy(topic, testid, 0, msgcnt);
 
         test_conf_set(conf, "session.timeout.ms", "6000");
         test_conf_set(conf, "max.poll.interval.ms", "20000");
+        /* Socket timeout must be greater than
+         * 10s: the interval during which group rebalance is hold
+         * when the first consumer isn't polling below.
+         * 10s corresponds to the duration of the JoinGroup call.
+         * on the second consumer. */
         test_conf_set(conf, "socket.timeout.ms", "3000");
         test_conf_set(conf, "auto.offset.reset", "earliest");
         /* Trigger other requests often */

package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c

@@ -169,6 +169,7 @@ static void do_test_produce_timeout(const char *topic, const int msgrate) {
         rk  = test_create_handle(RD_KAFKA_PRODUCER, conf);
         rkt = test_create_producer_topic(rk, topic, "message.timeout.ms",
                                          "5000", NULL);
+        test_wait_topic_exists(rk, topic, 5000);
 
         /* Create the topic to make sure connections are up and ready. */
         err = test_auto_create_topic_rkt(rk, rkt, tmout_multip(5000));