@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/0041-fetch_max_bytes.c

@@ -0,0 +1,97 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+/* Typical include path would be <librdkafka/rdkafka.h>, but this program
+ * is built from within the librdkafka source tree and thus differs. */
+#include "rdkafka.h" /* for Kafka driver */
+
+
+/**
+ * Issue #597: increase fetch.message.max.bytes until large messages can
+ *             be fetched.
+ *
+ * General idea:
+ *  - Produce 1000 small messages < MAX_BYTES
+ *  - Produce 1000 large messages > MAX_BYTES
+ *  - Create consumer with fetch.message.max.bytes=MAX_BYTES
+ *  - Consume from beginning
+ *  - All messages should be received.
+ */
+
+
+int main_0041_fetch_max_bytes(int argc, char **argv) {
+        const char *topic   = test_mk_topic_name(__FUNCTION__, 1);
+        const int partition = 0;
+        const int msgcnt    = 2 * 1000;
+        const int MAX_BYTES = 100000;
+        uint64_t testid;
+        rd_kafka_conf_t *conf;
+        rd_kafka_t *rk;
+        rd_kafka_topic_t *rkt;
+
+        test_conf_init(NULL, NULL, 60);
+
+        testid = test_id_generate();
+        rk     = test_create_producer();
+        rkt    = test_create_producer_topic(rk, topic, NULL);
+        test_wait_topic_exists(rk, topic, 5000);
+
+        test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt / 2, NULL,
+                          MAX_BYTES / 10);
+        test_produce_msgs(rk, rkt, testid, partition, msgcnt / 2, msgcnt / 2,
+                          NULL, MAX_BYTES * 5);
+
+        rd_kafka_topic_destroy(rkt);
+        rd_kafka_destroy(rk);
+
+        TEST_SAY("Creating consumer\n");
+        test_conf_init(&conf, NULL, 0);
+
+        test_conf_set(conf, "fetch.message.max.bytes",
+                      tsprintf("%d", MAX_BYTES));
+
+        /* This test may be slower when running with SSL or Helgrind,
+         * restart the timeout. */
+        test_timeout_set(60);
+
+        rk  = test_create_consumer(NULL, NULL, conf, NULL);
+        rkt = rd_kafka_topic_new(rk, topic, NULL);
+
+        test_consumer_start("CONSUME", rkt, partition,
+                            RD_KAFKA_OFFSET_BEGINNING);
+        test_consume_msgs("CONSUME", rkt, testid, partition, TEST_NO_SEEK, 0,
+                          msgcnt, 1);
+        test_consumer_stop("CONSUME", rkt, partition);
+
+        rd_kafka_topic_destroy(rkt);
+        rd_kafka_destroy(rk);
+
+        return 0;
+}

package/deps/librdkafka/tests/0042-many_topics.c

@@ -0,0 +1,252 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+/* Typical include path would be <librdkafka/rdkafka.h>, but this program
+ * is built from within the librdkafka source tree and thus differs. */
+#include "rdkafka.h" /* for Kafka driver */
+
+
+/**
+ * #781: handle many (?) topics.
+ */
+
+
+const int msgs_per_topic = 100;
+
+
+/**
+ * Request offset for nonexisting partition.
+ * Will cause rd_kafka_destroy() to hang.
+ */
+
+
+
+static void produce_many(char **topics, int topic_cnt, uint64_t testid) {
+        rd_kafka_t *rk;
+        test_timing_t t_rkt_create;
+        int i;
+        rd_kafka_topic_t **rkts;
+
+        TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
+
+        rk = test_create_producer();
+
+        TEST_SAY("Creating %d topic objects\n", topic_cnt);
+
+        rkts = malloc(sizeof(*rkts) * topic_cnt);
+        TIMING_START(&t_rkt_create, "Topic object create");
+        for (i = 0; i < topic_cnt; i++) {
+                rkts[i] = test_create_topic_object(rk, topics[i], "acks", "all",
+                                                   NULL);
+        }
+        TIMING_STOP(&t_rkt_create);
+
+        TEST_SAY("Producing %d messages to each %d topics\n", msgs_per_topic,
+                 topic_cnt);
+        /* Produce messages to each topic (so they are created) */
+        for (i = 0; i < topic_cnt; i++) {
+                test_produce_msgs(rk, rkts[i], testid, 0, i * msgs_per_topic,
+                                  msgs_per_topic, NULL, 100);
+        }
+
+        TEST_SAY("Destroying %d topic objects\n", topic_cnt);
+        for (i = 0; i < topic_cnt; i++) {
+                rd_kafka_topic_destroy(rkts[i]);
+        }
+        free(rkts);
+
+        test_flush(rk, 30000);
+
+        rd_kafka_destroy(rk);
+}
+
+
+static void legacy_consume_many(char **topics, int topic_cnt, uint64_t testid) {
+        rd_kafka_t *rk;
+        test_timing_t t_rkt_create;
+        int i;
+        rd_kafka_topic_t **rkts;
+        int msg_base = 0;
+
+        TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
+
+        test_conf_init(NULL, NULL, 60);
+
+        rk = test_create_consumer(NULL, NULL, NULL, NULL);
+
+        TEST_SAY("Creating %d topic objects\n", topic_cnt);
+
+        rkts = malloc(sizeof(*rkts) * topic_cnt);
+        TIMING_START(&t_rkt_create, "Topic object create");
+        for (i = 0; i < topic_cnt; i++)
+                rkts[i] = test_create_topic_object(rk, topics[i], NULL);
+        TIMING_STOP(&t_rkt_create);
+
+        TEST_SAY("Start consumer for %d topics\n", topic_cnt);
+        for (i = 0; i < topic_cnt; i++)
+                test_consumer_start("legacy", rkts[i], 0,
+                                    RD_KAFKA_OFFSET_BEGINNING);
+
+        TEST_SAY("Consuming from %d messages from each %d topics\n",
+                 msgs_per_topic, topic_cnt);
+        for (i = 0; i < topic_cnt; i++) {
+                test_consume_msgs("legacy", rkts[i], testid, 0, TEST_NO_SEEK,
+                                  msg_base, msgs_per_topic, 1);
+                msg_base += msgs_per_topic;
+        }
+
+        TEST_SAY("Stopping consumers\n");
+        for (i = 0; i < topic_cnt; i++)
+                test_consumer_stop("legacy", rkts[i], 0);
+
+
+        TEST_SAY("Destroying %d topic objects\n", topic_cnt);
+        for (i = 0; i < topic_cnt; i++)
+                rd_kafka_topic_destroy(rkts[i]);
+
+        free(rkts);
+
+        rd_kafka_destroy(rk);
+}
+
+
+
+static void
+subscribe_consume_many(char **topics, int topic_cnt, uint64_t testid) {
+        rd_kafka_t *rk;
+        int i;
+        rd_kafka_topic_conf_t *tconf;
+        rd_kafka_topic_partition_list_t *parts;
+        rd_kafka_resp_err_t err;
+        test_msgver_t mv;
+
+        TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
+
+        test_conf_init(NULL, &tconf, 60);
+        test_topic_conf_set(tconf, "auto.offset.reset", "earliest");
+        rk = test_create_consumer(__FUNCTION__, NULL, NULL, tconf);
+
+        parts = rd_kafka_topic_partition_list_new(topic_cnt);
+        for (i = 0; i < topic_cnt; i++)
+                rd_kafka_topic_partition_list_add(parts, topics[i],
+                                                  RD_KAFKA_PARTITION_UA);
+
+        TEST_SAY("Subscribing to %d topics\n", topic_cnt);
+        err = rd_kafka_subscribe(rk, parts);
+        if (err)
+                TEST_FAIL("subscribe() failed: %s\n", rd_kafka_err2str(err));
+
+        rd_kafka_topic_partition_list_destroy(parts);
+
+        test_msgver_init(&mv, testid);
+        test_consumer_poll("consume.subscribe", rk, testid, -1, 0,
+                           msgs_per_topic * topic_cnt, &mv);
+
+        for (i = 0; i < topic_cnt; i++)
+                test_msgver_verify_part("subscribe", &mv, TEST_MSGVER_ALL_PART,
+                                        topics[i], 0, i * msgs_per_topic,
+                                        msgs_per_topic);
+        test_msgver_clear(&mv);
+
+        test_consumer_close(rk);
+
+        rd_kafka_destroy(rk);
+}
+
+
+
+static void assign_consume_many(char **topics, int topic_cnt, uint64_t testid) {
+        rd_kafka_t *rk;
+        rd_kafka_topic_partition_list_t *parts;
+        int i;
+        test_msgver_t mv;
+
+        TEST_SAY(_C_MAG "%s\n" _C_CLR, __FUNCTION__);
+
+        test_conf_init(NULL, NULL, 60);
+        rk = test_create_consumer(__FUNCTION__, NULL, NULL, NULL);
+
+        parts = rd_kafka_topic_partition_list_new(topic_cnt);
+        for (i = 0; i < topic_cnt; i++)
+                rd_kafka_topic_partition_list_add(parts, topics[i], 0)->offset =
+                    RD_KAFKA_OFFSET_TAIL(msgs_per_topic);
+
+        test_consumer_assign("consume.assign", rk, parts);
+        rd_kafka_topic_partition_list_destroy(parts);
+
+        test_msgver_init(&mv, testid);
+        test_consumer_poll("consume.assign", rk, testid, -1, 0,
+                           msgs_per_topic * topic_cnt, &mv);
+
+        for (i = 0; i < topic_cnt; i++)
+                test_msgver_verify_part("assign", &mv, TEST_MSGVER_ALL_PART,
+                                        topics[i], 0, i * msgs_per_topic,
+                                        msgs_per_topic);
+        test_msgver_clear(&mv);
+
+        test_consumer_close(rk);
+
+        rd_kafka_destroy(rk);
+}
+
+
+
+int main_0042_many_topics(int argc, char **argv) {
+        char **topics;
+        int topic_cnt = test_quick ? 4 : 20; /* up this as needed,
+                                              * topic creation takes time so
+                                              * unless hunting a bug
+                                              * we keep this low to keep the
+                                              * test suite run time down. */
+        uint64_t testid;
+        int i;
+
+        test_conf_init(NULL, NULL, 60);
+
+        testid = test_id_generate();
+
+        /* Generate unique topic names */
+        topics = malloc(sizeof(*topics) * topic_cnt);
+        for (i = 0; i < topic_cnt; i++)
+                topics[i] = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
+
+        produce_many(topics, topic_cnt, testid);
+        legacy_consume_many(topics, topic_cnt, testid);
+        if (test_broker_version >= TEST_BRKVER(0, 9, 0, 0)) {
+                subscribe_consume_many(topics, topic_cnt, testid);
+                assign_consume_many(topics, topic_cnt, testid);
+        }
+
+        for (i = 0; i < topic_cnt; i++)
+                free(topics[i]);
+        free(topics);
+
+        return 0;
+}

package/deps/librdkafka/tests/0043-no_connection.c

@@ -0,0 +1,77 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+
+/**
+ * Make sure library behaves even if there is no broker connection.
+ */
+
+
+
+static void test_producer_no_connection(void) {
+        rd_kafka_t *rk;
+        rd_kafka_conf_t *conf;
+        rd_kafka_topic_t *rkt;
+        int i;
+        const int partition_cnt = 2;
+        int msgcnt              = 0;
+        test_timing_t t_destroy;
+
+        test_conf_init(&conf, NULL, 20);
+
+        test_conf_set(conf, "bootstrap.servers", NULL);
+
+        rk  = test_create_handle(RD_KAFKA_PRODUCER, conf);
+        rkt = test_create_topic_object(rk, __FUNCTION__, "message.timeout.ms",
+                                       "5000", NULL);
+
+        test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0, 100,
+                                 NULL, 100, 0, &msgcnt);
+        for (i = 0; i < partition_cnt; i++)
+                test_produce_msgs_nowait(rk, rkt, 0, i, 0, 100, NULL, 100, 0,
+                                         &msgcnt);
+
+        rd_kafka_poll(rk, 1000);
+
+        TEST_SAY("%d messages in queue\n", rd_kafka_outq_len(rk));
+
+        rd_kafka_topic_destroy(rkt);
+
+        TIMING_START(&t_destroy, "rd_kafka_destroy()");
+        rd_kafka_destroy(rk);
+        TIMING_STOP(&t_destroy);
+}
+
+int main_0043_no_connection(int argc, char **argv) {
+        test_producer_no_connection();
+
+        return 0;
+}

package/deps/librdkafka/tests/0044-partition_cnt.c

@@ -0,0 +1,94 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+
+/**
+ * Make sure library behaves when the partition count for a topic changes.
+ * This test requires to be run under trivup to be able to use kafka-topics.sh
+ */
+
+
+
+/**
+ * - Create topic with 2 partitions
+ * - Start producing messages to UA partition
+ * - Change to 4 partitions
+ * - Produce more messages to UA partition
+ * - Wait for DRs
+ * - Close
+ */
+
+static void test_producer_partition_cnt_change(void) {
+        rd_kafka_t *rk;
+        rd_kafka_conf_t *conf;
+        rd_kafka_topic_t *rkt;
+        const char *topic       = test_mk_topic_name(__FUNCTION__, 1);
+        const int partition_cnt = 4;
+        int msgcnt              = test_quick ? 500 : 100000;
+        test_timing_t t_destroy;
+        int produced = 0;
+
+        test_conf_init(&conf, NULL, 20);
+        rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb);
+        rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+        test_create_topic_wait_exists(rk, topic, partition_cnt / 2, 1, 5000);
+
+        rkt =
+            test_create_topic_object(rk, topic, "message.timeout.ms",
+                                     tsprintf("%d", tmout_multip(10000)), NULL);
+
+        test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0,
+                                 msgcnt / 2, NULL, 100, 0, &produced);
+
+        test_create_partitions(rk, topic, partition_cnt);
+
+        test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, msgcnt / 2,
+                                 msgcnt / 2, NULL, 100, 0, &produced);
+
+        test_wait_delivery(rk, &produced);
+
+        rd_kafka_topic_destroy(rkt);
+
+        TIMING_START(&t_destroy, "rd_kafka_destroy()");
+        rd_kafka_destroy(rk);
+        TIMING_STOP(&t_destroy);
+}
+
+int main_0044_partition_cnt(int argc, char **argv) {
+        if (!test_can_create_topics(1))
+                return 0;
+
+        test_producer_partition_cnt_change();
+
+        return 0;
+}