@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/0050-subscribe_adds.c

@@ -0,0 +1,145 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2023, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+#include "../src/rdkafka_proto.h"
+
+#include <stdarg.h>
+
+/**
+ * Verify that quick subscription additions work.
+ *  * Create topics T1,T2,T3
+ *  * Create consumer
+ *  * Subscribe to T1
+ *  * Subscribe to T1,T2
+ *  * Subscribe to T1,T2,T3
+ *  * Verify that all messages from all three topics are consumed
+ *  * Subscribe to T1,T3
+ *  * Verify that there were no duplicate messages.
+ *
+ *  @param partition_assignment_strategy Assignment strategy to test.
+ */
+static void
+test_no_duplicate_messages(const char *partition_assignment_strategy) {
+
+        SUB_TEST("%s", partition_assignment_strategy);
+        rd_kafka_t *rk;
+#define TOPIC_CNT 3
+        char *topic[TOPIC_CNT] = {
+            rd_strdup(test_mk_topic_name("0050_subscribe_adds_1", 1)),
+            rd_strdup(test_mk_topic_name("0050_subscribe_adds_2", 1)),
+            rd_strdup(test_mk_topic_name("0050_subscribe_adds_3", 1)),
+        };
+        uint64_t testid;
+        int msgcnt = test_quick ? 100 : 10000;
+        test_msgver_t mv;
+        rd_kafka_conf_t *conf;
+        rd_kafka_topic_conf_t *tconf;
+        int i;
+        rd_kafka_topic_partition_list_t *tlist;
+        rd_kafka_resp_err_t err;
+
+        msgcnt = (msgcnt / TOPIC_CNT) * TOPIC_CNT;
+        testid = test_id_generate();
+
+        rk = test_create_producer();
+        for (i = 0; i < TOPIC_CNT; i++) {
+                rd_kafka_topic_t *rkt;
+
+                rkt = test_create_producer_topic(rk, topic[i], NULL);
+                test_wait_topic_exists(rk, topic[i], 5000);
+
+                test_produce_msgs(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
+                                  (msgcnt / TOPIC_CNT) * i,
+                                  (msgcnt / TOPIC_CNT), NULL, 1000);
+
+                rd_kafka_topic_destroy(rkt);
+        }
+
+        rd_kafka_destroy(rk);
+
+        test_conf_init(&conf, &tconf, 60);
+        test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
+        test_conf_set(conf, "partition.assignment.strategy",
+                      partition_assignment_strategy);
+
+        rk = test_create_consumer(topic[0], NULL, conf, tconf);
+
+        tlist = rd_kafka_topic_partition_list_new(TOPIC_CNT);
+        for (i = 0; i < TOPIC_CNT; i++) {
+                rd_kafka_topic_partition_list_add(tlist, topic[i],
+                                                  RD_KAFKA_PARTITION_UA);
+                TEST_SAY("Subscribe to %d topic(s):\n", tlist->cnt);
+                test_print_partition_list(tlist);
+
+                err = rd_kafka_subscribe(rk, tlist);
+                TEST_ASSERT(!err, "subscribe() failed: %s",
+                            rd_kafka_err2str(err));
+        }
+
+        test_msgver_init(&mv, testid);
+
+        test_consumer_poll("consume", rk, testid, -1, 0, msgcnt, &mv);
+
+        /* Now remove T2 */
+        rd_kafka_topic_partition_list_del(tlist, topic[1],
+                                          RD_KAFKA_PARTITION_UA);
+        err = rd_kafka_subscribe(rk, tlist);
+        TEST_ASSERT(!err, "subscribe() failed: %s", rd_kafka_err2str(err));
+
+        test_consumer_poll_no_msgs("consume", rk, testid, (int)(3000));
+
+
+        test_msgver_verify("consume", &mv, TEST_MSGVER_ORDER | TEST_MSGVER_DUP,
+                           0, msgcnt);
+
+        test_msgver_clear(&mv);
+
+        rd_kafka_topic_partition_list_destroy(tlist);
+        test_consumer_close(rk);
+        rd_kafka_destroy(rk);
+
+        for (i = 0; i < TOPIC_CNT; i++)
+                rd_free(topic[i]);
+
+        SUB_TEST_PASS();
+#undef TOPIC_CNT
+}
+
+int main_0050_subscribe_adds(int argc, char **argv) {
+
+        test_no_duplicate_messages("range");
+
+        test_no_duplicate_messages("roundrobin");
+
+        test_no_duplicate_messages("cooperative-sticky");
+
+        return 0;
+}

package/deps/librdkafka/tests/0051-assign_adds.c

@@ -0,0 +1,126 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+#include <stdarg.h>
+
+/**
+ * Verify that quick assignment additions work.
+ *  * Create topics T1,T2,T3
+ *  * Create consumer
+ *  * Assign T1
+ *  * Assign T1,T2
+ *  * Assign T1,T2,T3
+ *  * Verify that all messages from all three topics are consumed
+ *  * Assign T1,T3
+ *  * Verify that there were no duplicate messages.
+ */
+
+int main_0051_assign_adds(int argc, char **argv) {
+        rd_kafka_t *rk;
+#define TOPIC_CNT 3
+        char *topic[TOPIC_CNT] = {
+            rd_strdup(test_mk_topic_name("0051_assign_adds_1", 1)),
+            rd_strdup(test_mk_topic_name("0051_assign_adds_2", 1)),
+            rd_strdup(test_mk_topic_name("0051_assign_adds_3", 1)),
+        };
+        uint64_t testid;
+        int msgcnt = test_quick ? 100 : 1000;
+        test_msgver_t mv;
+        rd_kafka_conf_t *conf;
+        rd_kafka_topic_conf_t *tconf;
+        int i;
+        rd_kafka_topic_partition_list_t *tlist;
+        rd_kafka_resp_err_t err;
+
+        msgcnt = (msgcnt / TOPIC_CNT) * TOPIC_CNT;
+        testid = test_id_generate();
+
+        rk = test_create_producer();
+        for (i = 0; i < TOPIC_CNT; i++) {
+                rd_kafka_topic_t *rkt;
+
+                rkt = test_create_producer_topic(rk, topic[i], NULL);
+                test_wait_topic_exists(rk, topic[i], 5000);
+
+                test_produce_msgs(rk, rkt, testid, 0, (msgcnt / TOPIC_CNT) * i,
+                                  (msgcnt / TOPIC_CNT), NULL, 100);
+
+                rd_kafka_topic_destroy(rkt);
+        }
+
+        rd_kafka_destroy(rk);
+
+        test_conf_init(&conf, &tconf, 60);
+        test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
+
+        rk = test_create_consumer(topic[0], NULL, conf, tconf);
+
+        tlist = rd_kafka_topic_partition_list_new(TOPIC_CNT);
+        for (i = 0; i < TOPIC_CNT; i++) {
+                rd_kafka_topic_partition_list_add(tlist, topic[i], 0);
+                TEST_SAY("Assign %d topic(s):\n", tlist->cnt);
+                test_print_partition_list(tlist);
+
+                err = rd_kafka_assign(rk, tlist);
+                TEST_ASSERT(!err, "assign() failed: %s", rd_kafka_err2str(err));
+        }
+
+        test_msgver_init(&mv, testid);
+
+        TEST_SAY("Expecting to consume all %d messages from %d topics\n",
+                 msgcnt, TOPIC_CNT);
+
+        test_consumer_poll("consume", rk, testid, -1, 0, msgcnt, &mv);
+
+        /* Now remove T2 */
+        rd_kafka_topic_partition_list_del(tlist, topic[1], 0);
+        err = rd_kafka_assign(rk, tlist);
+        TEST_ASSERT(!err, "assign() failed: %s", rd_kafka_err2str(err));
+
+        TEST_SAY(
+            "Should not see any messages for session.timeout.ms+some more\n");
+        test_consumer_poll_no_msgs("consume", rk, testid, (int)(6000 * 1.5));
+
+        test_msgver_verify("consume", &mv, TEST_MSGVER_ORDER | TEST_MSGVER_DUP,
+                           0, msgcnt);
+
+        test_msgver_clear(&mv);
+
+        rd_kafka_topic_partition_list_destroy(tlist);
+
+        test_consumer_close(rk);
+        rd_kafka_destroy(rk);
+
+        for (i = 0; i < TOPIC_CNT; i++)
+                rd_free(topic[i]);
+
+        return 0;
+}

package/deps/librdkafka/tests/0052-msg_timestamps.c

@@ -0,0 +1,238 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * Verify message timestamp behaviour on supporting brokers (>=0.10.0.0).
+ * Issue #858
+ */
+struct timestamp_range {
+        int64_t min;
+        int64_t max;
+};
+
+static const struct timestamp_range invalid_timestamp = {-1, -1};
+static struct timestamp_range broker_timestamp;
+static struct timestamp_range my_timestamp;
+
+static void prepare_timestamps(void) {
+        struct timeval ts;
+        rd_gettimeofday(&ts, NULL);
+
+        /* broker timestamps expected to be within 600 seconds */
+        broker_timestamp.min = (int64_t)ts.tv_sec * 1000LLU;
+        broker_timestamp.max = broker_timestamp.min + (600 * 1000LLU);
+
+        /* client timestamps: set in the future (24 hours)
+         * to be outside of broker timestamps */
+        my_timestamp.min = my_timestamp.max =
+            (int64_t)ts.tv_sec + (24 * 3600 * 1000LLU);
+}
+
+/**
+ * @brief Produce messages according to compress \p codec
+ */
+static void produce_msgs(const char *topic,
+                         int partition,
+                         uint64_t testid,
+                         int msgcnt,
+                         const char *broker_version,
+                         const char *codec) {
+        rd_kafka_conf_t *conf;
+        rd_kafka_t *rk;
+        int i;
+        char key[128], buf[100];
+        int msgcounter = msgcnt;
+
+        test_conf_init(&conf, NULL, 0);
+        rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb);
+        test_conf_set(conf, "compression.codec", codec);
+        test_conf_set(conf, "broker.version.fallback", broker_version);
+        if (!strncmp(broker_version, "0.8", 3) ||
+            !strncmp(broker_version, "0.9", 3)) {
+                test_conf_set(conf, "api.version.request", "false");
+                test_conf_set(conf, "enable.idempotence", "false");
+        }
+
+        /* Make sure to trigger a bunch of MessageSets */
+        test_conf_set(conf, "batch.num.messages", tsprintf("%d", msgcnt / 5));
+        rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+        for (i = 0; i < msgcnt; i++) {
+                rd_kafka_resp_err_t err;
+
+                test_prepare_msg(testid, partition, i, buf, sizeof(buf), key,
+                                 sizeof(key));
+
+                err = rd_kafka_producev(
+                    rk, RD_KAFKA_V_TOPIC(topic),
+                    RD_KAFKA_V_VALUE(buf, sizeof(buf)),
+                    RD_KAFKA_V_KEY(key, sizeof(key)),
+                    RD_KAFKA_V_TIMESTAMP(my_timestamp.min),
+                    RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+                    RD_KAFKA_V_OPAQUE(&msgcounter), RD_KAFKA_V_END);
+                if (err)
+                        TEST_FAIL("producev() failed at msg #%d/%d: %s", i,
+                                  msgcnt, rd_kafka_err2str(err));
+        }
+
+        TEST_SAY("Waiting for %d messages to be produced\n", msgcounter);
+        while (msgcounter > 0)
+                rd_kafka_poll(rk, 100);
+
+        rd_kafka_destroy(rk);
+}
+
+static void
+consume_msgs_verify_timestamps(const char *topic,
+                               int partition,
+                               uint64_t testid,
+                               int msgcnt,
+                               const struct timestamp_range *exp_timestamp) {
+        test_msgver_t mv;
+
+        test_msgver_init(&mv, testid);
+        test_consume_msgs_easy_mv(topic, topic, -1, testid, -1, msgcnt, NULL,
+                                  &mv);
+
+        test_msgver_verify0(
+            __FUNCTION__, __LINE__, topic, &mv,
+            TEST_MSGVER_RANGE | TEST_MSGVER_BY_MSGID | TEST_MSGVER_BY_TIMESTAMP,
+            (struct test_mv_vs) {.msg_base      = 0,
+                                 .exp_cnt       = msgcnt,
+                                 .timestamp_min = exp_timestamp->min,
+                                 .timestamp_max = exp_timestamp->max});
+
+        test_msgver_clear(&mv);
+}
+
+
+
+static void test_timestamps(const char *broker_tstype,
+                            const char *broker_version,
+                            const char *codec,
+                            const struct timestamp_range *exp_timestamps) {
+        const char *topic =
+            test_mk_topic_name(tsprintf("0052_msg_timestamps_%s_%s_%s",
+                                        broker_tstype, broker_version, codec),
+                               1);
+        const int msgcnt = 20;
+        uint64_t testid  = test_id_generate();
+
+        if ((!strncmp(broker_version, "0.9", 3) ||
+             !strncmp(broker_version, "0.8", 3)) &&
+            !test_conf_match(NULL, "sasl.mechanisms", "GSSAPI")) {
+                TEST_SAY(_C_YEL
+                         "Skipping %s, %s test: "
+                         "SaslHandshake not supported by broker v%s" _C_CLR
+                         "\n",
+                         broker_tstype, codec, broker_version);
+                return;
+        }
+
+        TEST_SAY(_C_MAG "Timestamp test using %s\n", topic);
+        test_timeout_set(30);
+
+        test_kafka_topics(
+            "--create --topic \"%s\" "
+            "--replication-factor 1 --partitions 1 "
+            "--config message.timestamp.type=%s",
+            topic, broker_tstype);
+        test_wait_topic_exists(NULL, topic, 5000);
+
+        TEST_SAY(_C_MAG "Producing %d messages to %s\n", msgcnt, topic);
+        produce_msgs(topic, 0, testid, msgcnt, broker_version, codec);
+
+        TEST_SAY(_C_MAG
+                 "Consuming and verifying %d messages from %s "
+                 "with expected timestamps %" PRId64 "..%" PRId64 "\n",
+                 msgcnt, topic, exp_timestamps->min, exp_timestamps->max);
+
+        consume_msgs_verify_timestamps(topic, 0, testid, msgcnt,
+                                       exp_timestamps);
+}
+
+
+int main_0052_msg_timestamps(int argc, char **argv) {
+        rd_bool_t test_with_apache_kafka_since_4_0 = rd_false;
+
+        if (!test_can_create_topics(1))
+                return 0;
+
+        if (test_needs_auth()) {
+                TEST_SKIP("Test cluster requires authentication/SSL\n");
+                return 0;
+        }
+
+        if (test_broker_version >= TEST_BRKVER(4, 0, 0, 0)) {
+                /* Code using `broker.version.fallback` and
+                 * `api.version.request=false` won't work
+                 * against Apache Kafka 4.0 */
+                test_with_apache_kafka_since_4_0 = rd_true;
+        }
+
+        /* Broker version limits the producer's feature set,
+         * for 0.9.0.0 no timestamp will be transmitted,
+         * but for 0.10.1.0 (or newer, api.version.request will be true)
+         * the producer will set the timestamp.
+         * In all cases we want a reasonable timestamp back.
+         *
+         * Explicit broker LogAppendTime setting will overwrite
+         * any producer-provided offset.
+         *
+         * Using the old non-timestamp-aware protocol without
+         * LogAppendTime will cause unset/invalid timestamps .
+         *
+         * Any other option should honour the producer create timestamps.
+         */
+        prepare_timestamps();
+
+        test_timestamps("CreateTime", "0.10.1.0", "none", &my_timestamp);
+        test_timestamps("LogAppendTime", "0.10.1.0", "none", &broker_timestamp);
+        if (!test_with_apache_kafka_since_4_0) {
+                test_timestamps("CreateTime", "0.9.0.0", "none",
+                                &invalid_timestamp);
+                test_timestamps("LogAppendTime", "0.9.0.0", "none",
+                                &broker_timestamp);
+        }
+#if WITH_ZLIB
+        test_timestamps("CreateTime", "0.10.1.0", "gzip", &my_timestamp);
+        test_timestamps("LogAppendTime", "0.10.1.0", "gzip", &broker_timestamp);
+        if (!test_with_apache_kafka_since_4_0) {
+                test_timestamps("CreateTime", "0.9.0.0", "gzip",
+                                &invalid_timestamp);
+                test_timestamps("LogAppendTime", "0.9.0.0", "gzip",
+                                &broker_timestamp);
+        }
+#endif
+
+        return 0;
+}