@point3/node-rdkafka 3.6.0-1

package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c

@@ -0,0 +1,318 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2020-2022, Magnus Edenhill
+ *               2025, Confluent Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+#include "../src/rdkafka_proto.h"
+
+
+/**
+ * @name Verify that the high-level consumer times out itself if
+ *       heartbeats are not successful (issue #2631).
+ */
+
+static const char *commit_type;
+static int rebalance_cnt;
+static rd_kafka_resp_err_t rebalance_exp_event;
+static rd_kafka_resp_err_t commit_exp_err;
+
+static void rebalance_cb(rd_kafka_t *rk,
+                         rd_kafka_resp_err_t err,
+                         rd_kafka_topic_partition_list_t *parts,
+                         void *opaque) {
+
+        rebalance_cnt++;
+        TEST_SAY("Rebalance #%d: %s: %d partition(s)\n", rebalance_cnt,
+                 rd_kafka_err2name(err), parts->cnt);
+
+        TEST_ASSERT(
+            err == rebalance_exp_event, "Expected rebalance event %s, not %s",
+            rd_kafka_err2name(rebalance_exp_event), rd_kafka_err2name(err));
+
+        if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
+                test_consumer_assign_by_rebalance_protocol("assign", rk, parts);
+        } else {
+                rd_kafka_resp_err_t commit_err;
+
+                if (strcmp(commit_type, "auto")) {
+                        rd_kafka_resp_err_t perr;
+
+                        TEST_SAY("Performing %s commit\n", commit_type);
+
+                        perr = rd_kafka_position(rk, parts);
+                        TEST_ASSERT(!perr, "Failed to acquire position: %s",
+                                    rd_kafka_err2str(perr));
+
+                        /* Sleep a short while so the broker times out the
+                         * member too. */
+                        rd_sleep(1);
+
+                        commit_err = rd_kafka_commit(
+                            rk, parts, !strcmp(commit_type, "async"));
+
+                        if (!strcmp(commit_type, "async"))
+                                TEST_ASSERT(!commit_err,
+                                            "Async commit should not fail, "
+                                            "but it returned %s",
+                                            rd_kafka_err2name(commit_err));
+                        else
+                                TEST_ASSERT(
+                                    commit_err == commit_exp_err ||
+                                        (!commit_exp_err &&
+                                         commit_err ==
+                                             RD_KAFKA_RESP_ERR__NO_OFFSET),
+                                    "Expected %s commit to return %s, "
+                                    "not %s",
+                                    commit_type,
+                                    rd_kafka_err2name(commit_exp_err),
+                                    rd_kafka_err2name(commit_err));
+                }
+                test_consumer_unassign_by_rebalance_protocol("unassign", rk,
+                                                             parts);
+        }
+
+        /* Make sure only one rebalance callback is served per poll()
+         * so that expect_rebalance() returns to the test logic on each
+         * rebalance. */
+        rd_kafka_yield(rk);
+}
+
+
+/**
+ * @brief Wait for an expected rebalance event, or fail.
+ */
+static void expect_rebalance(const char *what,
+                             rd_kafka_t *c,
+                             rd_kafka_resp_err_t exp_event,
+                             int timeout_s) {
+        int64_t tmout = test_clock() + (timeout_s * 1000000);
+        int start_cnt = rebalance_cnt;
+
+        TEST_SAY("Waiting for %s (%s) for %ds\n", what,
+                 rd_kafka_err2name(exp_event), timeout_s);
+
+        rebalance_exp_event = exp_event;
+
+        while (tmout > test_clock() && rebalance_cnt == start_cnt) {
+                if (test_consumer_poll_once(c, NULL, 1000))
+                        rd_sleep(1);
+        }
+
+        if (rebalance_cnt == start_cnt + 1) {
+                rebalance_exp_event = RD_KAFKA_RESP_ERR_NO_ERROR;
+                return;
+        }
+
+        TEST_FAIL("Timed out waiting for %s (%s)\n", what,
+                  rd_kafka_err2name(exp_event));
+}
+
+
+/**
+ * @brief Verify that session timeouts are handled by the consumer itself.
+ *
+ * @param use_commit_type "auto", "sync" (manual), "async" (manual)
+ */
+static void do_test_session_timeout(const char *use_commit_type) {
+        const char *bootstraps;
+        rd_kafka_mock_cluster_t *mcluster;
+        rd_kafka_conf_t *conf;
+        rd_kafka_t *c;
+        const char *groupid = "mygroup";
+        const char *topic   = "test";
+
+        rebalance_cnt = 0;
+        commit_type   = use_commit_type;
+
+        SUB_TEST0(!strcmp(use_commit_type, "sync") /*quick*/,
+                  "Test session timeout with %s commit", use_commit_type);
+
+        mcluster = test_mock_cluster_new(3, &bootstraps);
+
+        rd_kafka_mock_coordinator_set(mcluster, "group", groupid, 1);
+
+        /* Seed the topic with messages */
+        test_produce_msgs_easy_v(topic, 0, 0, 0, 100, 10, "bootstrap.servers",
+                                 bootstraps, "batch.num.messages", "10", NULL);
+
+        test_conf_init(&conf, NULL, 30);
+        test_conf_set(conf, "bootstrap.servers", bootstraps);
+        test_conf_set(conf, "security.protocol", "PLAINTEXT");
+        test_conf_set(conf, "group.id", groupid);
+        test_conf_set(conf, "session.timeout.ms", "5000");
+        test_conf_set(conf, "heartbeat.interval.ms", "1000");
+        test_conf_set(conf, "auto.offset.reset", "earliest");
+        test_conf_set(conf, "enable.auto.commit",
+                      !strcmp(commit_type, "auto") ? "true" : "false");
+        rd_kafka_mock_set_group_consumer_session_timeout_ms(mcluster, 1000);
+        rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster, 100);
+
+        c = test_create_consumer(groupid, rebalance_cb, conf, NULL);
+
+        test_consumer_subscribe(c, topic);
+
+        if (test_consumer_group_protocol_classic()) {
+                /* Let Heartbeats fail after a couple of successful ones */
+                rd_kafka_mock_push_request_errors(
+                    mcluster, RD_KAFKAP_Heartbeat, 9,
+                    RD_KAFKA_RESP_ERR_NO_ERROR, RD_KAFKA_RESP_ERR_NO_ERROR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR,
+                    RD_KAFKA_RESP_ERR_NOT_COORDINATOR);
+        }
+
+        expect_rebalance("initial assignment", c,
+                         RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, 5 + 2);
+
+        /* Consume a couple of messages so that we have something to commit */
+        test_consumer_poll("consume", c, 0, -1, 0, 10, NULL);
+
+        if (!test_consumer_group_protocol_classic()) {
+                /* Increase HB interval so member is fenced from the group */
+                rd_kafka_mock_set_group_consumer_heartbeat_interval_ms(mcluster,
+                                                                       2000);
+        }
+
+        /* The commit in the rebalance callback should fail when the
+         * member has timed out from the group. */
+        commit_exp_err = RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID;
+
+        expect_rebalance("session timeout revoke", c,
+                         RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS, 2 + 5 + 2);
+
+        expect_rebalance("second assignment", c,
+                         RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, 5 + 2);
+
+        /* Final rebalance in close().
+         * Its commit will work. */
+        rebalance_exp_event = RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS;
+        commit_exp_err      = RD_KAFKA_RESP_ERR_NO_ERROR;
+
+        test_consumer_close(c);
+
+        rd_kafka_destroy(c);
+
+        test_mock_cluster_destroy(mcluster);
+
+        SUB_TEST_PASS();
+}
+
+
+/**
+ * @brief Attempt manual commit when assignment has been lost (#3217)
+ */
+static void do_test_commit_on_lost(void) {
+        const char *bootstraps;
+        rd_kafka_mock_cluster_t *mcluster;
+        rd_kafka_conf_t *conf;
+        rd_kafka_t *c;
+        const char *groupid = "mygroup";
+        const char *topic   = "test";
+        rd_kafka_resp_err_t err;
+
+        SUB_TEST();
+
+        test_curr->is_fatal_cb = test_error_is_not_fatal_cb;
+
+        mcluster = test_mock_cluster_new(3, &bootstraps);
+
+        rd_kafka_mock_coordinator_set(mcluster, "group", groupid, 1);
+
+        /* Seed the topic with messages */
+        test_produce_msgs_easy_v(topic, 0, 0, 0, 100, 10, "bootstrap.servers",
+                                 bootstraps, "batch.num.messages", "10", NULL);
+
+        test_conf_init(&conf, NULL, 30);
+        test_conf_set(conf, "bootstrap.servers", bootstraps);
+        test_conf_set(conf, "security.protocol", "PLAINTEXT");
+        test_conf_set(conf, "group.id", groupid);
+        test_conf_set(conf, "session.timeout.ms", "5000");
+        test_conf_set(conf, "heartbeat.interval.ms", "1000");
+        test_conf_set(conf, "auto.offset.reset", "earliest");
+        test_conf_set(conf, "enable.auto.commit", "false");
+
+        c = test_create_consumer(groupid, test_rebalance_cb, conf, NULL);
+
+        test_consumer_subscribe(c, topic);
+
+        /* Consume a couple of messages so that we have something to commit */
+        test_consumer_poll("consume", c, 0, -1, 0, 10, NULL);
+
+        /* Make the coordinator unreachable, this will cause a local session
+         * timeout followed by a revoke and assignment lost. */
+        rd_kafka_mock_broker_set_down(mcluster, 1);
+
+        /* Wait until the assignment is lost */
+        TEST_SAY("Waiting for assignment to be lost...\n");
+        while (!rd_kafka_assignment_lost(c))
+                rd_sleep(1);
+
+        TEST_SAY("Assignment is lost, committing\n");
+        /* Perform manual commit */
+        err = rd_kafka_commit(c, NULL, 0 /*sync*/);
+        TEST_SAY("commit() returned: %s\n", rd_kafka_err2name(err));
+        TEST_ASSERT(err, "expected commit to fail");
+
+        test_consumer_close(c);
+
+        rd_kafka_destroy(c);
+
+        test_mock_cluster_destroy(mcluster);
+
+        test_curr->is_fatal_cb = NULL;
+
+        SUB_TEST_PASS();
+}
+
+
+int main_0106_cgrp_sess_timeout(int argc, char **argv) {
+
+        TEST_SKIP_MOCK_CLUSTER(0);
+
+        do_test_session_timeout("sync");
+        do_test_session_timeout("async");
+        do_test_session_timeout("auto");
+
+        /* With KIP-848 session timeout is remote only.
+         * Partitions will stay assigned and fetchable
+         * when there's no communication with the
+         * coordinator, but the messages won't be
+         * committed.
+         * TODO: see if session timeout can be received by
+         * the broker and be enforced locally too. */
+        if (test_consumer_group_protocol_classic()) {
+                do_test_commit_on_lost();
+        }
+
+        return 0;
+}

package/deps/librdkafka/tests/0107-topic_recreate.c

@@ -0,0 +1,259 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2020-2022, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+#include "../src/rdkafka_proto.h"
+
+
+/**
+ * @name Verify that producer and consumer resumes operation after
+ *       a topic has been deleted and recreated.
+ */
+
+/**
+ * The message value to produce, one of:
+ *   "before"  - before topic deletion
+ *   "during"  - during topic deletion
+ *   "after"   - after topic has been re-created
+ *   "end"     - stop producing
+ */
+static mtx_t value_mtx;
+static char *value;
+
+static const int msg_rate = 10; /**< Messages produced per second */
+
+static struct test *this_test; /**< Exposes current test struct (in TLS) to
+                                *   producer thread. */
+
+
+/**
+ * @brief Treat all error_cb as non-test-fatal.
+ */
+static int
+is_error_fatal(rd_kafka_t *rk, rd_kafka_resp_err_t err, const char *reason) {
+        return rd_false;
+}
+
+/**
+ * @brief Producing thread
+ */
+static int run_producer(void *arg) {
+        const char *topic    = arg;
+        rd_kafka_t *producer = test_create_producer();
+        int ret              = 0;
+
+        test_curr = this_test;
+
+        /* Don't check message status */
+        test_curr->exp_dr_status = (rd_kafka_msg_status_t)-1;
+
+        while (1) {
+                rd_kafka_resp_err_t err;
+
+                mtx_lock(&value_mtx);
+                if (!strcmp(value, "end")) {
+                        mtx_unlock(&value_mtx);
+                        break;
+                } else if (strcmp(value, "before")) {
+                        /* Ignore Delivery report errors after topic
+                         * has been deleted and eventually re-created,
+                         * we rely on the consumer to verify that
+                         * messages are produced. */
+                        test_curr->ignore_dr_err = rd_true;
+                }
+
+                err = rd_kafka_producev(
+                    producer, RD_KAFKA_V_TOPIC(topic),
+                    RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+                    RD_KAFKA_V_VALUE(value, strlen(value)), RD_KAFKA_V_END);
+
+                if (err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART ||
+                    err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC)
+                        TEST_SAY("Produce failed (expectedly): %s\n",
+                                 rd_kafka_err2name(err));
+                else
+                        TEST_ASSERT(!err, "producev() failed: %s",
+                                    rd_kafka_err2name(err));
+
+                mtx_unlock(&value_mtx);
+
+                rd_usleep(1000000 / msg_rate, NULL);
+
+                rd_kafka_poll(producer, 0);
+        }
+
+        if (rd_kafka_flush(producer, 5000)) {
+                TEST_WARN("Failed to flush all message(s), %d remain\n",
+                          rd_kafka_outq_len(producer));
+                /* Purge the messages to see which partition they were for */
+                rd_kafka_purge(producer, RD_KAFKA_PURGE_F_QUEUE |
+                                             RD_KAFKA_PURGE_F_INFLIGHT);
+                rd_kafka_flush(producer, 5000);
+                TEST_SAY("%d message(s) in queue after purge\n",
+                         rd_kafka_outq_len(producer));
+
+                ret = 1; /* Fail test from main thread */
+        }
+
+        rd_kafka_destroy(producer);
+
+        return ret;
+}
+
+
+/**
+ * @brief Expect at least \p cnt messages with value matching \p exp_value,
+ *        else fail the current test.
+ */
+static void
+expect_messages(rd_kafka_t *consumer, int cnt, const char *exp_value) {
+        int match_cnt = 0, other_cnt = 0, err_cnt = 0;
+        size_t exp_len = strlen(exp_value);
+
+        TEST_SAY("Expecting >= %d messages with value \"%s\"...\n", cnt,
+                 exp_value);
+
+        while (match_cnt < cnt) {
+                rd_kafka_message_t *rkmessage;
+
+                rkmessage = rd_kafka_consumer_poll(consumer, 1000);
+                if (!rkmessage)
+                        continue;
+
+                if (rkmessage->err) {
+                        TEST_SAY("Consume error: %s\n",
+                                 rd_kafka_message_errstr(rkmessage));
+                        err_cnt++;
+                } else if (rkmessage->len == exp_len &&
+                           !memcmp(rkmessage->payload, exp_value, exp_len)) {
+                        match_cnt++;
+                } else {
+                        TEST_SAYL(3,
+                                  "Received \"%.*s\", expected \"%s\": "
+                                  "ignored\n",
+                                  (int)rkmessage->len,
+                                  (const char *)rkmessage->payload, exp_value);
+                        other_cnt++;
+                }
+
+                rd_kafka_message_destroy(rkmessage);
+        }
+
+        TEST_SAY(
+            "Consumed %d messages matching \"%s\", "
+            "ignored %d others, saw %d error(s)\n",
+            match_cnt, exp_value, other_cnt, err_cnt);
+}
+
+
+/**
+ * @brief Test topic create + delete + create with first topic having
+ *        \p part_cnt_1 partitions and second topic having \p part_cnt_2 .
+ */
+static void do_test_create_delete_create(int part_cnt_1, int part_cnt_2) {
+        rd_kafka_t *consumer;
+        thrd_t producer_thread;
+        const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+        int ret           = 0;
+
+        TEST_SAY(_C_MAG
+                 "[ Test topic create(%d parts)+delete+create(%d parts) ]\n",
+                 part_cnt_1, part_cnt_2);
+
+        consumer = test_create_consumer(topic, NULL, NULL, NULL);
+
+        /* Create topic */
+        test_create_topic_wait_exists(consumer, topic, part_cnt_1, 3, 5000);
+
+        /* Start consumer */
+        test_consumer_subscribe(consumer, topic);
+        test_consumer_wait_assignment(consumer, rd_true);
+
+        mtx_lock(&value_mtx);
+        value = "before";
+        mtx_unlock(&value_mtx);
+
+        /* Create producer thread */
+        if (thrd_create(&producer_thread, run_producer, (void *)topic) !=
+            thrd_success)
+                TEST_FAIL("thrd_create failed");
+
+        /* Consume messages for 5s */
+        expect_messages(consumer, msg_rate * 5, value);
+
+        /* Delete topic */
+        mtx_lock(&value_mtx);
+        value = "during";
+        mtx_unlock(&value_mtx);
+
+        test_delete_topic(consumer, topic);
+        rd_sleep(5);
+
+        /* Re-create topic */
+        test_create_topic_wait_exists(consumer, topic, part_cnt_2, 3, 5000);
+
+        mtx_lock(&value_mtx);
+        value = "after";
+        mtx_unlock(&value_mtx);
+
+        /* Consume for 5 more seconds, should see new messages */
+        expect_messages(consumer, msg_rate * 5, value);
+
+        rd_kafka_destroy(consumer);
+
+        /* Wait for producer to exit */
+        mtx_lock(&value_mtx);
+        value = "end";
+        mtx_unlock(&value_mtx);
+
+        if (thrd_join(producer_thread, &ret) != thrd_success || ret != 0)
+                TEST_FAIL("Producer failed: see previous errors");
+
+        TEST_SAY(_C_GRN
+                 "[ Test topic create(%d parts)+delete+create(%d parts): "
+                 "PASS ]\n",
+                 part_cnt_1, part_cnt_2);
+}
+
+
+int main_0107_topic_recreate(int argc, char **argv) {
+        this_test = test_curr; /* Need to expose current test struct (in TLS)
+                                * to producer thread. */
+
+        this_test->is_fatal_cb = is_error_fatal;
+
+        mtx_init(&value_mtx, mtx_plain);
+
+        test_conf_init(NULL, NULL, 60);
+
+        do_test_create_delete_create(10, 3);
+        do_test_create_delete_create(3, 6);
+
+        return 0;
+}